diff --git a/CHANGELOG.md b/CHANGELOG.md index abe718c389..7aeb135788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,10 @@ Unreleased changes template. allow specifying links to create within the venv site packages (only applicable with {obj}`--bootstrap_impl=script`) ([#2156](https://github.com/bazelbuild/rules_python/issues/2156)). +* (toolchains) Local Python installs can be used to create a toolchain + equivalent to the standard toolchains. See [Local toolchains] docs for how to + configure them. + {#v0-0-0-removed} ### Removed diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 29eac6e714..25da682012 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -108,6 +108,7 @@ sphinx_stardocs( "//python/cc:py_cc_toolchain_bzl", "//python/cc:py_cc_toolchain_info_bzl", "//python/entry_points:py_console_script_binary_bzl", + "//python/local_toolchains:repos_bzl", "//python/private:attr_builders_bzl", "//python/private:builders_util_bzl", "//python/private:py_binary_rule_bzl", diff --git a/docs/toolchains.md b/docs/toolchains.md index 73a8a48121..5cd9eb268e 100644 --- a/docs/toolchains.md +++ b/docs/toolchains.md @@ -199,10 +199,10 @@ Remember to call `use_repo()` to make repos visible to your module: :::{deprecated} 1.1.0 -The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. +The toolchain specific `py_binary` and `py_test` symbols are aliases to the regular rules. i.e. Deprecated `load("@python_versions//3.11:defs.bzl", "py_binary")` & `load("@python_versions//3.11:defs.bzl", "py_test")` -Usages of them should be changed to load the regular rules directly; +Usages of them should be changed to load the regular rules directly; i.e. Use `load("@rules_python//python:py_binary.bzl", "py_binary")` & `load("@rules_python//python:py_test.bzl", "py_test")` and then specify the `python_version` when using the rules corresponding to the python version you defined in your toolchain. {ref}`Library modules with version constraints` ::: @@ -327,7 +327,97 @@ After registration, your Python targets will use the toolchain's interpreter dur is still used to 'bootstrap' Python targets (see https://github.com/bazel-contrib/rules_python/issues/691). You may also find some quirks while using this toolchain. Please refer to [python-build-standalone documentation's _Quirks_ section](https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html). -## Autodetecting toolchain +## Local toolchain + +It's possible to use a locally installed Python runtime instead of the regular +prebuilt, remotely downloaded ones. A local toolchain contains the Python +runtime metadata (Python version, headers, ABI flags, etc) that the regular +remotely downloaded runtimes contain, which makes it possible to build e.g. C +extensions (unlike the autodetecting and runtime environment toolchains). + +For simple cases, some rules are provided that will introspect +a Python installation and create an appropriate Bazel definition from +it. To do this, three pieces need to be wired together: + +1. Specify a path or command to a Python interpreter (multiple can be defined). +2. Create toolchains for the runtimes in (1) +3. Register the toolchains created by (2) + +The below is an example that will use `python3` from PATH to find the +interpreter, then introspect its installation to generate a full toolchain. + +```starlark +# File: MODULE.bazel + +local_runtime_repo = use_repo_rule( + "@rules_python//python/local_toolchains:repos.bzl", + "local_runtime_repo", + dev_dependency = True, +) + +local_runtime_toolchains_repo = use_repo_rule( + "@rules_python//python/local_toolchains:repos.bzl" + "local_runtime_toolchains_repo" + dev_dependency = True, +) + +# Step 1: Define the Python runtime +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", +) + +# Step 2: Create toolchains for the runtimes +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +# Step 3: Register the toolchains +register_toolchains("@local_toolchains//:all", dev_dependency = True) +``` + +Note that `register_toolchains` will insert the local toolchain earlier in the +toolchain ordering, so it will take precedence over other registered toolchains. + +:::{important} +Be sure to set `dev_dependency = True`. Using a local toolchain only makes sense +for the root module. + +If an intermediate module does it, then the `register_toolchains()` call will +take precedence over the default rules_python toolchains and cause problems for +downstream modules. +::: + +Multiple runtimes and/or toolchains can be defined, which allows for multiple +Python versions and/or platforms to be configured in a single `MODULE.bazel`. + +## Runtime environment toolchain + +The runtime environment toolchain is a minimal toolchain that doesn't provide +information about Python at build time. In particular, this means it is not able +to build C extensions -- doing so requires knowing, at build time, what Python +headers to use. + +In effect, all it does is generate a small wrapper script that simply calls e.g. +`/usr/bin/env python3` to run a program. This makes it easy to change what +Python is used to run a program, but also makes it easy to use a Python version +that isn't compatible with build-time assumptions. + +``` +register_toolchains("@rules_python//python/runtime_env_toolchains:all") +``` + +Note that this toolchain has no constraints, i.e. it will match any platform, +Python version, etc. + +:::{seealso} +[Local toolchain], which creates a more full featured toolchain from a +locally installed Python. +::: + +### Autodetecting toolchain The autodetecting toolchain is a deprecated toolchain that is built into Bazel. It's name is a bit misleading: it doesn't autodetect anything. All it does is @@ -345,7 +435,6 @@ To aid migration off the Bazel-builtin toolchain, rules_python provides {bzl:obj}`@rules_python//python/runtime_env_toolchains:all`. This is an equivalent toolchain, but is implemented using rules_python's objects. - ## Custom toolchains While rules_python provides toolchains by default, it is not required to use diff --git a/python/BUILD.bazel b/python/BUILD.bazel index a699c81cc4..3389a0dacc 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -41,6 +41,7 @@ filegroup( "//python/constraints:distribution", "//python/entry_points:distribution", "//python/extensions:distribution", + "//python/local_toolchains:distribution", "//python/pip_install:distribution", "//python/private:distribution", "//python/runfiles:distribution", diff --git a/python/local_toolchains/BUILD.bazel b/python/local_toolchains/BUILD.bazel new file mode 100644 index 0000000000..211f3e21a7 --- /dev/null +++ b/python/local_toolchains/BUILD.bazel @@ -0,0 +1,18 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package(default_visibility = ["//:__subpackages__"]) + +bzl_library( + name = "repos_bzl", + srcs = ["repos.bzl"], + visibility = ["//visibility:public"], + deps = [ + "//python/private:local_runtime_repo_bzl", + "//python/private:local_runtime_toolchains_repo_bzl", + ], +) + +filegroup( + name = "distribution", + srcs = glob(["**"]), +) diff --git a/python/local_toolchains/repos.bzl b/python/local_toolchains/repos.bzl new file mode 100644 index 0000000000..d1b45cfd7f --- /dev/null +++ b/python/local_toolchains/repos.bzl @@ -0,0 +1,18 @@ +"""Rules/macros for repository phase for local toolchains. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +""" + +load( + "@rules_python//python/private:local_runtime_repo.bzl", + _local_runtime_repo = "local_runtime_repo", +) +load( + "@rules_python//python/private:local_runtime_toolchains_repo.bzl", + _local_runtime_toolchains_repo = "local_runtime_toolchains_repo", +) + +local_runtime_repo = _local_runtime_repo + +local_runtime_toolchains_repo = _local_runtime_toolchains_repo diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index ef4580e1ce..b63f446be3 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -205,6 +205,24 @@ bzl_library( ], ) +bzl_library( + name = "local_runtime_repo_bzl", + srcs = ["local_runtime_repo.bzl"], + deps = [ + ":enum_bzl", + ":repo_utils.bzl", + ], +) + +bzl_library( + name = "local_runtime_toolchains_repo_bzl", + srcs = ["local_runtime_toolchains_repo.bzl"], + deps = [ + ":repo_utils.bzl", + ":text_util_bzl", + ], +) + bzl_library( name = "normalize_name_bzl", srcs = ["normalize_name.bzl"], diff --git a/tests/integration/local_toolchains/MODULE.bazel b/tests/integration/local_toolchains/MODULE.bazel index d4ef12e952..98f1ed9ac4 100644 --- a/tests/integration/local_toolchains/MODULE.bazel +++ b/tests/integration/local_toolchains/MODULE.bazel @@ -19,9 +19,9 @@ local_path_override( path = "../../..", ) -local_runtime_repo = use_repo_rule("@rules_python//python/private:local_runtime_repo.bzl", "local_runtime_repo") +local_runtime_repo = use_repo_rule("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_repo") -local_runtime_toolchains_repo = use_repo_rule("@rules_python//python/private:local_runtime_toolchains_repo.bzl", "local_runtime_toolchains_repo") +local_runtime_toolchains_repo = use_repo_rule("@rules_python//python/local_toolchains:repos.bzl", "local_runtime_toolchains_repo") local_runtime_repo( name = "local_python3",