diff --git a/CHANGELOG.md b/CHANGELOG.md index da775748f0..8f97eef933 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,9 @@ Unreleased changes template. ([#1169](https://github.com/bazelbuild/rules_python/issues/1169)). * (gazelle) Don't collapse depsets to a list or into args when generating the modules mapping file. Support spilling modules mapping args into a params file. +* (pypi) From now on `python` invocations in repository and module extension + evaluation contexts will invoke Python interpreter with `-B` to avoid + creating `.pyc` files. {#v0-0-0-added} ### Added diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index ec5f576945..028657f716 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -55,12 +55,12 @@ def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpre pypi_repo_utils.execute_checked( mrctx, op = "ResolveRequirementEnvMarkers({})".format(in_file), + python = pypi_repo_utils.resolve_python_interpreter( + mrctx, + python_interpreter = python_interpreter, + python_interpreter_target = python_interpreter_target, + ), arguments = [ - pypi_repo_utils.resolve_python_interpreter( - mrctx, - python_interpreter = python_interpreter, - python_interpreter_target = python_interpreter_target, - ), "-m", "python.private.pypi.requirements_parser.resolve_target_platforms", in_file, diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index a7da224321..c839f2e4d6 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -27,8 +27,8 @@ other patches ensures that the users have overview on exactly what has changed within the wheel. """ -load("//python/private:repo_utils.bzl", "repo_utils") load(":parse_whl_name.bzl", "parse_whl_name") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") _rules_python_root = Label("//:BUILD.bazel") @@ -102,10 +102,14 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): record_patch = rctx.path("RECORD.patch") whl_patched = patched_whl_name(whl_input.basename) - repo_utils.execute_checked( + pypi_repo_utils.execute_checked( rctx, + python = python_interpreter, + srcs = [ + Label("//python/private/pypi:repack_whl.py"), + Label("//tools:wheelmaker.py"), + ], arguments = [ - python_interpreter, "-m", "python.private.pypi.repack_whl", "--record-patch", diff --git a/python/private/pypi/pypi_repo_utils.bzl b/python/private/pypi/pypi_repo_utils.bzl index 196431636f..bb2acc850a 100644 --- a/python/private/pypi/pypi_repo_utils.bzl +++ b/python/private/pypi/pypi_repo_utils.bzl @@ -104,11 +104,30 @@ def _construct_pypath(mrctx, *, entries): ]) return pypath -def _execute_checked(mrctx, *, srcs, **kwargs): +def _execute_prep(mrctx, *, python, srcs, **kwargs): + for src in srcs: + # This will ensure that we will re-evaluate the bzlmod extension or + # refetch the repository_rule when the srcs change. This should work on + # Bazel versions without `mrctx.watch` as well. + repo_utils.watch(mrctx, mrctx.path(src)) + + environment = kwargs.pop("environment", {}) + pythonpath = environment.get("PYTHONPATH", "") + if pythonpath and not types.is_string(pythonpath): + environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) + kwargs["environment"] = environment + + # -B is added to prevent the repo-phase invocation from creating timestamp + # based pyc files, which contributes to race conditions and non-determinism + kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", []) + return kwargs + +def _execute_checked(mrctx, *, python, srcs, **kwargs): """Helper function to run a python script and modify the PYTHONPATH to include external deps. Args: mrctx: Handle to the module_ctx or repository_ctx. + python: The python interpreter to use. srcs: The src files that the script depends on. This is important to ensure that the Bazel repository cache or the bzlmod lock file gets invalidated when any one file changes. It is advisable to use @@ -118,26 +137,34 @@ def _execute_checked(mrctx, *, srcs, **kwargs): the `environment` has a value `PYTHONPATH` and it is a list, then it will be passed to `construct_pythonpath` function. """ + return repo_utils.execute_checked( + mrctx, + **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs) + ) - for src in srcs: - # This will ensure that we will re-evaluate the bzlmod extension or - # refetch the repository_rule when the srcs change. This should work on - # Bazel versions without `mrctx.watch` as well. - repo_utils.watch(mrctx, mrctx.path(src)) - - env = kwargs.pop("environment", {}) - pythonpath = env.get("PYTHONPATH", "") - if pythonpath and not types.is_string(pythonpath): - env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) +def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs): + """Helper function to run a python script and modify the PYTHONPATH to include external deps. - return repo_utils.execute_checked( + Args: + mrctx: Handle to the module_ctx or repository_ctx. + python: The python interpreter to use. + srcs: The src files that the script depends on. This is important to + ensure that the Bazel repository cache or the bzlmod lock file gets + invalidated when any one file changes. It is advisable to use + `RECORD` files for external deps and the list of srcs from the + rules_python repo for any scripts. + **kwargs: Arguments forwarded to `repo_utils.execute_checked`. If + the `environment` has a value `PYTHONPATH` and it is a list, then + it will be passed to `construct_pythonpath` function. + """ + return repo_utils.execute_checked_stdout( mrctx, - environment = env, - **kwargs + **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs) ) pypi_repo_utils = struct( construct_pythonpath = _construct_pypath, execute_checked = _execute_checked, + execute_checked_stdout = _execute_checked_stdout, resolve_python_interpreter = _resolve_python_interpreter, ) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index ef4077fa41..bdcf7849ad 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -75,14 +75,15 @@ def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None): if not is_standalone_interpreter(rctx, python_interpreter, logger = logger): return [] - stdout = repo_utils.execute_checked_stdout( + stdout = pypi_repo_utils.execute_checked_stdout( rctx, op = "GetPythonVersionForUnixCflags", + python = python_interpreter, arguments = [ - python_interpreter, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", ], + srcs = [], ) _python_version = stdout include_path = "{}/include/python{}".format( @@ -181,7 +182,6 @@ def _whl_library_impl(rctx): python_interpreter_target = rctx.attr.python_interpreter_target, ) args = [ - python_interpreter, "-m", "python.private.pypi.whl_installer.wheel_installer", "--requirement", @@ -247,6 +247,7 @@ def _whl_library_impl(rctx): # truncate the requirement value when logging it / reporting # progress since it may contain several ' --hash=sha256:... # --hash=sha256:...' substrings that fill up the console + python = python_interpreter, op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]), arguments = args, environment = environment, @@ -295,6 +296,7 @@ def _whl_library_impl(rctx): pypi_repo_utils.execute_checked( rctx, op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), + python = python_interpreter, arguments = args + [ "--whl-file", whl_path,