From dea649392fb9ef5480d7ef34acb9e8adb326a811 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 1 Mar 2025 18:10:11 +0900 Subject: [PATCH 1/5] fix(pypi): use python -B for all invocations Before this change we would just invoke the Python interpreter. This means that in the `rules_python` directory there would be `__pycache__` folders created in the source tree and the same `__pycache__` folders would be created in the python interpreter repository rules if the directories were writable. This change ensures that we are executing `python` with `-B` in those contexts and reduces any likelihood of us doing the wrong thing. Work towards #1169. --- CHANGELOG.md | 3 ++ python/private/pypi/evaluate_markers.bzl | 10 ++--- python/private/pypi/patch_whl.bzl | 10 +++-- python/private/pypi/pypi_repo_utils.bzl | 52 +++++++++++++++++------- python/private/pypi/whl_library.bzl | 8 ++-- 5 files changed, 58 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e447012c98..0bbd2a8e89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -109,6 +109,9 @@ Unreleased changes template. {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at build time (they will be created at runtime instead). (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489)) +* (pypi) From now on `python` invocations in repository and module extension + evaluation contexts will invoke Python interpreter with `-B` to avoid + creating `.pyc` files. {#v1-2-0-added} ### Added diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index ec5f576945..028657f716 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -55,12 +55,12 @@ def evaluate_markers(mrctx, *, requirements, python_interpreter, python_interpre pypi_repo_utils.execute_checked( mrctx, op = "ResolveRequirementEnvMarkers({})".format(in_file), + python = pypi_repo_utils.resolve_python_interpreter( + mrctx, + python_interpreter = python_interpreter, + python_interpreter_target = python_interpreter_target, + ), arguments = [ - pypi_repo_utils.resolve_python_interpreter( - mrctx, - python_interpreter = python_interpreter, - python_interpreter_target = python_interpreter_target, - ), "-m", "python.private.pypi.requirements_parser.resolve_target_platforms", in_file, diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index a7da224321..84fc6c74f3 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -27,8 +27,8 @@ other patches ensures that the users have overview on exactly what has changed within the wheel. """ -load("//python/private:repo_utils.bzl", "repo_utils") load(":parse_whl_name.bzl", "parse_whl_name") +load(":pypi_repo_utils.bzl", "pypi_repo_utils") _rules_python_root = Label("//:BUILD.bazel") @@ -102,10 +102,14 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): record_patch = rctx.path("RECORD.patch") whl_patched = patched_whl_name(whl_input.basename) - repo_utils.execute_checked( + pypi_repo_utils.execute_checked( rctx, + python = python_interpreter, + srcs = [ + "//python/private/pypi:repack_whl.py", + "//tools:wheelmaker.py", + ], arguments = [ - python_interpreter, "-m", "python.private.pypi.repack_whl", "--record-patch", diff --git a/python/private/pypi/pypi_repo_utils.bzl b/python/private/pypi/pypi_repo_utils.bzl index 196431636f..f634b84733 100644 --- a/python/private/pypi/pypi_repo_utils.bzl +++ b/python/private/pypi/pypi_repo_utils.bzl @@ -104,11 +104,27 @@ def _construct_pypath(mrctx, *, entries): ]) return pypath -def _execute_checked(mrctx, *, srcs, **kwargs): +def _execute_prep(mrctx, *, python, srcs, **kwargs): + for src in srcs: + # This will ensure that we will re-evaluate the bzlmod extension or + # refetch the repository_rule when the srcs change. This should work on + # Bazel versions without `mrctx.watch` as well. + repo_utils.watch(mrctx, mrctx.path(src)) + + environment = kwargs.pop("environment", {}) + pythonpath = environment.get("PYTHONPATH", "") + if pythonpath and not types.is_string(pythonpath): + environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) + kwargs["environment"] = environment + kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", []) + return kwargs + +def _execute_checked(mrctx, *, python, srcs, **kwargs): """Helper function to run a python script and modify the PYTHONPATH to include external deps. Args: mrctx: Handle to the module_ctx or repository_ctx. + python: The python interpreter to use. srcs: The src files that the script depends on. This is important to ensure that the Bazel repository cache or the bzlmod lock file gets invalidated when any one file changes. It is advisable to use @@ -118,26 +134,34 @@ def _execute_checked(mrctx, *, srcs, **kwargs): the `environment` has a value `PYTHONPATH` and it is a list, then it will be passed to `construct_pythonpath` function. """ + return repo_utils.execute_checked( + mrctx, + **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs) + ) - for src in srcs: - # This will ensure that we will re-evaluate the bzlmod extension or - # refetch the repository_rule when the srcs change. This should work on - # Bazel versions without `mrctx.watch` as well. - repo_utils.watch(mrctx, mrctx.path(src)) - - env = kwargs.pop("environment", {}) - pythonpath = env.get("PYTHONPATH", "") - if pythonpath and not types.is_string(pythonpath): - env["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) +def _execute_checked_stdout(mrctx, *, python, srcs, **kwargs): + """Helper function to run a python script and modify the PYTHONPATH to include external deps. - return repo_utils.execute_checked( + Args: + mrctx: Handle to the module_ctx or repository_ctx. + python: The python interpreter to use. + srcs: The src files that the script depends on. This is important to + ensure that the Bazel repository cache or the bzlmod lock file gets + invalidated when any one file changes. It is advisable to use + `RECORD` files for external deps and the list of srcs from the + rules_python repo for any scripts. + **kwargs: Arguments forwarded to `repo_utils.execute_checked`. If + the `environment` has a value `PYTHONPATH` and it is a list, then + it will be passed to `construct_pythonpath` function. + """ + return repo_utils.execute_checked_stdout( mrctx, - environment = env, - **kwargs + **_execute_prep(mrctx, python = python, srcs = srcs, **kwargs) ) pypi_repo_utils = struct( construct_pythonpath = _construct_pypath, execute_checked = _execute_checked, + execute_checked_stdout = _execute_checked_stdout, resolve_python_interpreter = _resolve_python_interpreter, ) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index ef4077fa41..bdcf7849ad 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -75,14 +75,15 @@ def _get_toolchain_unix_cflags(rctx, python_interpreter, logger = None): if not is_standalone_interpreter(rctx, python_interpreter, logger = logger): return [] - stdout = repo_utils.execute_checked_stdout( + stdout = pypi_repo_utils.execute_checked_stdout( rctx, op = "GetPythonVersionForUnixCflags", + python = python_interpreter, arguments = [ - python_interpreter, "-c", "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}', end='')", ], + srcs = [], ) _python_version = stdout include_path = "{}/include/python{}".format( @@ -181,7 +182,6 @@ def _whl_library_impl(rctx): python_interpreter_target = rctx.attr.python_interpreter_target, ) args = [ - python_interpreter, "-m", "python.private.pypi.whl_installer.wheel_installer", "--requirement", @@ -247,6 +247,7 @@ def _whl_library_impl(rctx): # truncate the requirement value when logging it / reporting # progress since it may contain several ' --hash=sha256:... # --hash=sha256:...' substrings that fill up the console + python = python_interpreter, op = op_tmpl.format(name = rctx.attr.name, requirement = rctx.attr.requirement.split(" ", 1)[0]), arguments = args, environment = environment, @@ -295,6 +296,7 @@ def _whl_library_impl(rctx): pypi_repo_utils.execute_checked( rctx, op = "whl_library.ExtractWheel({}, {})".format(rctx.attr.name, whl_path), + python = python_interpreter, arguments = args + [ "--whl-file", whl_path, From 08513f84c71838d01e07fa144a6e952933fc2b8f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 3 Mar 2025 23:18:15 +0900 Subject: [PATCH 2/5] Apply suggestions from code review --- python/private/pypi/patch_whl.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/patch_whl.bzl b/python/private/pypi/patch_whl.bzl index 84fc6c74f3..c839f2e4d6 100644 --- a/python/private/pypi/patch_whl.bzl +++ b/python/private/pypi/patch_whl.bzl @@ -106,8 +106,8 @@ def patch_whl(rctx, *, python_interpreter, whl_path, patches, **kwargs): rctx, python = python_interpreter, srcs = [ - "//python/private/pypi:repack_whl.py", - "//tools:wheelmaker.py", + Label("//python/private/pypi:repack_whl.py"), + Label("//tools:wheelmaker.py"), ], arguments = [ "-m", From 4760cf6321f0d2eaaafa796df296cd2f2b21a55f Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 3 Mar 2025 15:55:20 -0800 Subject: [PATCH 3/5] Update python/private/pypi/pypi_repo_utils.bzl --- python/private/pypi/pypi_repo_utils.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/private/pypi/pypi_repo_utils.bzl b/python/private/pypi/pypi_repo_utils.bzl index f634b84733..c69eefc14a 100644 --- a/python/private/pypi/pypi_repo_utils.bzl +++ b/python/private/pypi/pypi_repo_utils.bzl @@ -116,6 +116,8 @@ def _execute_prep(mrctx, *, python, srcs, **kwargs): if pythonpath and not types.is_string(pythonpath): environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) kwargs["environment"] = environment + # -B is added to prevent the repo-phase invocation from creating timestamp + # based pyc files, which contributes to race conditions and non-determinism kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", []) return kwargs From b75307a4c310428642cef0d1ed0ed67b1ad520d1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 7 Mar 2025 01:37:22 +0000 Subject: [PATCH 4/5] buildifier --- python/private/pypi/pypi_repo_utils.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/python/private/pypi/pypi_repo_utils.bzl b/python/private/pypi/pypi_repo_utils.bzl index c69eefc14a..bb2acc850a 100644 --- a/python/private/pypi/pypi_repo_utils.bzl +++ b/python/private/pypi/pypi_repo_utils.bzl @@ -116,6 +116,7 @@ def _execute_prep(mrctx, *, python, srcs, **kwargs): if pythonpath and not types.is_string(pythonpath): environment["PYTHONPATH"] = _construct_pypath(mrctx, entries = pythonpath) kwargs["environment"] = environment + # -B is added to prevent the repo-phase invocation from creating timestamp # based pyc files, which contributes to race conditions and non-determinism kwargs["arguments"] = [python, "-B"] + kwargs.get("arguments", []) From 88fa51d622a273b5166c5885e3d5641920b1d5de Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 7 Mar 2025 01:38:16 +0000 Subject: [PATCH 5/5] move the changelog entry --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f0ca57511..8f97eef933 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,9 @@ Unreleased changes template. ([#1169](https://github.com/bazelbuild/rules_python/issues/1169)). * (gazelle) Don't collapse depsets to a list or into args when generating the modules mapping file. Support spilling modules mapping args into a params file. +* (pypi) From now on `python` invocations in repository and module extension + evaluation contexts will invoke Python interpreter with `-B` to avoid + creating `.pyc` files. {#v0-0-0-added} ### Added @@ -117,9 +120,6 @@ Unreleased changes template. {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at build time (they will be created at runtime instead). (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489)) -* (pypi) From now on `python` invocations in repository and module extension - evaluation contexts will invoke Python interpreter with `-B` to avoid - creating `.pyc` files. {#v1-2-0-added} ### Added