diff --git a/python/private/builders.bzl b/python/private/builders.bzl index 50aa3ed91a..bf5dbb8667 100644 --- a/python/private/builders.bzl +++ b/python/private/builders.bzl @@ -96,6 +96,145 @@ def _DepsetBuilder_build(self): kwargs["order"] = self._order[0] return depset(direct = self.direct, transitive = self.transitive, **kwargs) +def _Optional(*initial): + """A wrapper for a re-assignable value that may or may not be set. + + This allows structs to have attributes that aren't inherently mutable + and must be re-assigned to have their value updated. + + Args: + *initial: A single vararg to be the initial value, or no args + to leave it unset. + + Returns: + {type}`Optional` + """ + if len(initial) > 1: + fail("Only zero or one positional arg allowed") + + # buildifier: disable=uninitialized + self = struct( + _value = list(initial), + present = lambda *a, **k: _Optional_present(self, *a, **k), + set = lambda *a, **k: _Optional_set(self, *a, **k), + get = lambda *a, **k: _Optional_get(self, *a, **k), + ) + return self + +def _Optional_set(self, value): + """Sets the value of the optional. + + Args: + self: implicitly added + value: the value to set. + """ + if len(self._value) == 0: + self._value.append(value) + else: + self._value[0] = value + +def _Optional_get(self): + """Gets the value of the optional, or error. + + Args: + self: implicitly added + + Returns: + The stored value, or error if not set. + """ + if not len(self._value): + fail("Value not present") + return self._value[0] + +def _Optional_present(self): + """Tells if a value is present. + + Args: + self: implicitly added + + Returns: + {type}`bool` True if the value is set, False if not. + """ + return len(self._value) > 0 + +def _RuleBuilder(implementation = None, **kwargs): + """Builder for creating rules. + + Args: + implementation: {type}`callable` The rule implementation function. + **kwargs: The same as the `rule()` function, but using builders + for the non-mutable Bazel objects. + """ + + # buildifier: disable=uninitialized + self = struct( + attrs = dict(kwargs.pop("attrs", None) or {}), + cfg = kwargs.pop("cfg", None) or _TransitionBuilder(), + exec_groups = dict(kwargs.pop("exec_groups", None) or {}), + executable = _Optional(), + fragments = list(kwargs.pop("fragments", None) or []), + implementation = _Optional(implementation), + extra_kwargs = kwargs, + provides = list(kwargs.pop("provides", None) or []), + test = _Optional(), + toolchains = list(kwargs.pop("toolchains", None) or []), + build = lambda *a, **k: _RuleBuilder_build(self, *a, **k), + to_kwargs = lambda *a, **k: _RuleBuilder_to_kwargs(self, *a, **k), + ) + if "test" in kwargs: + self.test.set(kwargs.pop("test")) + if "executable" in kwargs: + self.executable.set(kwargs.pop("executable")) + return self + +def _RuleBuilder_build(self, debug = ""): + """Builds a `rule` object + + Args: + self: implicitly added + debug: {type}`str` If set, prints the args used to create the rule. + + Returns: + {type}`rule` + """ + kwargs = self.to_kwargs() + if debug: + lines = ["=" * 80, "rule kwargs: {}:".format(debug)] + for k, v in sorted(kwargs.items()): + lines.append(" {}={}".format(k, v)) + print("\n".join(lines)) # buildifier: disable=print + return rule(**kwargs) + +def _RuleBuilder_to_kwargs(self): + """Builds the arguments for calling `rule()`. + + Args: + self: implicitly added + + Returns: + {type}`dict` + """ + kwargs = {} + if self.executable.present(): + kwargs["executable"] = self.executable.get() + if self.test.present(): + kwargs["test"] = self.test.get() + + kwargs.update( + implementation = self.implementation.get(), + cfg = self.cfg.build() if self.cfg.implementation.present() else None, + attrs = { + k: (v.build() if hasattr(v, "build") else v) + for k, v in self.attrs.items() + }, + exec_groups = self.exec_groups, + fragments = self.fragments, + provides = self.provides, + toolchains = self.toolchains, + ) + kwargs.update(self.extra_kwargs) + return kwargs + def _RunfilesBuilder(): """Creates a `RunfilesBuilder`. @@ -177,6 +316,91 @@ def _RunfilesBuilder_build(self, ctx, **kwargs): **kwargs ).merge_all(self.runfiles) +def _SetBuilder(initial = None): + """Builder for list of unique values. + + Args: + initial: {type}`list | None` The initial values. + + Returns: + {type}`SetBuilder` + """ + initial = {} if not initial else {v: None for v in initial} + + # buildifier: disable=uninitialized + self = struct( + # TODO - Switch this to use set() builtin when available + # https://bazel.build/rules/lib/core/set + _values = initial, + update = lambda *a, **k: _SetBuilder_update(self, *a, **k), + build = lambda *a, **k: _SetBuilder_build(self, *a, **k), + ) + return self + +def _SetBuilder_build(self): + """Builds the values into a list + + Returns: + {type}`list` + """ + return self._values.keys() + +def _SetBuilder_update(self, *others): + """Adds values to the builder. + + Args: + self: implicitly added + *others: {type}`list` values to add to the set. + """ + for other in others: + for value in other: + if value not in self._values: + self._values[value] = None + +def _TransitionBuilder(implementation = None, inputs = None, outputs = None, **kwargs): + """Builder for transition objects. + + Args: + implementation: {type}`callable` the transition implementation function. + inputs: {type}`list[str]` the inputs for the transition. + outputs: {type}`list[str]` the outputs of the transition. + **kwargs: Extra keyword args to use when building. + + Returns: + {type}`TransitionBuilder` + """ + + # buildifier: disable=uninitialized + self = struct( + implementation = _Optional(implementation), + # Bazel requires transition.inputs to have unique values, so use set + # semantics so extenders of a transition can easily add/remove values. + # TODO - Use set builtin instead of custom builder, when available. + # https://bazel.build/rules/lib/core/set + inputs = _SetBuilder(inputs), + # Bazel requires transition.inputs to have unique values, so use set + # semantics so extenders of a transition can easily add/remove values. + # TODO - Use set builtin instead of custom builder, when available. + # https://bazel.build/rules/lib/core/set + outputs = _SetBuilder(outputs), + extra_kwargs = kwargs, + build = lambda *a, **k: _TransitionBuilder_build(self, *a, **k), + ) + return self + +def _TransitionBuilder_build(self): + """Creates a transition from the builder. + + Returns: + {type}`transition` + """ + return transition( + implementation = self.implementation.get(), + inputs = self.inputs.build(), + outputs = self.outputs.build(), + **self.extra_kwargs + ) + # Skylib's types module doesn't have is_file, so roll our own def _is_file(value): return type(value) == "File" @@ -187,4 +411,8 @@ def _is_runfiles(value): builders = struct( DepsetBuilder = _DepsetBuilder, RunfilesBuilder = _RunfilesBuilder, + RuleBuilder = _RuleBuilder, + TransitionBuilder = _TransitionBuilder, + SetBuilder = _SetBuilder, + Optional = _Optional, ) diff --git a/python/private/py_binary_macro.bzl b/python/private/py_binary_macro.bzl index d1269f2321..fa10f2e8a3 100644 --- a/python/private/py_binary_macro.bzl +++ b/python/private/py_binary_macro.bzl @@ -17,5 +17,8 @@ load(":py_binary_rule.bzl", py_binary_rule = "py_binary") load(":py_executable.bzl", "convert_legacy_create_init_to_int") def py_binary(**kwargs): + py_binary_macro(py_binary_rule, **kwargs) + +def py_binary_macro(py_rule, **kwargs): convert_legacy_create_init_to_int(kwargs) - py_binary_rule(**kwargs) + py_rule(**kwargs) diff --git a/python/private/py_binary_rule.bzl b/python/private/py_binary_rule.bzl index f1c8eb1325..5b40f52198 100644 --- a/python/private/py_binary_rule.bzl +++ b/python/private/py_binary_rule.bzl @@ -13,15 +13,14 @@ # limitations under the License. """Rule implementation of py_binary for Bazel.""" -load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_BINARY_ATTRS") load( ":py_executable.bzl", - "create_executable_rule", + "create_executable_rule_builder", "py_executable_impl", ) -_PY_TEST_ATTRS = { +_COVERAGE_ATTRS = { # Magic attribute to help C++ coverage work. There's no # docs about this; see TestActionBuilder.java "_collect_cc_coverage": attr.label( @@ -45,8 +44,13 @@ def _py_binary_impl(ctx): inherited_environment = [], ) -py_binary = create_executable_rule( - implementation = _py_binary_impl, - attrs = dicts.add(AGNOSTIC_BINARY_ATTRS, _PY_TEST_ATTRS), - executable = True, -) +def create_binary_rule_builder(): + builder = create_executable_rule_builder( + implementation = _py_binary_impl, + executable = True, + ) + builder.attrs.update(AGNOSTIC_BINARY_ATTRS) + builder.attrs.update(_COVERAGE_ATTRS) + return builder + +py_binary = create_binary_rule_builder().build() diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 18a7a707fc..2b2bf6636a 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -1747,16 +1747,6 @@ def _transition_executable_impl(input_settings, attr): settings[_PYTHON_VERSION_FLAG] = attr.python_version return settings -_transition_executable = transition( - implementation = _transition_executable_impl, - inputs = [ - _PYTHON_VERSION_FLAG, - ], - outputs = [ - _PYTHON_VERSION_FLAG, - ], -) - def create_executable_rule(*, attrs, **kwargs): return create_base_executable_rule( attrs = attrs, @@ -1764,33 +1754,33 @@ def create_executable_rule(*, attrs, **kwargs): **kwargs ) -def create_base_executable_rule(*, attrs, fragments = [], **kwargs): +def create_base_executable_rule(): """Create a function for defining for Python binary/test targets. - Args: - attrs: Rule attributes - fragments: List of str; extra config fragments that are required. - **kwargs: Additional args to pass onto `rule()` - Returns: A rule function """ - if "py" not in fragments: - # The list might be frozen, so use concatentation - fragments = fragments + ["py"] - kwargs.setdefault("provides", []).append(PyExecutableInfo) - kwargs["exec_groups"] = REQUIRED_EXEC_GROUPS | (kwargs.get("exec_groups") or {}) - kwargs.setdefault("cfg", _transition_executable) - return rule( - # TODO: add ability to remove attrs, i.e. for imports attr - attrs = dicts.add(EXECUTABLE_ATTRS, attrs), + return create_executable_rule_builder().build() + +def create_executable_rule_builder(implementation, **kwargs): + builder = builders.RuleBuilder( + implementation = implementation, + attrs = EXECUTABLE_ATTRS, + exec_groups = REQUIRED_EXEC_GROUPS, + fragments = ["py", "bazel_py"], + provides = [PyExecutableInfo], toolchains = [ TOOLCHAIN_TYPE, config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN_TYPE, mandatory = False), ] + _CC_TOOLCHAINS, - fragments = fragments, + cfg = builders.TransitionBuilder( + implementation = _transition_executable_impl, + inputs = [_PYTHON_VERSION_FLAG], + outputs = [_PYTHON_VERSION_FLAG], + ), **kwargs ) + return builder def cc_configure_features( ctx, diff --git a/python/private/py_test_macro.bzl b/python/private/py_test_macro.bzl index 348e877225..028dee6678 100644 --- a/python/private/py_test_macro.bzl +++ b/python/private/py_test_macro.bzl @@ -17,5 +17,8 @@ load(":py_executable.bzl", "convert_legacy_create_init_to_int") load(":py_test_rule.bzl", py_test_rule = "py_test") def py_test(**kwargs): + py_test_macro(py_test_rule, **kwargs) + +def py_test_macro(py_rule, **kwargs): convert_legacy_create_init_to_int(kwargs) - py_test_rule(**kwargs) + py_rule(**kwargs) diff --git a/python/private/py_test_rule.bzl b/python/private/py_test_rule.bzl index 63000c7255..6ad4fbddb8 100644 --- a/python/private/py_test_rule.bzl +++ b/python/private/py_test_rule.bzl @@ -13,12 +13,11 @@ # limitations under the License. """Implementation of py_test rule.""" -load("@bazel_skylib//lib:dicts.bzl", "dicts") load(":attributes.bzl", "AGNOSTIC_TEST_ATTRS") load(":common.bzl", "maybe_add_test_execution_info") load( ":py_executable.bzl", - "create_executable_rule", + "create_executable_rule_builder", "py_executable_impl", ) @@ -48,8 +47,13 @@ def _py_test_impl(ctx): maybe_add_test_execution_info(providers, ctx) return providers -py_test = create_executable_rule( - implementation = _py_test_impl, - attrs = dicts.add(AGNOSTIC_TEST_ATTRS, _BAZEL_PY_TEST_ATTRS), - test = True, -) +def create_test_rule_builder(): + builder = create_executable_rule_builder( + implementation = _py_test_impl, + test = True, + ) + builder.attrs.update(AGNOSTIC_TEST_ATTRS) + builder.attrs.update(_BAZEL_PY_TEST_ATTRS) + return builder + +py_test = create_test_rule_builder().build() diff --git a/tests/support/sh_py_run_test.bzl b/tests/support/sh_py_run_test.bzl index 4fa53ebd66..a1da285864 100644 --- a/tests/support/sh_py_run_test.bzl +++ b/tests/support/sh_py_run_test.bzl @@ -18,162 +18,77 @@ without the overhead of a bazel-in-bazel integration test. """ load("@rules_shell//shell:sh_test.bzl", "sh_test") -load("//python:py_binary.bzl", "py_binary") -load("//python:py_test.bzl", "py_test") +load("//python/private:py_binary_macro.bzl", "py_binary_macro") # buildifier: disable=bzl-visibility +load("//python/private:py_binary_rule.bzl", "create_binary_rule_builder") # buildifier: disable=bzl-visibility +load("//python/private:py_test_macro.bzl", "py_test_macro") # buildifier: disable=bzl-visibility +load("//python/private:py_test_rule.bzl", "create_test_rule_builder") # buildifier: disable=bzl-visibility load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") # buildifier: disable=bzl-visibility load("//tests/support:support.bzl", "VISIBLE_FOR_TESTING") -def _perform_transition_impl(input_settings, attr): - settings = dict(input_settings) +def _perform_transition_impl(input_settings, attr, base_impl): + settings = {k: input_settings[k] for k in _RECONFIG_INHERITED_OUTPUTS if k in input_settings} + settings.update(base_impl(input_settings, attr)) + settings[VISIBLE_FOR_TESTING] = True settings["//command_line_option:build_python_zip"] = attr.build_python_zip if attr.bootstrap_impl: settings["//python/config_settings:bootstrap_impl"] = attr.bootstrap_impl if attr.extra_toolchains: settings["//command_line_option:extra_toolchains"] = attr.extra_toolchains - if attr.python_version: - settings["//python/config_settings:python_version"] = attr.python_version if attr.venvs_use_declare_symlink: settings["//python/config_settings:venvs_use_declare_symlink"] = attr.venvs_use_declare_symlink return settings -_perform_transition = transition( - implementation = _perform_transition_impl, - inputs = [ - "//python/config_settings:bootstrap_impl", - "//command_line_option:extra_toolchains", - "//python/config_settings:python_version", - "//python/config_settings:venvs_use_declare_symlink", - ], - outputs = [ - "//command_line_option:build_python_zip", - "//command_line_option:extra_toolchains", - "//python/config_settings:bootstrap_impl", - "//python/config_settings:python_version", - "//python/config_settings:venvs_use_declare_symlink", - VISIBLE_FOR_TESTING, - ], -) - -def _py_reconfig_impl(ctx): - default_info = ctx.attr.target[DefaultInfo] - exe_ext = default_info.files_to_run.executable.extension - if exe_ext: - exe_ext = "." + exe_ext - exe_name = ctx.label.name + exe_ext - - executable = ctx.actions.declare_file(exe_name) - ctx.actions.symlink(output = executable, target_file = default_info.files_to_run.executable) - - default_outputs = [executable] - - # todo: could probably check target.owner vs src.owner to check if it should - # be symlinked or included as-is - # For simplicity of implementation, we're assuming the target being run is - # py_binary-like. In order for Windows to work, we need to make sure the - # file that the .exe launcher runs (the .zip or underlying non-exe - # executable) is a sibling of the .exe file with the same base name. - for src in default_info.files.to_list(): - if src.extension in ("", "zip"): - ext = ("." if src.extension else "") + src.extension - output = ctx.actions.declare_file(ctx.label.name + ext) - ctx.actions.symlink(output = output, target_file = src) - default_outputs.append(output) - - return [ - DefaultInfo( - executable = executable, - files = depset(default_outputs), - # On windows, the other default outputs must also be included - # in runfiles so the exe launcher can find the backing file. - runfiles = ctx.runfiles(default_outputs).merge( - default_info.default_runfiles, - ), - ), - ctx.attr.target[OutputGroupInfo], - # Inherit the expanded environment from the inner target. - ctx.attr.target[RunEnvironmentInfo], - ] - -def _make_reconfig_rule(**kwargs): - attrs = { - "bootstrap_impl": attr.string(), - "build_python_zip": attr.string(default = "auto"), - "extra_toolchains": attr.string_list( - doc = """ +_RECONFIG_INPUTS = [ + "//python/config_settings:bootstrap_impl", + "//command_line_option:extra_toolchains", + "//python/config_settings:venvs_use_declare_symlink", +] +_RECONFIG_OUTPUTS = _RECONFIG_INPUTS + [ + "//command_line_option:build_python_zip", + VISIBLE_FOR_TESTING, +] +_RECONFIG_INHERITED_OUTPUTS = [v for v in _RECONFIG_OUTPUTS if v in _RECONFIG_INPUTS] + +_RECONFIG_ATTRS = { + "bootstrap_impl": attr.string(), + "build_python_zip": attr.string(default = "auto"), + "extra_toolchains": attr.string_list( + doc = """ Value for the --extra_toolchains flag. NOTE: You'll likely have to also specify //tests/support/cc_toolchains:all (or some CC toolchain) to make the RBE presubmits happy, which disable auto-detection of a CC toolchain. """, - ), - "python_version": attr.string(), - "target": attr.label(executable = True, cfg = "target"), - "venvs_use_declare_symlink": attr.string(), - "_allowlist_function_transition": attr.label( - default = "@bazel_tools//tools/allowlists/function_transition_allowlist", - ), - } - return rule( - implementation = _py_reconfig_impl, - attrs = attrs, - cfg = _perform_transition, - **kwargs - ) + ), + "venvs_use_declare_symlink": attr.string(), +} -_py_reconfig_binary = _make_reconfig_rule(executable = True) - -_py_reconfig_test = _make_reconfig_rule(test = True) - -def _py_reconfig_executable(*, name, py_reconfig_rule, py_inner_rule, **kwargs): - reconfig_only_kwarg_names = [ - # keep sorted - "bootstrap_impl", - "build_python_zip", - "extra_toolchains", - "python_version", - "venvs_use_declare_symlink", - ] - reconfig_kwargs = { - key: kwargs.pop(key, None) - for key in reconfig_only_kwarg_names - } - reconfig_kwargs["target_compatible_with"] = kwargs.get("target_compatible_with") - - inner_name = "_{}_inner".format(name) - py_reconfig_rule( - name = name, - target = inner_name, - **reconfig_kwargs - ) - py_inner_rule( - name = inner_name, - tags = ["manual"], - **kwargs - ) +def _create_reconfig_rule(builder): + builder.attrs.update(_RECONFIG_ATTRS) + + base_cfg_impl = builder.cfg.implementation.get() + builder.cfg.implementation.set(lambda *args: _perform_transition_impl(base_impl = base_cfg_impl, *args)) + builder.cfg.inputs.update(_RECONFIG_INPUTS) + builder.cfg.outputs.update(_RECONFIG_OUTPUTS) + + return builder.build() + +_py_reconfig_binary = _create_reconfig_rule(create_binary_rule_builder()) -def py_reconfig_test(*, name, **kwargs): +_py_reconfig_test = _create_reconfig_rule(create_test_rule_builder()) + +def py_reconfig_test(**kwargs): """Create a py_test with customized build settings for testing. Args: - name: str, name of teset target. - **kwargs: kwargs to pass along to _py_reconfig_test and py_test. + **kwargs: kwargs to pass along to _py_reconfig_test. """ - _py_reconfig_executable( - name = name, - py_reconfig_rule = _py_reconfig_test, - py_inner_rule = py_test, - **kwargs - ) + py_test_macro(_py_reconfig_test, **kwargs) -def py_reconfig_binary(*, name, **kwargs): - _py_reconfig_executable( - name = name, - py_reconfig_rule = _py_reconfig_binary, - py_inner_rule = py_binary, - **kwargs - ) +def py_reconfig_binary(**kwargs): + py_binary_macro(_py_reconfig_binary, **kwargs) def sh_py_run_test(*, name, sh_src, py_src, **kwargs): """Run a py_binary within a sh_test. @@ -196,26 +111,12 @@ def sh_py_run_test(*, name, sh_src, py_src, **kwargs): "BIN_RLOCATION": "$(rlocationpaths {})".format(bin_name), }, ) - - py_binary_kwargs = { - key: kwargs.pop(key) - for key in ("imports", "deps", "env") - if key in kwargs - } - - _py_reconfig_binary( + py_reconfig_binary( name = bin_name, - tags = ["manual"], - target = "_{}_plain_bin".format(name), - **kwargs - ) - - py_binary( - name = "_{}_plain_bin".format(name), srcs = [py_src], main = py_src, tags = ["manual"], - **py_binary_kwargs + **kwargs ) def _current_build_settings_impl(ctx): diff --git a/tests/toolchains/python_toolchain_test.py b/tests/toolchains/python_toolchain_test.py index 371b252a4a..591d7dbe8a 100644 --- a/tests/toolchains/python_toolchain_test.py +++ b/tests/toolchains/python_toolchain_test.py @@ -1,6 +1,7 @@ import json import os import pathlib +import pprint import sys import unittest @@ -18,7 +19,13 @@ def test_expected_toolchain_matches(self): settings = json.loads(pathlib.Path(settings_path).read_text()) expected = "python_{}".format(expect_version.replace(".", "_")) - self.assertIn(expected, settings["toolchain_label"], str(settings)) + msg = ( + "Expected toolchain not found\n" + + f"Expected toolchain label to contain: {expected}\n" + + "Actual build settings:\n" + + pprint.pformat(settings) + ) + self.assertIn(expected, settings["toolchain_label"], msg) actual = "{v.major}.{v.minor}.{v.micro}".format(v=sys.version_info) self.assertEqual(actual, expect_version)