From 3e15ccafec0c5fcb6ee9352eb53d36ea73d829a1 Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Wed, 8 Oct 2025 12:26:28 +1000 Subject: [PATCH 1/3] feat: add basic support for Python in gen-build-spec Signed-off-by: behnazh-w --- src/macaron/__main__.py | 6 +- .../build_spec_generator.py | 70 ++- .../common_spec/__init__.py | 2 + .../common_spec/base_spec.py | 89 ++++ .../build_spec_generator/common_spec/core.py | 454 ++++++++++++++++++ .../{ => common_spec}/jdk_finder.py | 0 .../jdk_version_normalizer.py | 0 .../common_spec/maven_spec.py | 69 +++ .../common_spec/pypi_spec.py | 36 ++ .../reproducible_central.py | 446 ++--------------- src/macaron/slsa_analyzer/analyzer.py | 17 +- .../build_tool/base_build_tool.py | 34 ++ .../slsa_analyzer/build_tool/language.py | 2 +- .../common_spec/test_core.py | 143 ++++++ .../test_reproducible_central.py | 192 +++----- .../test_jdk_version_finder.py | 2 +- .../test_jdk_version_normalizer.py | 2 +- .../expected_macaron.buildspec | 26 +- .../expected_reproducible_central.buildspec | 19 + .../test.yaml | 12 +- ...> expected_reproducible_central.buildspec} | 0 .../test.yaml | 4 +- 22 files changed, 1016 insertions(+), 609 deletions(-) create mode 100644 src/macaron/build_spec_generator/common_spec/__init__.py create mode 100644 src/macaron/build_spec_generator/common_spec/base_spec.py create mode 100644 src/macaron/build_spec_generator/common_spec/core.py rename src/macaron/build_spec_generator/{ => common_spec}/jdk_finder.py (100%) rename src/macaron/build_spec_generator/{ => common_spec}/jdk_version_normalizer.py (100%) create mode 100644 src/macaron/build_spec_generator/common_spec/maven_spec.py create mode 100644 src/macaron/build_spec_generator/common_spec/pypi_spec.py create mode 100644 tests/build_spec_generator/common_spec/test_core.py create mode 100644 tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec rename tests/integration/cases/micronaut-projects_micronaut-core/{expected_macaron.buildspec => expected_reproducible_central.buildspec} (100%) diff --git a/src/macaron/__main__.py b/src/macaron/__main__.py index a312afd1d..c887a721e 100644 --- a/src/macaron/__main__.py +++ b/src/macaron/__main__.py @@ -618,8 +618,10 @@ def main(argv: list[str] | None = None) -> None: gen_build_spec_parser.add_argument( "--output-format", type=str, - help=('The output format. Can be rc-buildspec (Reproducible-central build spec) (default "rc-buildspec")'), - default="rc-buildspec", + help=( + "The output format. Can be default-buildspec (default) or rc-buildspec (Reproducible-central build spec)" + ), + default="default-buildspec", ) args = main_parser.parse_args(argv) diff --git a/src/macaron/build_spec_generator/build_spec_generator.py b/src/macaron/build_spec_generator/build_spec_generator.py index dd1217cd0..856969685 100644 --- a/src/macaron/build_spec_generator/build_spec_generator.py +++ b/src/macaron/build_spec_generator/build_spec_generator.py @@ -3,6 +3,7 @@ """This module contains the functions used for generating build specs from the Macaron database.""" +import json import logging import os from collections.abc import Mapping @@ -13,8 +14,10 @@ from sqlalchemy.orm import Session from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType +from macaron.build_spec_generator.common_spec.core import gen_generic_build_spec from macaron.build_spec_generator.reproducible_central.reproducible_central import gen_reproducible_central_build_spec from macaron.console import access_handler +from macaron.errors import GenerateBuildSpecError from macaron.path_utils.purl_based_path import get_purl_based_dir logger: logging.Logger = logging.getLogger(__name__) @@ -25,6 +28,8 @@ class BuildSpecFormat(str, Enum): REPRODUCIBLE_CENTRAL = "rc-buildspec" + DEFAULT = "default-buildspec" + CLI_COMMAND_PATCHES: dict[ PatchCommandBuildTool, @@ -96,14 +101,38 @@ def gen_build_spec_for_purl( db_engine = create_engine(f"sqlite+pysqlite:///file:{database_path}?mode=ro&uri=true", echo=False) build_spec_content = None + build_spec_dir_path = os.path.join( + output_path, + "buildspec", + get_purl_based_dir( + purl_name=purl.name, + purl_namespace=purl.namespace, + purl_type=purl.type, + ), + ) + with Session(db_engine) as session, session.begin(): + try: + build_spec = gen_generic_build_spec(purl=purl, session=session, patches=CLI_COMMAND_PATCHES) + except GenerateBuildSpecError as error: + logger.error("Error while generating the build spec: %s.", error) + return os.EX_DATAERR match build_spec_format: case BuildSpecFormat.REPRODUCIBLE_CENTRAL: - build_spec_content = gen_reproducible_central_build_spec( - purl=purl, - session=session, - patches=CLI_COMMAND_PATCHES, - ) + try: + build_spec_content = gen_reproducible_central_build_spec(build_spec) + except GenerateBuildSpecError as error: + logger.error("Error while generating the build spec: %s.", error) + return os.EX_DATAERR + build_spec_file_path = os.path.join(build_spec_dir_path, "reproducible_central.buildspec") + # Default build spec. + case BuildSpecFormat.DEFAULT: + try: + build_spec_content = json.dumps(build_spec) + except ValueError as error: + logger.error("Error while serializing the build spec: %s.", error) + return os.EX_DATAERR + build_spec_file_path = os.path.join(build_spec_dir_path, "macaron.buildspec") if not build_spec_content: logger.error("Error while generating the build spec.") @@ -111,36 +140,29 @@ def gen_build_spec_for_purl( logger.debug("Build spec content: \n%s", build_spec_content) - build_spec_filepath = os.path.join( - output_path, - "buildspec", - get_purl_based_dir( - purl_name=purl.name, - purl_namespace=purl.namespace, - purl_type=purl.type, - ), - "macaron.buildspec", - ) - - os.makedirs( - name=os.path.dirname(build_spec_filepath), - exist_ok=True, - ) + try: + os.makedirs( + name=build_spec_dir_path, + exist_ok=True, + ) + except OSError as error: + logger.error("Unable to create the output file: %s.", error) + return os.EX_OSERR logger.info( - "Generating the %s format build spec to %s.", + "Generating the %s format build spec to %s", build_spec_format.value, - os.path.relpath(build_spec_filepath, os.getcwd()), + os.path.relpath(build_spec_file_path, os.getcwd()), ) rich_handler = access_handler.get_handler() rich_handler.update_gen_build_spec("Build Spec Path:", os.path.relpath(build_spec_filepath, os.getcwd())) try: - with open(build_spec_filepath, mode="w", encoding="utf-8") as file: + with open(build_spec_file_path, mode="w", encoding="utf-8") as file: file.write(build_spec_content) except OSError as error: logger.error( "Could not create the build spec at %s. Error: %s", - os.path.relpath(build_spec_filepath, os.getcwd()), + os.path.relpath(build_spec_file_path, os.getcwd()), error, ) return os.EX_OSERR diff --git a/src/macaron/build_spec_generator/common_spec/__init__.py b/src/macaron/build_spec_generator/common_spec/__init__.py new file mode 100644 index 000000000..8e17a3508 --- /dev/null +++ b/src/macaron/build_spec_generator/common_spec/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. diff --git a/src/macaron/build_spec_generator/common_spec/base_spec.py b/src/macaron/build_spec_generator/common_spec/base_spec.py new file mode 100644 index 000000000..9fe4e0e0d --- /dev/null +++ b/src/macaron/build_spec_generator/common_spec/base_spec.py @@ -0,0 +1,89 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module includes base build specification and helper classes.""" + +from abc import ABC, abstractmethod +from typing import NotRequired, Required, TypedDict + +from packageurl import PackageURL + + +class BaseBuildSpecDict(TypedDict, total=False): + """ + Initialize base build specification. + + It supports multiple languages, build tools, and additional metadata for enhanced traceability. + """ + + #: The package ecosystem. + ecosystem: Required[str] + + #: The package identifier. + purl: Required[str] + + #: The programming language, e.g., 'java', 'python', 'javascript'. + language: Required[str] + + #: The build tool or package manager, e.g., 'maven', 'gradle', 'pip', 'poetry', 'npm', 'yarn'. + build_tool: Required[str] + + #: The version of Macaron used for generating the spec. + macaron_version: Required[str] + + #: The group identifier for the project/component. + group_id: NotRequired[str | None] + + #: The artifact identifier for the project/component. + artifact_id: Required[str] + + #: The version of the package or component. + version: Required[str] + + #: The remote path or URL of the git repository. + git_repo: NotRequired[str] + + #: The commit SHA or tag in the VCS repository. + git_tag: NotRequired[str] + + #: The type of line endings used (e.g., 'lf', 'crlf'). + newline: NotRequired[str] + + #: The version of the programming language or runtime, e.g., '11' for JDK, '3.11' for Python. + language_version: Required[str] + + #: List of release dependencies. + dependencies: NotRequired[list[str]] + + #: List of build dependencies, which includes tests. + build_dependencies: NotRequired[list[str]] + + #: List of shell commands to build the project. + build_commands: NotRequired[list[list[str]]] + + #: List of shell commands to test the project. + test_commands: NotRequired[list[str]] + + #: Environment variables required during build or test. + environment: NotRequired[dict[str, str]] + + #: Path or location of the build artifact/output. + artifact_path: NotRequired[str | None] + + #: Entry point script, class, or binary for running the project. + entry_point: NotRequired[str | None] + + +class BaseBuildSpec(ABC): + """Abstract base class for build specification behavior and field resolution.""" + + @abstractmethod + def resolve_fields(self, purl: PackageURL) -> None: + """ + Resolve fields that require special logic for a specific build ecosystem. + + Notes + ----- + This method should be implemented by subclasses to handle + logic specific to a given package ecosystem, such as Maven or PyPI. + """ diff --git a/src/macaron/build_spec_generator/common_spec/core.py b/src/macaron/build_spec_generator/common_spec/core.py new file mode 100644 index 000000000..e7f8610e3 --- /dev/null +++ b/src/macaron/build_spec_generator/common_spec/core.py @@ -0,0 +1,454 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the logic to generate a build spec in a generic format that can be transformed if needed.""" + +import logging +import pprint +import shlex +from collections.abc import Mapping, Sequence +from enum import Enum +from importlib import metadata as importlib_metadata +from pprint import pformat + +import sqlalchemy.orm +from packageurl import PackageURL + +from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType, patch_commands +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.maven_spec import MavenBuildSpec +from macaron.build_spec_generator.common_spec.pypi_spec import PyPIBuildSpec +from macaron.build_spec_generator.macaron_db_extractor import ( + GenericBuildCommandInfo, + lookup_any_build_command, + lookup_build_tools_check, + lookup_latest_component, +) +from macaron.errors import GenerateBuildSpecError, QueryMacaronDatabaseError +from macaron.slsa_analyzer.checks.build_tool_check import BuildToolFacts + +logger: logging.Logger = logging.getLogger(__name__) + + +class ECOSYSTEMS(Enum): + """This Enum provides implementation mappings for supported ecosystems.""" + + #: The Maven build specification. + MAVEN = MavenBuildSpec + #: The PyPI build specification. + PYPI = PyPIBuildSpec + + +class LANGUAGES(Enum): + """This Enum provides mappings for supported languages.""" + + #: The language used in the Maven build ecosystem. + MAVEN = "java" + #: The language used in the PyPI build ecosystem. + PYPI = "python" + + +class MacaronBuildToolName(str, Enum): + """Represent the name of a build tool that Macaron stores in the database. + + This doesn't cover all build tools that Macaron supports, and ONLY includes the ones that we + support generating build spec for. + """ + + MAVEN = "maven" + GRADLE = "gradle" + PIP = "pip" + POETRY = "poetry" + + +def format_build_command_info(build_command_info: list[GenericBuildCommandInfo]) -> str: + """Return the prettified str format for a list of `GenericBuildCommandInfo` instances. + + Parameters + ---------- + build_command_info: GenericBuildCommandInfo + A list of ``GenericBuildCommandInfo`` instances. + + Returns + ------- + str + The prettified output. + """ + pretty_formatted_ouput = [pprint.pformat(build_command_info) for build_command_info in build_command_info] + return "\n".join(pretty_formatted_ouput) + + +def remove_shell_quote(cmd: list[str]) -> list[str]: + """Remove shell quotes from a shell command. + + Parameters + ---------- + cmd: list[str] + The shell command as list[str]ing. + + Returns + ------- + list[str] + The shell command with all quote removed. + + Examples + -------- + >>> cmd = "mvn -f fit/core-reference/pom.xml verify '-Dit.test=RESTITCase' '-Dmodernizer.skip=true' '-Drat.skip=true'" + >>> remove_shell_quote(cmd.split()) + ['mvn', '-f', 'fit/core-reference/pom.xml', 'verify', '-Dit.test=RESTITCase', '-Dmodernizer.skip=true', '-Drat.skip=true'] + """ + return shlex.split(" ".join(cmd)) + + +def compose_shell_commands(cmds_sequence: list[list[str]]) -> str: + """ + Combine a sequence of command fragments into a single shell command suitable for a build spec. + + Parameters + ---------- + cmds_sequence : list[list[str]] + The sequence of build command fragments. + + Returns + ------- + str + A shell command string to be used in the build specification's command field. + """ + removed_shell_quote = [" ".join(remove_shell_quote(cmds)) for cmds in cmds_sequence] + result = " && ".join(removed_shell_quote) + return result + + +def get_default_build_command( + build_tool_name: MacaronBuildToolName, +) -> list[str] | None: + """Return a default build command for the build tool. + + Parameters + ---------- + build_tool_name: MacaronBuildToolName + The type of build tool to get the default build command. + + Returns + ------- + list[str] | None + The build command as a list[str] or None if we cannot get one for this tool. + """ + default_build_command = None + + match build_tool_name: + case MacaronBuildToolName.MAVEN: + default_build_command = "mvn clean package".split() + case MacaronBuildToolName.GRADLE: + default_build_command = "./gradlew clean assemble publishToMavenLocal".split() + case MacaronBuildToolName.PIP: + default_build_command = "python -m build".split() + case MacaronBuildToolName.POETRY: + default_build_command = "poetry build".split() + case _: + pass + + if not default_build_command: + logger.critical( + "There is no default build command available for the build tool %s.", + build_tool_name, + ) + return None + + return default_build_command + + +def get_macaron_build_tool_name( + build_tool_facts: Sequence[BuildToolFacts], target_language: str +) -> MacaronBuildToolName | None: + """ + Retrieve the Macaron build tool name for supported projects from the database facts. + + Iterates over the provided build tool facts and returns the first valid `MacaronBuildToolName` + for a supported language. If no valid build tool name is found, returns None. + + .. note:: + If multiple build tools are present in the database, only the first valid one encountered + in the sequence is returned. + + Parameters + ---------- + build_tool_facts : Sequence[BuildToolFacts] + A sequence of build tool fact records to be searched. + target_language: str + The target build language. + + Returns + ------- + MacaronBuildToolName or None + The corresponding Macaron build tool name if found, otherwise None. + """ + for fact in build_tool_facts: + if fact.language.lower() == target_language: + try: + macaron_build_tool_name = MacaronBuildToolName(fact.build_tool_name) + except ValueError: + continue + + # TODO: What happen if we report multiple build tools in the database? + return macaron_build_tool_name + + return None + + +def get_build_tool_name( + component_id: int, session: sqlalchemy.orm.Session, target_language: str +) -> MacaronBuildToolName | None: + """ + Retrieve the Macaron build tool name for a given component. + + Queries the database for build tool facts associated with the specified component ID + and returns the corresponding `MacaronBuildToolName` if found. If no valid build tool + information is available or an error occurs during the query, returns None. + + Parameters + ---------- + component_id : int + The ID of the component for which to retrieve the build tool name. + session : sqlalchemy.orm.Session + The SQLAlchemy session used to access the database. + target_language: str + The target build language. + + Returns + ------- + MacaronBuildToolName or None + The corresponding build tool name for the component if available, otherwise None. + """ + try: + build_tool_facts = lookup_build_tools_check( + component_id=component_id, + session=session, + ) + except QueryMacaronDatabaseError as lookup_build_tools_error: + logger.error( + "Unexpected result from querying build tools for component id %s. Error: %s", + component_id, + lookup_build_tools_error, + ) + return None + if not build_tool_facts: + logger.error( + "Cannot find any build tool for component id %s in the database.", + component_id, + ) + return None + logger.info( + "Build tools discovered from the %s table: %s", + BuildToolFacts.__tablename__, + [(fact.build_tool_name, fact.language) for fact in build_tool_facts], + ) + + return get_macaron_build_tool_name(build_tool_facts, target_language) + + +def get_build_command_info( + component_id: int, + session: sqlalchemy.orm.Session, +) -> GenericBuildCommandInfo | None: + """Return the highest confidence build command information from the database for a component. + + The build command is found by looking up CheckFacts for build-related checks. + + Parameters + ---------- + component_id: int + The id of the component we are finding the build command for. + session: sqlalchemy.orm.Session + The SQLAlchemy Session opened for the database to extract build information. + + Returns + ------- + GenericBuildCommandInfo | None + The GenericBuildCommandInfo object for the highest confidence build command; or None if there was + an error, or no build command is found from the database. + """ + try: + lookup_build_command_info = lookup_any_build_command(component_id, session) + except QueryMacaronDatabaseError as lookup_build_command_error: + logger.error( + "Unexpected result from querying all build command information for component id %s. Error: %s", + component_id, + lookup_build_command_error, + ) + return None + logger.debug( + "Build command information discovered\n%s", + format_build_command_info(lookup_build_command_info), + ) + + return lookup_build_command_info[0] if lookup_build_command_info else None + + +def get_language_version( + build_command_info: GenericBuildCommandInfo, +) -> str | None: + """Retrieve the language version from a GenericBuildCommandInfo object. + + If available, returns a language version from the `language_versions` list associated with + the provided GenericBuildCommandInfo object. Currently, this function returns the last + element in the list. If the list is empty, returns None. + + Notes + ----- + The selection of the last element from `language_versions` is a temporary strategy, + as more robust selection logic may be implemented in the future depending on + requirements for specific language/runtime versions (e.g., multiple JDK versions). + + Parameters + ---------- + build_command_info : GenericBuildCommandInfo + The object containing language version information. + + Returns + ------- + str | None + The selected language version as a string, or None if not available. + """ + if build_command_info.language_versions: + # There isn't a concrete reason why we select the last element. + # We just use this at this point because we haven't looked into + # a better way to select the jdk version obtained from the database. + return build_command_info.language_versions.pop() + + return None + + +def gen_generic_build_spec( + purl: PackageURL, + session: sqlalchemy.orm.Session, + patches: Mapping[ + PatchCommandBuildTool, + Mapping[str, PatchValueType | None], + ], +) -> BaseBuildSpecDict: + """ + Generate and return the Buildspec file. + + Parameters + ---------- + purl : PackageURL + The PackageURL to generate build spec for. + session : sqlalchemy.orm.Session + The SQLAlchemy Session opened for the database to extract build information. + patches : Mapping[PatchCommandBuildTool, Mapping[str, PatchValueType | None]] + The patches to apply to the build commands in ``build_info`` before being populated in + the output Buildspec. + + Returns + ------- + BaseBuildSpecDict + The generated build spec. + + Raises + ------ + GenerateBuildSpecError + Raised if generation of the build spec fails due to any of the following reasons: + 1. The input PURL is invalid. + 2. There is no supported build tool for this PURL. + 3. Failed to patch the build commands using the provided ``patches``. + 4. The database from ``session`` doesn't contain enough information. + + """ + if purl.type not in [e.name.lower() for e in ECOSYSTEMS]: + raise GenerateBuildSpecError( + f"PURL type '{purl.type}' is not supported. Supported: {[e.name.lower() for e in ECOSYSTEMS]}" + ) + + logger.debug( + "Generating build spec for %s with command patches:\n%s", + purl, + pformat(patches), + ) + + target_language = LANGUAGES[purl.type.upper()].value + group = purl.namespace + artifact = purl.name + version = purl.version + if version is None: + raise GenerateBuildSpecError(f"Missing version for purl {purl}.") + + try: + latest_component = lookup_latest_component( + purl=purl, + session=session, + ) + except QueryMacaronDatabaseError as lookup_component_error: + raise GenerateBuildSpecError( + f"Unexpected result from querying latest component for {purl}. " + ) from lookup_component_error + if not latest_component: + raise GenerateBuildSpecError( + f"Cannot find an analysis result for PackageURL {purl} in the database. " + "Please check if an analysis for it exists in the database." + ) + + latest_component_repository = latest_component.repository + if not latest_component_repository: + raise GenerateBuildSpecError(f"Cannot find any repository information for {purl} in the database.") + + logger.info( + "Repository information for purl %s: url %s, commit %s", + purl, + latest_component_repository.remote_path, + latest_component_repository.commit_sha, + ) + + build_tool_name = get_build_tool_name( + component_id=latest_component.id, session=session, target_language=target_language + ) + if not build_tool_name: + raise GenerateBuildSpecError(f"Failed to determine build tool for {purl}.") + + build_command_info = get_build_command_info( + component_id=latest_component.id, + session=session, + ) + logger.info( + "Attempted to find build command from the database. Result: %s", + build_command_info or "Cannot find any.", + ) + + selected_build_command = ( + build_command_info.command + if build_command_info + else get_default_build_command( + build_tool_name, + ) + ) + if not selected_build_command: + raise GenerateBuildSpecError(f"Failed to get a build command for {purl}.") + + patched_build_commands = patch_commands( + cmds_sequence=[selected_build_command], + patches=patches, + ) + if not patched_build_commands: + raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_command}.") + + lang_version = get_language_version(build_command_info) if build_command_info else "" + + base_build_spec_dict = BaseBuildSpecDict( + { + "macaron_version": importlib_metadata.version("macaron"), + "group_id": group, + "artifact_id": artifact, + "version": version, + "git_repo": latest_component_repository.remote_path, + "git_tag": latest_component_repository.commit_sha, + "newline": "lf", + "language_version": lang_version or "", + "ecosystem": purl.type, + "purl": str(purl), + "language": target_language, + "build_tool": build_tool_name.value, + "build_commands": patched_build_commands, + } + ) + ECOSYSTEMS[purl.type.upper()].value(base_build_spec_dict).resolve_fields(purl) + return base_build_spec_dict diff --git a/src/macaron/build_spec_generator/jdk_finder.py b/src/macaron/build_spec_generator/common_spec/jdk_finder.py similarity index 100% rename from src/macaron/build_spec_generator/jdk_finder.py rename to src/macaron/build_spec_generator/common_spec/jdk_finder.py diff --git a/src/macaron/build_spec_generator/jdk_version_normalizer.py b/src/macaron/build_spec_generator/common_spec/jdk_version_normalizer.py similarity index 100% rename from src/macaron/build_spec_generator/jdk_version_normalizer.py rename to src/macaron/build_spec_generator/common_spec/jdk_version_normalizer.py diff --git a/src/macaron/build_spec_generator/common_spec/maven_spec.py b/src/macaron/build_spec_generator/common_spec/maven_spec.py new file mode 100644 index 000000000..f08602f28 --- /dev/null +++ b/src/macaron/build_spec_generator/common_spec/maven_spec.py @@ -0,0 +1,69 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module includes build specification and helper classes for Maven packages.""" + + +import logging + +from packageurl import PackageURL + +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.jdk_finder import find_jdk_version_from_central_maven_repo +from macaron.build_spec_generator.common_spec.jdk_version_normalizer import normalize_jdk_version + +logger: logging.Logger = logging.getLogger(__name__) + + +class MavenBuildSpec(BaseBuildSpec): + """This class implements build spec inferences for Maven packages.""" + + def __init__(self, data: BaseBuildSpecDict): + """ + Initialize the object. + + Parameters + ---------- + data : BaseBuildSpecDict + The data object containing the build configuration fields. + """ + self.data = data + + def resolve_fields(self, purl: PackageURL) -> None: + """ + Resolve Maven-specific fields in the build specification. + + Parameters + ---------- + purl: str + The target software component Package URL. + """ + if purl.namespace is None or purl.version is None: + missing_fields = [] + if purl.namespace is None: + missing_fields.append("group ID (namespace)") + if purl.version is None: + missing_fields.append("version") + logger.error("Purl %s is missing required field(s): %s.", purl, ", ".join(missing_fields)) + return + + # We always attempt to get the JDK version from maven central JAR for this GAV artifact. + jdk_from_jar = find_jdk_version_from_central_maven_repo( + group_id=purl.namespace, + artifact_id=purl.name, + version=purl.version, + ) + logger.info( + "Attempted to find JDK from Maven Central JAR. Result: %s", + jdk_from_jar or "Cannot find any.", + ) + + # Select JDK from jar or another source, with a default of version 8. + selected_jdk_version = jdk_from_jar or self.data["language_version"] if self.data["language_version"] else "8" + + major_jdk_version = normalize_jdk_version(selected_jdk_version) + if not major_jdk_version: + logger.error("Failed to obtain the major version of %s", selected_jdk_version) + return + + self.data["language_version"] = major_jdk_version diff --git a/src/macaron/build_spec_generator/common_spec/pypi_spec.py b/src/macaron/build_spec_generator/common_spec/pypi_spec.py new file mode 100644 index 000000000..94489883c --- /dev/null +++ b/src/macaron/build_spec_generator/common_spec/pypi_spec.py @@ -0,0 +1,36 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module includes build specification and helper classes for PyPI packages.""" + + +from packageurl import PackageURL + +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpec, BaseBuildSpecDict + + +class PyPIBuildSpec( + BaseBuildSpec, +): + """This class implements build spec inferences for PyPI packages.""" + + def __init__(self, data: BaseBuildSpecDict): + """ + Initialize the object. + + Parameters + ---------- + data : BaseBuildSpecDict + The data object containing the build configuration fields. + """ + self.data = data + + def resolve_fields(self, purl: PackageURL) -> None: + """ + Resolve PyPI-specific fields in the build specification. + + Parameters + ---------- + purl: str + The target software component Package URL. + """ diff --git a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py index 305d42be4..df9a7b099 100644 --- a/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py +++ b/src/macaron/build_spec_generator/reproducible_central/reproducible_central.py @@ -4,28 +4,13 @@ """This module contains the logic to generate a build spec in the Reproducible Central format.""" import logging -import pprint -import shlex -from collections.abc import Mapping, Sequence from enum import Enum -from importlib import metadata as importlib_metadata -from pprint import pformat -import sqlalchemy.orm -from packageurl import PackageURL +import importlib_metadata -from macaron.build_spec_generator.build_command_patcher import PatchCommandBuildTool, PatchValueType, patch_commands -from macaron.build_spec_generator.jdk_finder import find_jdk_version_from_central_maven_repo -from macaron.build_spec_generator.jdk_version_normalizer import normalize_jdk_version -from macaron.build_spec_generator.macaron_db_extractor import ( - GenericBuildCommandInfo, - lookup_any_build_command, - lookup_build_tools_check, - lookup_latest_component, -) -from macaron.console import access_handler -from macaron.errors import QueryMacaronDatabaseError -from macaron.slsa_analyzer.checks.build_tool_check import BuildToolFacts +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.core import compose_shell_commands +from macaron.errors import GenerateBuildSpecError logger: logging.Logger = logging.getLogger(__name__) @@ -39,8 +24,7 @@ # - We only work with git repository and its commit hash. Therefore `gitRepo` and `gitTag` are used only. # Even though it's called gitTag, a commit hash would work. # https://github.com/jvm-repo-rebuild/reproducible-central/blob/46de9b405cb30ff94effe0ba47c1ebecc5a1c17e/bin/includes/fetchSource.sh#L59C1-L59C72 -STRING_TEMPLATE = """# Copyright (c) 2025, Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. +STRING_TEMPLATE = """ # Generated by Macaron version {macaron_version} groupId={group_id} @@ -62,17 +46,6 @@ """ -class _MacaronBuildToolName(str, Enum): - """Represents the name of a build tool that Macaron stores in the database. - - This doesn't cover all build tools that Macaron supports, and ONLY includes the ones that we - support generating Reproducible Central Buildspec for. - """ - - MAVEN = "maven" - GRADLE = "gradle" - - class ReproducibleCentralBuildTool(str, Enum): """Represent the name of the build tool used in the Reproducible Central's Buildspec. @@ -81,404 +54,49 @@ class ReproducibleCentralBuildTool(str, Enum): MAVEN = "mvn" GRADLE = "gradle" - SBT = "sbt" - - -def format_build_command_info(build_command_info: list[GenericBuildCommandInfo]) -> str: - """Return the prettified str format for a list of `GenericBuildCommandInfo` instances. - - Parameters - ---------- - build_command_info: GenericBuildCommandInfo - A list of ``GenericBuildCommandInfo`` instances. - - Returns - ------- - str - The prettified output. - """ - pretty_formatted_ouput = [pprint.pformat(build_command_info) for build_command_info in build_command_info] - return "\n".join(pretty_formatted_ouput) - - -def remove_shell_quote(cmd: list[str]) -> list[str]: - """Remove shell quotes from a shell command. - - Parameters - ---------- - cmd: list[str] - The shell command as list of string. - - Returns - ------- - list[str] - The shell command with all quote removed. - - Examples - -------- - >>> cmd = "mvn -f fit/core-reference/pom.xml verify '-Dit.test=RESTITCase' '-Dmodernizer.skip=true' '-Drat.skip=true'" - >>> remove_shell_quote(cmd.split()) - ['mvn', '-f', 'fit/core-reference/pom.xml', 'verify', '-Dit.test=RESTITCase', '-Dmodernizer.skip=true', '-Drat.skip=true'] - """ - return shlex.split(" ".join(cmd)) - - -def get_rc_build_command(cmds_sequence: list[list[str]]) -> str: - """Return a single command as a string to be used in RC buildspec from a sequence of commands. - - The build commands in the sequence will be ``&&`` together, because RC's build spec - is a shell script. - - Parameters - ---------- - cmds_sequence: list[list[str]] - The sequence of build commands. - - Returns - ------- - str - A bash command to be used in RC's command field. - """ - removed_shell_quote = [" ".join(remove_shell_quote(cmds)) for cmds in cmds_sequence] - result = " && ".join(removed_shell_quote) - return result - - -def get_rc_default_build_command( - rc_build_tool_name: ReproducibleCentralBuildTool, -) -> list[str] | None: - """Return a default build command for a type of Reproducible Central build tool type. - - Parameters - ---------- - rc_build_tool_name: ReproducibleCentralBuildTool - The type of build tool to get the default build command. - - Returns - ------- - list[str] | None - The build command as a list of strings or None if we cannot get one for this tool. - """ - default_build_command = None - - match rc_build_tool_name: - case ReproducibleCentralBuildTool.MAVEN: - default_build_command = "mvn clean package".split() - case ReproducibleCentralBuildTool.GRADLE: - default_build_command = "./gradlew clean assemble publishToMavenLocal".split() - case _: - pass - - if not default_build_command: - logger.critical( - "There is no default build command available for RC build tool %s.", - rc_build_tool_name, - ) - return None - - return default_build_command - -def _get_macaron_build_tool_name(build_tool_facts: Sequence[BuildToolFacts]) -> _MacaronBuildToolName | None: - """Return the build tool name reported by Macaron from the database.""" - for fact in build_tool_facts: - if fact.language in {"java"}: - try: - macaron_build_tool_name = _MacaronBuildToolName(fact.build_tool_name) - except ValueError: - continue - # TODO: What happen if we report multiple build tools in the database? - return macaron_build_tool_name - - return None - - -def _get_rc_build_tool_name_from_build_facts( - build_tool_facts: Sequence[BuildToolFacts], -) -> ReproducibleCentralBuildTool | None: - """Return the build tool name to be put into the RC buildspec from a sequence of BuildToolFacts instances.""" - macaron_build_tool_name = _get_macaron_build_tool_name(build_tool_facts) - if not macaron_build_tool_name: - logger.error( - "No supported build tool are found. Expect %s", - [build_tool.value for build_tool in _MacaronBuildToolName], - ) - return None - - match macaron_build_tool_name: - case _MacaronBuildToolName.MAVEN: - return ReproducibleCentralBuildTool.MAVEN - case _MacaronBuildToolName.GRADLE: - return ReproducibleCentralBuildTool.GRADLE - - -def get_rc_build_tool_name( - component_id: int, - session: sqlalchemy.orm.Session, -) -> ReproducibleCentralBuildTool | None: - """Return the ``ReproducibleCentralBuildTool`` instance corresponding to the build tool of the component. - - Parameters - ---------- - component_id: int - The id of the component we are finding build command for. - session: sqlalchemy.orm.Session - The SQLAlchemy Session opened for the database to extract build information. - - Returns - ------- - ReproducibleCentralBuildTool | None - The ``ReproducibleCentralBuildTool`` instance for this component. - """ - try: - build_tool_facts = lookup_build_tools_check( - component_id=component_id, - session=session, - ) - except QueryMacaronDatabaseError as lookup_build_tools_error: - logger.error( - "Unexpected result from querying build tools for component id %s. Error: %s", - component_id, - lookup_build_tools_error, - ) - return None - if not build_tool_facts: - logger.error( - "Cannot find any build tool for component id %s in the database.", - component_id, - ) - return None - logger.info( - "Build tools discovered from the %s table: %s", - BuildToolFacts.__tablename__, - [(fact.build_tool_name, fact.language) for fact in build_tool_facts], - ) - rich_handler = access_handler.get_handler() - rich_handler.update_gen_build_spec( - "Build Tools:", - "\n".join([f"{fact.build_tool_name} ({fact.language})" for fact in build_tool_facts]), - ) - - return _get_rc_build_tool_name_from_build_facts(build_tool_facts) - - -def get_lookup_build_command_info( - component_id: int, - session: sqlalchemy.orm.Session, -) -> GenericBuildCommandInfo | None: - """Return the highest confidence build command information from the database for a component. - - The build command is found by looking up CheckFacts for build-related checks. - - Parameters - ---------- - component_id: int - The id of the component we are finding the build command for. - session: sqlalchemy.orm.Session - The SQLAlchemy Session opened for the database to extract build information. - - Returns - ------- - GenericBuildCommandInfo | None - The GenericBuildCommandInfo object for the highest confidence build command; or None if there was - an error, or no build command is found from the database. - """ - try: - lookup_build_command_info = lookup_any_build_command(component_id, session) - except QueryMacaronDatabaseError as lookup_build_command_error: - logger.error( - "Unexpected result from querying all build command information for component id %s. Error: %s", - component_id, - lookup_build_command_error, - ) - return None - logger.debug( - "Build command information discovered\n%s", - format_build_command_info(lookup_build_command_info), - ) - - return lookup_build_command_info[0] if lookup_build_command_info else None - - -def get_lookup_build_command_jdk( - build_command_info: GenericBuildCommandInfo, -) -> str | None: - """Return the JDK version from a GenericBuildCommandInfo object.""" - if build_command_info.language_versions: - # There isn't a concrete reason why we select the last element. - # We just use this at this point because we haven't looked into - # a better way to select the jdk version obtained from the database. - return build_command_info.language_versions.pop() - - return None - - -def gen_reproducible_central_build_spec( - purl: PackageURL, - session: sqlalchemy.orm.Session, - patches: Mapping[ - PatchCommandBuildTool, - Mapping[str, PatchValueType | None], - ], -) -> str | None: - """Return the content of a Reproducible Central Buildspec File. +def gen_reproducible_central_build_spec(build_spec: BaseBuildSpecDict) -> str | None: + """Translate the build specification to ensure compatibility with Reproducible Central. The Reproducible Central Buildspec File Format can be found here: https://github.com/jvm-repo-rebuild/reproducible-central/blob/e1708dd8dde3cdbe66b0cec9948812b601e90ba6/doc/BUILDSPEC.md#format Parameters ---------- - purl: PackageURL - The PackageURL to generate build spec for. - session: sqlalchemy.orm.Session - The SQLAlchemy Session opened for the database to extract build information. - patches: Mapping[PatchCommandBuildTool, Mapping[str, PatchValueType | None]] - The patches to apply to the build commands in ``build_info`` before being populated in - the output Buildspec. + build_spec: BaseBuildSpecDict + The base build spec generated for the Maven artifact. Returns ------- - str | None - The content of the Buildspec as string or None if there is an error. - The errors that can happen are: 1. The input PURL is invalid, 2. There is no supported build tool - for this PURL, 3. Failed to patch the build commands using the provided ``patches``, 4. The database from - ``session`` doesn't contain enough information. - """ - logger.debug( - "Generating build spec for %s with command patches:\n%s", - purl, - pformat(patches), - ) - - # Getting groupid, artifactid and version from PURL. - group = purl.namespace - artifact = purl.name - version = purl.version - rich_handler = access_handler.get_handler() - rich_handler.update_gen_build_spec("Package URL:", purl.to_string()) - if group is None or version is None: - logger.error("Missing group and/or version for purl %s.", purl.to_string()) - rich_handler.update_gen_build_spec("Repository URL:", "[red]FAILED[/]") - rich_handler.update_gen_build_spec("Commit Hash:", "[red]FAILED[/]") - rich_handler.update_gen_build_spec("Build Tools:", "[red]FAILED[/]") - return None - - try: - latest_component = lookup_latest_component( - purl=purl, - session=session, - ) - except QueryMacaronDatabaseError as lookup_component_error: - logger.error( - "Unexpected result from querying latest component for %s. Error: %s", - purl.to_string(), - lookup_component_error, - ) - return None - if not latest_component: - logger.error( - "Cannot find an analysis result for PackageURL %s in the database. " - + "Please check if an analysis for it exists in the database.", - purl.to_string(), - ) - rich_handler.update_gen_build_spec("Repository URL:", "[red]FAILED[/]") - rich_handler.update_gen_build_spec("Commit Hash:", "[red]FAILED[/]") - rich_handler.update_gen_build_spec("Build Tools:", "[red]FAILED[/]") - return None - - latest_component_repository = latest_component.repository - if not latest_component_repository: - logger.error( - "Cannot find any repository information for %s in the database.", - purl.to_string(), - ) - return None - logger.info( - "Repository information for purl %s: url %s, commit %s", - purl, - latest_component_repository.remote_path, - latest_component_repository.commit_sha, - ) - rich_handler.update_gen_build_spec("Repository URL:", latest_component_repository.remote_path) - rich_handler.update_gen_build_spec("Commit Hash:", latest_component_repository.commit_sha) - - # Getting the RC build tool name from the build tool check facts. - rc_build_tool_name = get_rc_build_tool_name( - component_id=latest_component.id, - session=session, - ) - if not rc_build_tool_name: - return None - - # We always attempt to get the JDK version from maven central JAR for this GAV artifact. - jdk_from_jar = find_jdk_version_from_central_maven_repo( - group_id=group, - artifact_id=artifact, - version=version, - ) - logger.info( - "Attempted to find JDK from Maven Central JAR. Result: %s", - jdk_from_jar or "Cannot find any.", - ) - - # Obtain the highest confidence build command info from the database. - lookup_build_command_info = get_lookup_build_command_info( - component_id=latest_component.id, - session=session, - ) - logger.info( - "Attempted to find build command from the database. Result: %s", - lookup_build_command_info or "Cannot find any.", - ) - - # Select JDK from jar or another source, with a default of version 8. - selected_jdk_version = ( - jdk_from_jar - or (get_lookup_build_command_jdk(lookup_build_command_info) if lookup_build_command_info else None) - or "8" - ) - - major_jdk_version = normalize_jdk_version(selected_jdk_version) - if not major_jdk_version: - logger.error("Failed to obtain the major version of %s", selected_jdk_version) - return None - - # Select build commands from lookup or use a default one. - selected_build_command = ( - lookup_build_command_info.command - if lookup_build_command_info - else get_rc_default_build_command( - rc_build_tool_name, - ) - ) - if not selected_build_command: - logger.error("Failed to get a build command for %s.", purl.to_string()) - return None + str + The generated build spec content. - patched_build_commands = patch_commands( - cmds_sequence=[selected_build_command], - patches=patches, - ) - if not patched_build_commands: - logger.error( - "Failed to patch command sequences %s.", - [selected_build_command], + Raises + ------ + GenerateBuildSpecError + Raised if generation of the build spec fails. + """ + if build_spec["build_tool"].upper() not in (e.name for e in ReproducibleCentralBuildTool): + raise GenerateBuildSpecError( + f"Build tool {build_spec['build_tool']} is not supported by Reproducible Central. " + f"Supported build tools: {[build.name for build in ReproducibleCentralBuildTool]}" ) - return None + if build_spec["group_id"] is None: + raise GenerateBuildSpecError(f"Version is missing in PURL {build_spec['purl']}") template_format_values: dict[str, str] = { "macaron_version": importlib_metadata.version("macaron"), - "group_id": group, - "artifact_id": artifact, - "version": version, - "git_repo": latest_component_repository.remote_path, - "git_tag": latest_component_repository.commit_sha, - "tool": rc_build_tool_name.value, - "newline": "lf", - "buildinfo": f"target/{artifact}-{version}.buildinfo", - "jdk": major_jdk_version, - "command": get_rc_build_command(patched_build_commands), + "group_id": build_spec["group_id"], + "artifact_id": build_spec["artifact_id"], + "version": build_spec["version"], + "git_repo": build_spec["git_repo"], + "git_tag": build_spec["git_tag"], + "tool": ReproducibleCentralBuildTool[build_spec["build_tool"].upper()].value, + "newline": build_spec["newline"], + "buildinfo": f"target/{build_spec['artifact_id']}-{build_spec['version']}.buildinfo", + "jdk": build_spec["language_version"], + "command": compose_shell_commands(build_spec["build_commands"]), } return STRING_TEMPLATE.format_map(template_format_values) diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index c2a0fbc5a..38b8aa21d 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -1038,15 +1038,16 @@ def _determine_build_tools(self, analyze_ctx: AnalyzeContext, git_service: BaseG if not analyze_ctx.component.repository: continue - logger.info( - "Checking if the repo %s uses build tool %s", - analyze_ctx.component.repository.complete_name, - build_tool.name, - ) + if build_tool.match_purl_type(analyze_ctx.component.type): + logger.info( + "Checking if the repo %s uses build tool %s", + analyze_ctx.component.repository.complete_name, + build_tool.name, + ) - if build_tool.is_detected(analyze_ctx.component.repository.fs_path): - logger.info("The repo uses %s build tool.", build_tool.name) - analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) + if build_tool.is_detected(analyze_ctx.component.repository.fs_path): + logger.info("The repo uses %s build tool.", build_tool.name) + analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) if not analyze_ctx.dynamic_data["build_spec"]["tools"]: if analyze_ctx.component.repository: diff --git a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py index bf0e025ac..dfc286c4a 100644 --- a/src/macaron/slsa_analyzer/build_tool/base_build_tool.py +++ b/src/macaron/slsa_analyzer/build_tool/base_build_tool.py @@ -12,6 +12,7 @@ from collections import deque from collections.abc import Iterable from dataclasses import dataclass +from enum import Enum from pathlib import Path from typing import TypedDict @@ -24,6 +25,16 @@ logger: logging.Logger = logging.getLogger(__name__) +class BuildEcosystem(str, Enum): + """The supported build ecosystems.""" + + MAVEN = "maven" + PYPI = "pypi" + GOLONAG = "golang" + NPM = "npm" + DOCKER = "docker" + + class BuildToolCommand(TypedDict): """This class is an abstraction for build tool commands storing useful contextual data for analysis.""" @@ -211,6 +222,29 @@ def load_defaults(self) -> None: if "builder" in defaults: self.path_filters = defaults.get_list("builder", "build_tool_path_filters", fallback=[]) + def match_purl_type(self, component_purl_type: str) -> bool: + """ + Determine if the given component PURL type matches this build tool's PURL type. + + Returns ``False`` if the component PURL type matches a supported build ecosystem but does not + match the build tool's ``purl_type``. Otherwise, returns ``True`` to allow for repositories or + other non-standard types. + + Parameters + ---------- + component_purl_type : str + The PURL type of the component to check. + + Returns + ------- + bool + True if the type matches or is not restricted; False otherwise. + """ + if component_purl_type.upper() in [b.name for b in BuildEcosystem] and component_purl_type != self.purl_type: + return False + # Otherwise return True because the component PURL type can repositories, like github. + return True + def get_dep_analyzer(self) -> DependencyAnalyzer: """Create a DependencyAnalyzer for the build tool. diff --git a/src/macaron/slsa_analyzer/build_tool/language.py b/src/macaron/slsa_analyzer/build_tool/language.py index 2ddc7b196..5b5d47d9f 100644 --- a/src/macaron/slsa_analyzer/build_tool/language.py +++ b/src/macaron/slsa_analyzer/build_tool/language.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module contains abstractions for build languages.""" diff --git a/tests/build_spec_generator/common_spec/test_core.py b/tests/build_spec_generator/common_spec/test_core.py new file mode 100644 index 000000000..9f2eec87e --- /dev/null +++ b/tests/build_spec_generator/common_spec/test_core.py @@ -0,0 +1,143 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module contains the tests for build spec generation""" + +import pytest + +from macaron.build_spec_generator.common_spec.core import ( + MacaronBuildToolName, + compose_shell_commands, + get_language_version, + get_macaron_build_tool_name, +) +from macaron.build_spec_generator.macaron_db_extractor import GenericBuildCommandInfo +from macaron.slsa_analyzer.checks.build_tool_check import BuildToolFacts + + +@pytest.mark.parametrize( + ("cmds_sequence", "expected"), + [ + pytest.param( + [ + "make clean".split(), + "mvn clean package".split(), + ], + "make clean && mvn clean package", + ), + pytest.param( + [ + "mvn clean package".split(), + ], + "mvn clean package", + ), + ], +) +def test_compose_shell_commands( + cmds_sequence: list[list[str]], + expected: str, +) -> None: + """Test the compose_shell_commands function.""" + assert compose_shell_commands(cmds_sequence) == expected + + +@pytest.mark.parametrize( + ("build_tool_facts", "language", "expected"), + [ + pytest.param( + [ + BuildToolFacts( + language="python", + build_tool_name="pip", + ) + ], + "python", + MacaronBuildToolName.PIP, + id="python_pip_supported", + ), + pytest.param( + [ + BuildToolFacts( + language="java", + build_tool_name="gradle", + ) + ], + "java", + MacaronBuildToolName.GRADLE, + id="build_tool_gradle", + ), + pytest.param( + [ + BuildToolFacts( + language="java", + build_tool_name="maven", + ) + ], + "java", + MacaronBuildToolName.MAVEN, + id="build_tool_maven", + ), + pytest.param( + [ + BuildToolFacts( + language="not_java", + build_tool_name="maven", + ) + ], + "java", + None, + id="java_is_the_only_supported_language", + ), + pytest.param( + [ + BuildToolFacts( + language="java", + build_tool_name="some_java_build_tool", + ) + ], + "java", + None, + id="test_unsupported_java_build_tool", + ), + ], +) +def test_get_build_tool_name( + build_tool_facts: list[BuildToolFacts], + language: str, + expected: MacaronBuildToolName | None, +) -> None: + """Test build tool name detection.""" + assert get_macaron_build_tool_name(build_tool_facts, target_language=language) == expected + + +@pytest.mark.parametrize( + ("build_command_info", "expected"), + [ + pytest.param( + GenericBuildCommandInfo( + command=["mvn", "package"], + language="java", + language_versions=["8"], + build_tool_name="maven", + ), + "8", + id="has_language_version", + ), + pytest.param( + GenericBuildCommandInfo( + command=["mvn", "package"], + language="java", + language_versions=[], + build_tool_name="maven", + ), + None, + id="no_language_version", + ), + ], +) +def test_get_language_version( + build_command_info: GenericBuildCommandInfo, + expected: str | None, +) -> None: + """Test the get_language_version function.""" + assert get_language_version(build_command_info) == expected diff --git a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py index e07878411..5b7cbf664 100644 --- a/tests/build_spec_generator/reproducible_central/test_reproducible_central.py +++ b/tests/build_spec_generator/reproducible_central/test_reproducible_central.py @@ -1,143 +1,85 @@ # Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This module contains the tests for Reproducible Central build spec generation""" +"""This module contains tests for Reproducible Central build spec generation.""" import pytest -from macaron.build_spec_generator.macaron_db_extractor import GenericBuildCommandInfo -from macaron.build_spec_generator.reproducible_central.reproducible_central import ( - ReproducibleCentralBuildTool, - _get_rc_build_tool_name_from_build_facts, - get_lookup_build_command_jdk, - get_rc_build_command, - get_rc_default_build_command, -) -from macaron.slsa_analyzer.checks.build_tool_check import BuildToolFacts +from macaron.build_spec_generator.common_spec.base_spec import BaseBuildSpecDict +from macaron.build_spec_generator.common_spec.core import compose_shell_commands +from macaron.build_spec_generator.reproducible_central.reproducible_central import gen_reproducible_central_build_spec +from macaron.errors import GenerateBuildSpecError -@pytest.mark.parametrize( - ("cmds_sequence", "expected"), - [ - pytest.param( - [ - "make clean".split(), - "mvn clean package".split(), - ], - "make clean && mvn clean package", - ), - pytest.param( - [ - "mvn clean package".split(), - ], - "mvn clean package", - ), - ], -) -def test_get_rc_build_command( - cmds_sequence: list[list[str]], - expected: str, -) -> None: - """Test the _get_build_command_sequence function.""" - assert get_rc_build_command(cmds_sequence) == expected +@pytest.fixture(name="base_build_spec") +def fixture_base_build_spec() -> BaseBuildSpecDict: + """Create a base build spec object.""" + return BaseBuildSpecDict( + { + "macaron_version": "1.0.0", + "ecosystem": "maven", + "language": "java", + "group_id": "com.oracle", + "artifact_id": "example-artifact", + "version": "1.2.3", + "git_repo": "https://github.com/oracle/example-artifact.git", + "git_tag": "sampletag", + "build_tool": "maven", + "newline": "lf", + "language_version": "17", + "build_commands": [["mvn", "package"]], + "purl": "pkg:maven/com.oracle/example-artifact@1.2.3", + } + ) -@pytest.mark.parametrize( - ("build_tool_facts", "expected"), - [ - pytest.param( - [ - BuildToolFacts( - language="python", - build_tool_name="pip", - ) - ], - None, - id="python_is_not_supported_for_rc", - ), - pytest.param( - [ - BuildToolFacts( - language="java", - build_tool_name="gradle", - ) - ], - ReproducibleCentralBuildTool.GRADLE, - id="build_tool_gradle", - ), - pytest.param( - [ - BuildToolFacts( - language="java", - build_tool_name="maven", - ) - ], - ReproducibleCentralBuildTool.MAVEN, - id="build_tool_maven", - ), - pytest.param( - [ - BuildToolFacts( - language="not_java", - build_tool_name="maven", - ) - ], - None, - id="java_is_the_only_supported_language", - ), - pytest.param( - [ - BuildToolFacts( - language="java", - build_tool_name="some_java_build_tool", - ) - ], - None, - id="test_unsupported_java_build_tool", - ), - ], -) -def test_get_rc_build_tool_name( - build_tool_facts: list[BuildToolFacts], - expected: ReproducibleCentralBuildTool | None, -) -> None: - """Test the _get_rc_build_tool_name function.""" - assert _get_rc_build_tool_name_from_build_facts(build_tool_facts) == expected +def test_successful_build_spec(base_build_spec: BaseBuildSpecDict) -> None: + """Check the build spec content.""" + content = gen_reproducible_central_build_spec(base_build_spec) + assert isinstance(content, str), "Expected this build spec to be a string." + assert "groupId=com.oracle" in content + assert "artifactId=example-artifact" in content + assert "tool=mvn" in content + assert 'command="mvn package"' in content + +def test_unsupported_build_tool(base_build_spec: BaseBuildSpecDict) -> None: + """Test an unsupported build tool name.""" + base_build_spec["build_tool"] = "unsupported_tool" + with pytest.raises(GenerateBuildSpecError) as excinfo: + gen_reproducible_central_build_spec(base_build_spec) + assert "is not supported by Reproducible Central" in str(excinfo.value) -def test_get_rc_default_build_command_unsupported() -> None: - """Test the get_rc_default_build_command function for an unsupported RC build tool.""" - assert not get_rc_default_build_command(ReproducibleCentralBuildTool.SBT) + +def test_missing_group_id(base_build_spec: BaseBuildSpecDict) -> None: + """Test when group ID is None.""" + base_build_spec["group_id"] = None + with pytest.raises(GenerateBuildSpecError) as excinfo: + gen_reproducible_central_build_spec(base_build_spec) + assert "Version is missing in PURL" in str(excinfo.value) @pytest.mark.parametrize( - ("build_command_info", "expected"), + ("build_tool", "expected"), [ - pytest.param( - GenericBuildCommandInfo( - command=["mvn", "package"], - language="java", - language_versions=["8"], - build_tool_name="maven", - ), - "8", - id="has_language_version", - ), - pytest.param( - GenericBuildCommandInfo( - command=["mvn", "package"], - language="java", - language_versions=[], - build_tool_name="maven", - ), - None, - id="no_language_version", - ), + ("maven", "mvn"), + ("gradle", "gradle"), + ("MAVEN", "mvn"), + ("GRADLE", "gradle"), ], ) -def test_get_lookup_build_command_jdk( - build_command_info: GenericBuildCommandInfo, - expected: str | None, -) -> None: - """Test the get_lookup_build_command_jdk function.""" - assert get_lookup_build_command_jdk(build_command_info) == expected +def test_build_tool_name_variants(base_build_spec: BaseBuildSpecDict, build_tool: str, expected: str) -> None: + """Test the correct handling of build tool names.""" + base_build_spec["build_tool"] = build_tool + content = gen_reproducible_central_build_spec(base_build_spec) + assert content + assert f"tool={expected}" in content + + +def test_compose_shell_commands_integration(base_build_spec: BaseBuildSpecDict) -> None: + """Test that the correct compose_shell_commands function is used.""" + base_build_spec["build_commands"] = [["mvn", "clean", "package"], ["echo", "done"]] + content = gen_reproducible_central_build_spec(base_build_spec) + expected_commands = compose_shell_commands([["mvn", "clean", "package"], ["echo", "done"]]) + assert content + assert f'command="{expected_commands}"' in content diff --git a/tests/build_spec_generator/test_jdk_version_finder.py b/tests/build_spec_generator/test_jdk_version_finder.py index f9df00569..505c0fd58 100644 --- a/tests/build_spec_generator/test_jdk_version_finder.py +++ b/tests/build_spec_generator/test_jdk_version_finder.py @@ -8,7 +8,7 @@ import pytest -from macaron.build_spec_generator.jdk_finder import get_jdk_version_from_jar, join_remote_maven_repo_url +from macaron.build_spec_generator.common_spec.jdk_finder import get_jdk_version_from_jar, join_remote_maven_repo_url @pytest.mark.parametrize( diff --git a/tests/build_spec_generator/test_jdk_version_normalizer.py b/tests/build_spec_generator/test_jdk_version_normalizer.py index 61f085c1d..4932914cc 100644 --- a/tests/build_spec_generator/test_jdk_version_normalizer.py +++ b/tests/build_spec_generator/test_jdk_version_normalizer.py @@ -5,7 +5,7 @@ import pytest -from macaron.build_spec_generator.jdk_version_normalizer import normalize_jdk_version +from macaron.build_spec_generator.common_spec.jdk_version_normalizer import normalize_jdk_version @pytest.mark.parametrize( diff --git a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec index b77755498..2fe2e28c8 100644 --- a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec +++ b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_macaron.buildspec @@ -1,25 +1 @@ -# Copyright (c) 2025, Oracle and/or its affiliates. -# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -# Generated by Macaron version 0.15.0 - -# Input PURL - pkg:github/behnazh-w/example-maven-app@1.0 -# Initial default JDK version 8 and default build command [['mvn', '-DskipTests=true', '-Dmaven.test.skip=true', '-Dmaven.site.skip=true', '-Drat.skip=true', '-Dmaven.javadoc.skip=true', 'clean', 'package']]. -# The lookup build command: ['./mvnw', 'clean', 'package'] -# Jdk version from lookup build command 17. - -groupId=behnazh-w -artifactId=example-maven-app -version=1.0 - -gitRepo=https://github.com/behnazh-w/example-maven-app - -gitTag=2deca75ed5dd365eaf1558a82347b1f11306135f - -tool=mvn -jdk=17 - -newline=lf - -command="./mvnw -DskipTests=true -Dmaven.test.skip=true -Dmaven.site.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true clean package" - -buildinfo=target/example-maven-app-1.0.buildinfo +{"macaron_version": "0.17.0", "group_id": "io.github.behnazh-w.demo", "artifact_id": "core", "version": "2.0.3", "git_repo": "https://github.com/behnazh-w/example-maven-provenance", "git_tag": "597be192fb50f03b86c34f1bfc494fea1eab264f", "newline": "lf", "language_version": "17", "ecosystem": "maven", "purl": "pkg:maven/io.github.behnazh-w.demo/core@2.0.3", "language": "java", "build_tool": "maven", "build_commands": [["./mvnw", "-DskipTests=true", "-Dmaven.test.skip=true", "-Dmaven.site.skip=true", "-Drat.skip=true", "-Dmaven.javadoc.skip=true", "clean", "package"]]} diff --git a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec new file mode 100644 index 000000000..b7ef0f639 --- /dev/null +++ b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/expected_reproducible_central.buildspec @@ -0,0 +1,19 @@ + +# Generated by Macaron version 0.17.0 + +groupId=io.github.behnazh-w.demo +artifactId=core +version=2.0.3 + +gitRepo=https://github.com/behnazh-w/example-maven-provenance + +gitTag=597be192fb50f03b86c34f1bfc494fea1eab264f + +tool=mvn +jdk=17 + +newline=lf + +command="./mvnw -DskipTests=true -Dmaven.test.skip=true -Dmaven.site.skip=true -Drat.skip=true -Dmaven.javadoc.skip=true clean package" + +buildinfo=target/core-2.0.3.buildinfo diff --git a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/test.yaml b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/test.yaml index 1843e1f20..d3fc98ebc 100644 --- a/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/test.yaml +++ b/tests/integration/cases/behnazh-w_example-maven-app_gen_rc_build_spec/test.yaml @@ -16,18 +16,18 @@ steps: options: command_args: - -purl - - pkg:github/behnazh-w/example-maven-app@1.0 -- name: Run Reproducible-central build spec generation + - pkg:maven/io.github.behnazh-w.demo/core@2.0.3 +- name: Run Reproducible Central build spec generation kind: gen-build-spec options: command_args: - -purl - - pkg:github/behnazh-w/example-maven-app@1.0 + - pkg:maven/io.github.behnazh-w.demo/core@2.0.3 - --output-format - rc-buildspec -- name: Compare Buildspec. +- name: Compare Buildspec for Reproducible Central. kind: compare options: kind: rc_build_spec - result: ./output/buildspec/github/behnazh-w/example-maven-app/macaron.buildspec - expected: expected_macaron.buildspec + result: ./output/buildspec/maven/io_github_behnazh-w_demo/core/reproducible_central.buildspec + expected: expected_reproducible_central.buildspec diff --git a/tests/integration/cases/micronaut-projects_micronaut-core/expected_macaron.buildspec b/tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec similarity index 100% rename from tests/integration/cases/micronaut-projects_micronaut-core/expected_macaron.buildspec rename to tests/integration/cases/micronaut-projects_micronaut-core/expected_reproducible_central.buildspec diff --git a/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml b/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml index 26361681d..27d437cc6 100644 --- a/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml +++ b/tests/integration/cases/micronaut-projects_micronaut-core/test.yaml @@ -44,5 +44,5 @@ steps: kind: compare options: kind: rc_build_spec - result: ./output/buildspec/maven/io_micronaut/micronaut-core/macaron.buildspec - expected: expected_macaron.buildspec + result: ./output/buildspec/maven/io_micronaut/micronaut-core/reproducible_central.buildspec + expected: expected_reproducible_central.buildspec From 21c3bbf4da57393993f792c4497c650c0da625ed Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Fri, 10 Oct 2025 12:34:00 +1000 Subject: [PATCH 2/3] chore: resolve merge conflicts Signed-off-by: behnazh-w --- src/macaron/build_spec_generator/build_spec_generator.py | 2 +- src/macaron/console.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/macaron/build_spec_generator/build_spec_generator.py b/src/macaron/build_spec_generator/build_spec_generator.py index 856969685..ac89ce2fd 100644 --- a/src/macaron/build_spec_generator/build_spec_generator.py +++ b/src/macaron/build_spec_generator/build_spec_generator.py @@ -155,7 +155,7 @@ def gen_build_spec_for_purl( os.path.relpath(build_spec_file_path, os.getcwd()), ) rich_handler = access_handler.get_handler() - rich_handler.update_gen_build_spec("Build Spec Path:", os.path.relpath(build_spec_filepath, os.getcwd())) + rich_handler.update_gen_build_spec("Build Spec Path:", os.path.relpath(build_spec_file_path, os.getcwd())) try: with open(build_spec_file_path, mode="w", encoding="utf-8") as file: file.write(build_spec_content) diff --git a/src/macaron/console.py b/src/macaron/console.py index 725a8af0b..08b351b3d 100644 --- a/src/macaron/console.py +++ b/src/macaron/console.py @@ -86,10 +86,6 @@ def __init__(self, *args: Any, verbose: bool = False, **kwargs: Any) -> None: self.find_source_table.add_row(key, value) self.dump_defaults: str | Status = Status("[green]Generating[/]") self.gen_build_spec: dict[str, str | Status] = { - "Package URL:": Status("[green]Processing[/]"), - "Repository URL:": Status("[green]Processing[/]"), - "Commit Hash:": Status("[green]Processing[/]"), - "Build Tools:": Status("[green]Processing[/]"), "Build Spec Path:": "Not Generated", } self.gen_build_spec_table = Table(show_header=False, box=None) From c3144f72854ecf229d48f58a828133800e85494d Mon Sep 17 00:00:00 2001 From: behnazh-w Date: Wed, 29 Oct 2025 14:11:25 +1000 Subject: [PATCH 3/3] test: add an integration test and relevant compare script Signed-off-by: behnazh-w --- ...caron.build_spec_generator.common_spec.rst | 58 ++++ .../apidoc/macaron.build_spec_generator.rst | 17 +- .../common_spec/compare_default_buildspec.py | 247 ++++++++++++++++++ .../computer-k8s/expected_default.buildspec | 1 + .../computer-k8s/test.yaml | 44 ++++ .../pypi_toga/expected_default.buildspec | 1 + .../policy.dl | 0 .../test.yaml | 16 ++ tests/integration/run.py | 1 + 9 files changed, 369 insertions(+), 16 deletions(-) create mode 100644 docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.common_spec.rst create mode 100644 tests/build_spec_generator/common_spec/compare_default_buildspec.py create mode 100644 tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec create mode 100644 tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml create mode 100644 tests/integration/cases/pypi_toga/expected_default.buildspec rename tests/integration/cases/{pypi_toga_provenance_authentic => pypi_toga}/policy.dl (100%) rename tests/integration/cases/{pypi_toga_provenance_authentic => pypi_toga}/test.yaml (59%) diff --git a/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.common_spec.rst b/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.common_spec.rst new file mode 100644 index 000000000..31d03a4cd --- /dev/null +++ b/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.common_spec.rst @@ -0,0 +1,58 @@ +macaron.build\_spec\_generator.common\_spec package +=================================================== + +.. automodule:: macaron.build_spec_generator.common_spec + :members: + :show-inheritance: + :undoc-members: + +Submodules +---------- + +macaron.build\_spec\_generator.common\_spec.base\_spec module +------------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.base_spec + :members: + :show-inheritance: + :undoc-members: + +macaron.build\_spec\_generator.common\_spec.core module +------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.core + :members: + :show-inheritance: + :undoc-members: + +macaron.build\_spec\_generator.common\_spec.jdk\_finder module +-------------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.jdk_finder + :members: + :show-inheritance: + :undoc-members: + +macaron.build\_spec\_generator.common\_spec.jdk\_version\_normalizer module +--------------------------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.jdk_version_normalizer + :members: + :show-inheritance: + :undoc-members: + +macaron.build\_spec\_generator.common\_spec.maven\_spec module +-------------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.maven_spec + :members: + :show-inheritance: + :undoc-members: + +macaron.build\_spec\_generator.common\_spec.pypi\_spec module +------------------------------------------------------------- + +.. automodule:: macaron.build_spec_generator.common_spec.pypi_spec + :members: + :show-inheritance: + :undoc-members: diff --git a/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.rst b/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.rst index f89734501..5bc830015 100644 --- a/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.rst +++ b/docs/source/pages/developers_guide/apidoc/macaron.build_spec_generator.rst @@ -13,6 +13,7 @@ Subpackages :maxdepth: 1 macaron.build_spec_generator.cli_command_parser + macaron.build_spec_generator.common_spec macaron.build_spec_generator.reproducible_central Submodules @@ -34,22 +35,6 @@ macaron.build\_spec\_generator.build\_spec\_generator module :show-inheritance: :undoc-members: -macaron.build\_spec\_generator.jdk\_finder module -------------------------------------------------- - -.. automodule:: macaron.build_spec_generator.jdk_finder - :members: - :show-inheritance: - :undoc-members: - -macaron.build\_spec\_generator.jdk\_version\_normalizer module --------------------------------------------------------------- - -.. automodule:: macaron.build_spec_generator.jdk_version_normalizer - :members: - :show-inheritance: - :undoc-members: - macaron.build\_spec\_generator.macaron\_db\_extractor module ------------------------------------------------------------ diff --git a/tests/build_spec_generator/common_spec/compare_default_buildspec.py b/tests/build_spec_generator/common_spec/compare_default_buildspec.py new file mode 100644 index 000000000..51fd9ea1c --- /dev/null +++ b/tests/build_spec_generator/common_spec/compare_default_buildspec.py @@ -0,0 +1,247 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""Script to compare a generated default buildspec.""" + + +import argparse +import json +import logging +from collections.abc import Callable + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logging.basicConfig(format="[%(filename)s:%(lineno)s %(tag)s] %(message)s") + + +def log_with_tag(tag: str) -> Callable[[str], None]: + """Generate a log function that prints the name of the file and a tag at the beginning of each line.""" + + def log_fn(msg: str) -> None: + logger.info(msg, extra={"tag": tag}) + + return log_fn + + +log_info = log_with_tag("INFO") +log_err = log_with_tag("ERROR") +log_passed = log_with_tag("PASSED") +log_failed = log_with_tag("FAILED") + + +def log_diff(name: str, result: object, expected: object) -> None: + """Pretty-print the diff of two Python objects.""" + output = [ + f"'{name}'", + *("---- Result ---", json.dumps(result, indent=4)), + *("---- Expected ---", json.dumps(expected, indent=4)), + "-----------------", + ] + log_info("\n".join(output)) + + +CompareFn = Callable[[object, object], bool] + + +def skip_compare(_result: object, _expected: object) -> bool: + """Return ``True`` always. + + This compare function is used when we want to skip comparing a field. + """ + return True + + +def compare_json( + result: object, + expected: object, + compare_fn_map: dict[str, CompareFn], + name: str = "", +) -> bool: + """Compare two JSON values. + + This function should not try to return immediately when it encounters a mismatch. + Rather, it should try to report as many mismatches as possible. + + Parameters + ---------- + result : object + The result value. + expected : object + The expected value. + compare_fn_map : dict[str, CompareFn] + A map from field name to corresponding compare function. + name : str + The name of the field. + Field names must follow the following rules: + - At the top level: empty string "" + - A subfield "bar" in an object field with name ".foo" has the name ".foo.bar". + - A subfield "baz" in an object field with name ".foo.bar" has the name ".foo.bar.baz". + - The ith element in an array field with name ".foo" have the name ".foo[i]". + + Returns + ------- + bool + ``True`` if the comparison is successful, ``False`` otherwise. + """ + if name in compare_fn_map: + return compare_fn_map[name](result, expected) + + if isinstance(expected, list): + if not isinstance(result, list): + log_err(f"Expected '{name}' to be a JSON array.") + log_diff(name, result, expected) + # Nothing else to check. + return False + return compare_list(result, expected, compare_fn_map, name) + if isinstance(expected, dict): + if not isinstance(result, dict): + log_err(f"Expected '{name}' to be a JSON object.") + log_diff(name, result, expected) + # Nothing else to check. + return False + return compare_dict(result, expected, compare_fn_map, name) + + if result != expected: + log_err(f"Mismatch found in '{name}'") + log_diff(name, result, expected) + return False + + return True + + +def compare_list( + result: list, + expected: list, + compare_fn_map: dict[str, CompareFn], + name: str, +) -> bool: + """Compare two JSON arrays. + + Parameters + ---------- + result : list + The result array. + expected : list + The expected array. + compare_fn_map : dict[str, CompareFn] + A map from field name to corresponding compare function. + name : str + The name of the field whose value is being compared in this function. + + Returns + ------- + bool + ``True`` if the comparison is successful, ``False`` otherwise. + """ + if len(result) != len(expected): + log_err(f"Expected field '{name}' of length {len(result)} in result to have length {len(expected)}") + log_diff(name, result, expected) + # Nothing else to compare. + return False + + equal = True + + for i, (result_element, expected_element) in enumerate(zip(result, expected)): + equal &= compare_json( + result=result_element, + expected=expected_element, + compare_fn_map=compare_fn_map, + name=f"{name}[{i}]", + ) + + return equal + + +def compare_dict( + result: dict, + expected: dict, + compare_fn_map: dict[str, CompareFn], + name: str, +) -> bool: + """Compare two JSON objects. + + Parameters + ---------- + result : dict + The result object. + expected : dict + The expected object. + compare_fn_map : dict[str, CompareFn] + A map from field name to corresponding compare function. + name : str + The name of the field whose value is being compared in this function. + + Returns + ------- + bool + ``True`` if the comparison is successful, ``False`` otherwise. + """ + result_keys_only = result.keys() - expected.keys() + expected_keys_only = expected.keys() - result.keys() + + equal = True + + if len(result_keys_only) > 0: + log_err(f"'{name}' in result has the following extraneous fields: {result_keys_only}") + equal = False + + if len(expected_keys_only) > 0: + log_err(f"'{name}' in result does not contain these expected fields: {expected_keys_only}") + equal = False + + common_keys = set(result.keys()).intersection(set(expected.keys())) + + for key in common_keys: + equal &= compare_json( + result=result[key], + expected=expected[key], + name=f"{name}.{key}", + compare_fn_map=compare_fn_map, + ) + + return equal + + +def main() -> int: + """Compare a Macaron generated default buildspec. + + Returns + ------- + int + 0 if the generated buildspec matches the expected output, or non-zero otherwise. + """ + parser = argparse.ArgumentParser() + parser.add_argument("result_file", help="the result buildspec file") + parser.add_argument("expected_buildspec_file", help="the expected buildspec file") + args = parser.parse_args() + + with open(args.result_file, encoding="utf-8") as file: + buildspec = json.load(file) + + with open(args.expected_buildspec_file, encoding="utf-8") as file: + expected_buildspec = json.load(file) + + log_info( + f"Comparing the buildspec file {args.result_file} with the expected output file {args.expected_buildspec_file}" + ) + + equal = compare_json( + result=buildspec, + expected=expected_buildspec, + compare_fn_map={ + # We should not compare Macaron version against the snapshot test buildspec because the test will + # fail once we bump the Macaron version in the next release, failing the automatic release altogether! + ".macaron_version": skip_compare, + }, + ) + + if not equal: + log_failed("The generated buildspec does not match the expected output.") + return 1 + + log_passed("The generated buildspec matches the expected output") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec new file mode 100644 index 000000000..5a05b20d7 --- /dev/null +++ b/tests/integration/cases/org_apache_hugegraph/computer-k8s/expected_default.buildspec @@ -0,0 +1 @@ +{"macaron_version": "0.18.0", "group_id": "org.apache.hugegraph", "artifact_id": "computer-k8s", "version": "1.0.0", "git_repo": "https://github.com/apache/hugegraph-computer", "git_tag": "d2b95262091d6572cc12dcda57d89f9cd44ac88b", "newline": "lf", "language_version": "11", "ecosystem": "maven", "purl": "pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0", "language": "java", "build_tool": "maven", "build_commands": [["mvn", "-DskipTests=true", "-Dmaven.test.skip=true", "-Dmaven.site.skip=true", "-Drat.skip=true", "-Dmaven.javadoc.skip=true", "clean", "package"]]} diff --git a/tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml b/tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml new file mode 100644 index 000000000..1a4b5f034 --- /dev/null +++ b/tests/integration/cases/org_apache_hugegraph/computer-k8s/test.yaml @@ -0,0 +1,44 @@ +# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Generate a buildspec for this PURL and validate the buildspec content. + +tags: +- macaron-python-package +- macaron-gen-build-spec +- tutorial + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - -purl + - pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0 +- name: Run the default buildspec generation + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0 +- name: Compare Buildspec. + kind: compare + options: + kind: default_build_spec + result: output/buildspec/maven/org_apache_hugegraph/computer-k8s/macaron.buildspec + expected: expected_default.buildspec +- name: Run the buildspec generation explicitly providing the default output format option. + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:maven/org.apache.hugegraph/computer-k8s@1.0.0 + - --output-format + - default-buildspec +- name: Compare Buildspec. + kind: compare + options: + kind: default_build_spec + result: output/buildspec/maven/org_apache_hugegraph/computer-k8s/macaron.buildspec + expected: expected_default.buildspec diff --git a/tests/integration/cases/pypi_toga/expected_default.buildspec b/tests/integration/cases/pypi_toga/expected_default.buildspec new file mode 100644 index 000000000..ab168ac03 --- /dev/null +++ b/tests/integration/cases/pypi_toga/expected_default.buildspec @@ -0,0 +1 @@ +{"macaron_version": "0.18.0", "group_id": null, "artifact_id": "toga", "version": "0.5.1", "git_repo": "https://github.com/beeware/toga", "git_tag": "ef1912b0a1b5c07793f9aa372409f5b9d36f2604", "newline": "lf", "language_version": "", "ecosystem": "pypi", "purl": "pkg:pypi/toga@0.5.1", "language": "python", "build_tool": "pip", "build_commands": [["pip", "install", "-U", "pip"]]} diff --git a/tests/integration/cases/pypi_toga_provenance_authentic/policy.dl b/tests/integration/cases/pypi_toga/policy.dl similarity index 100% rename from tests/integration/cases/pypi_toga_provenance_authentic/policy.dl rename to tests/integration/cases/pypi_toga/policy.dl diff --git a/tests/integration/cases/pypi_toga_provenance_authentic/test.yaml b/tests/integration/cases/pypi_toga/test.yaml similarity index 59% rename from tests/integration/cases/pypi_toga_provenance_authentic/test.yaml rename to tests/integration/cases/pypi_toga/test.yaml index 1773a5a26..4b721d0c0 100644 --- a/tests/integration/cases/pypi_toga_provenance_authentic/test.yaml +++ b/tests/integration/cases/pypi_toga/test.yaml @@ -3,9 +3,11 @@ description: | Analyzing a PyPI PURL that has provenance available on the PyPI registry, and passes the SCM authenticity check. + It also tests buildspec generation. tags: - macaron-python-package +- tutorial steps: - name: Run macaron analyze @@ -18,3 +20,17 @@ steps: kind: verify options: policy: policy.dl +- name: Generate the buildspec + kind: gen-build-spec + options: + command_args: + - -purl + - pkg:pypi/toga@0.5.1 + - --output-format + - default-buildspec +- name: Compare Buildspec. + kind: compare + options: + kind: default_build_spec + result: output/buildspec/pypi/toga/macaron.buildspec + expected: expected_default.buildspec diff --git a/tests/integration/run.py b/tests/integration/run.py index ff4cb474d..3cc50fde7 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -81,6 +81,7 @@ def configure_logging(verbose: bool) -> None: "vsa": ["tests", "vsa", "compare_vsa.py"], "find_source": ["tests", "find_source", "compare_source_reports.py"], "rc_build_spec": ["tests", "build_spec_generator", "reproducible_central", "compare_rc_build_spec.py"], + "default_build_spec": ["tests", "build_spec_generator", "common_spec", "compare_default_buildspec.py"], } VALIDATE_SCHEMA_SCRIPTS: dict[str, Sequence[str]] = {