From 80a01484a38d90e49695032b0ad0cb3340d0bb12 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 16:03:30 -0400 Subject: [PATCH 01/19] feat: get released version from versions.txt --- .../generate_composed_library.py | 13 +++++---- library_generation/generate_repo.py | 5 ++-- library_generation/model/repo_config.py | 29 +++++++++++++++++-- library_generation/owlbot/bin/entrypoint.sh | 4 +++ .../owlbot/templates/java_library/README.md | 16 ++++++++++ library_generation/postprocess_library.sh | 6 +++- library_generation/utils/utilities.py | 27 ++++++++++++----- 7 files changed, 82 insertions(+), 18 deletions(-) diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 46e1491ccc..4d7e31f324 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -36,6 +36,7 @@ from library_generation.model.gapic_inputs import GapicInputs from library_generation.model.library_config import LibraryConfig from library_generation.model.gapic_inputs import parse as parse_build_file +from library_generation.model.repo_config import RepoConfig script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -44,8 +45,7 @@ def generate_composed_library( config: GenerationConfig, library_path: str, library: LibraryConfig, - output_folder: str, - versions_file: str, + repo_config: RepoConfig, ) -> None: """ Generate libraries composed of more than one service or service version @@ -55,10 +55,10 @@ def generate_composed_library( :param library_path: the path to which the generated file goes :param library: a LibraryConfig object contained inside config, passed here for convenience and to prevent all libraries to be processed - :param output_folder: the folder to where tools go - :param versions_file: the file containing version of libraries + :param repo_config: :return None """ + output_folder = repo_config.output_folder util.pull_api_definition( config=config, library=library, output_folder=output_folder ) @@ -102,16 +102,19 @@ def generate_composed_library( cwd=output_folder, ) + _, artifact_id = util.get_distribution_name(library=library) + library_version = repo_config.library_versions.get(artifact_id, None) # call postprocess library util.run_process_and_print_output( [ f"{script_dir}/postprocess_library.sh", f"{library_path}", "", - versions_file, + repo_config.versions_file, owlbot_cli_source_folder, str(config.is_monorepo()).lower(), config.libraries_bom_version, + library_version, ], "Library postprocessing", ) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index b5cf9229fb..a4f6a5382e 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -44,14 +44,13 @@ def generate_from_yaml( gen_config=config, library_config=target_libraries, repo_path=repository_path ) - for library_path, library in repo_config.libraries.items(): + for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") generate_composed_library( config=config, library_path=library_path, library=library, - output_folder=repo_config.output_folder, - versions_file=repo_config.versions_file, + repo_config=repo_config, ) if not config.is_monorepo() or config.contains_common_protos(): diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index 7f42720fe3..faeadbb106 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -12,10 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict from library_generation.model.library_config import LibraryConfig +GRPC_PREFIX = "grpc-" +PROTO_PREFIX = "proto-" + + class RepoConfig: """ Class that represents a generated repository @@ -24,7 +27,7 @@ class RepoConfig: def __init__( self, output_folder: str, - libraries: Dict[str, LibraryConfig], + libraries: dict[str, LibraryConfig], versions_file: str, ): """ @@ -36,3 +39,25 @@ def __init__( self.output_folder = output_folder self.libraries = libraries self.versions_file = versions_file + self.library_versions = RepoConfig.__parse_version_from(self.versions_file) + + def get_libraries(self) -> dict[str, LibraryConfig]: + return self.libraries + + def get_library_versions(self): + return self.library_versions + + @staticmethod + def __parse_version_from(version_file: str) -> dict[str, str]: + library_versions = dict() + with open(version_file) as f: + for line in f.readlines(): + sections = line.split(":") + artifact_id = sections[0] + released_version = sections[1] + if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( + PROTO_PREFIX + ): + continue + library_versions[artifact_id] = released_version + return library_versions diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index b5b84cf1ac..a81cd5795e 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -22,6 +22,7 @@ # both to README and pom.xml files # 3: is_monorepo: whether we are postprocessing a monorepo # 4: libraries_bom_version: used to render the readme +# 5: library_version: used to render the readme # The scripts assumes the CWD is the folder where postprocessing is going to be # applied @@ -31,6 +32,7 @@ scripts_root=$1 versions_file=$2 is_monorepo=$3 libraries_bom_version=$4 +library_version=$5 if [[ "${is_monorepo}" == "true" ]]; then @@ -48,10 +50,12 @@ then # synthtool library considering the way owlbot.py files are written export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates" export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}" + export SYNTHTOOL_LIBRARY_VERSION="${library_version}" # defaults to run owlbot.py python3 owlbot.py unset SYNTHTOOL_TEMPLATES unset SYNTHTOOL_LIBRARIES_BOM_VERSION + unset SYNTHTOOL_LIBRARY_VERSION fi echo "...done" diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 04f1f4e834..1e1a558f69 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -71,7 +71,11 @@ If you are using Maven, add this to your pom.xml file: {{ group_id }} {{ artifact_id }} + {% if 'library_version' in metadata %} + {{ metadata['library_version'] }} + {% else %} {{ metadata['latest_version'] }} + {% endif %} {% endif -%} ``` @@ -89,13 +93,21 @@ implementation '{{ group_id }}:{{ artifact_id }}' If you are using Gradle without BOM, add this to your dependencies: ```Groovy +{% if 'library_version' in metadata %} +implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['library_version'] }}' +{% else %} implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' +{% endif %} ``` If you are using SBT, add this to your dependencies: ```Scala +{% if 'library_version' in metadata %} +libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['library_version'] }}" +{% else %} libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}" +{% endif %} ``` @@ -264,7 +276,11 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html [stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} [maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg +{% if 'library_version' in metadata %} +[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['library_version'] }} +{% else %} [maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} +{% endif %} [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 392301814f..eeec07156e 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -19,6 +19,8 @@ # different logic # 6 - libraries_bom_version: used by our implementation of owlbot to render the # readme +# 7 - library_version: used by our implementation of owlbot to render the +# readme set -exo pipefail scripts_root=$(dirname "$(readlink -f "$0")") @@ -28,6 +30,7 @@ versions_file=$3 owlbot_cli_source_folder=$4 is_monorepo=$5 libraries_bom_version=$6 +library_version=$7 owlbot_yaml_file_name=".OwlBot-hermetic.yaml" source "${scripts_root}"/utils/utilities.sh @@ -102,6 +105,7 @@ bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ "${scripts_root}" \ "${versions_file}" \ "${is_monorepo}" \ - "${libraries_bom_version}" + "${libraries_bom_version}" \ + "${library_version}" popd # postprocessing_target diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 0490ad9e2a..4f8208fd55 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -183,6 +183,25 @@ def pull_api_definition( ) +def get_distribution_name(library: LibraryConfig) -> tuple[str, str]: + """ + Determines the full Maven coordinate and a shortened name for a library. + + :param library: the library configuration + :return: A tuple containing: the full Maven coordinate in the format + "groupId:artifactId"; a shortened version of the artifact ID. + """ + cloud_prefix = "cloud-" if library.cloud_api else "" + library_name = library.get_library_name() + distribution_name = ( + library.distribution_name + if library.distribution_name + else f"{library.group_id}:google-{cloud_prefix}{library_name}" + ) + distribution_name_short = re.split(r"[:/]", distribution_name)[-1] + return distribution_name, distribution_name_short + + def generate_postprocessing_prerequisite_files( config: GenerationConfig, library: LibraryConfig, @@ -204,14 +223,8 @@ def generate_postprocessing_prerequisite_files( :param language: programming language of the library :return: None """ - cloud_prefix = "cloud-" if library.cloud_api else "" + distribution_name, distribution_name_short = get_distribution_name(library) library_name = library.get_library_name() - distribution_name = ( - library.distribution_name - if library.distribution_name - else f"{library.group_id}:google-{cloud_prefix}{library_name}" - ) - distribution_name_short = re.split(r"[:/]", distribution_name)[-1] if config.contains_common_protos(): repo = SDK_PLATFORM_JAVA elif config.is_monorepo(): From e1e53d5f021236cdda7945af81dc2097705bf34c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 16:35:01 -0400 Subject: [PATCH 02/19] fix broken unit tests --- library_generation/model/repo_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index faeadbb106..f00f72d1dc 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -53,6 +53,9 @@ def __parse_version_from(version_file: str) -> dict[str, str]: with open(version_file) as f: for line in f.readlines(): sections = line.split(":") + # skip comments and whitespace. + if len(sections) != 3: + continue artifact_id = sections[0] released_version = sections[1] if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( From 3583ecdabeca516ba73a9ebb5a1044ab107bd79f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 17:04:37 -0400 Subject: [PATCH 03/19] change synthtool commit --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 9de0bc1c2c..5a2232b1eb 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/cloud-devrel-public-resources/python SHELL [ "/bin/bash", "-c" ] -ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940 +ARG SYNTHTOOL_COMMITTISH=37772ca3ad299708a7fec2edd318e1f14c7d584d ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 ENV HOME=/home From ccd5e8aedf6af299d0e7b4dc4c71e37f7a27969e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 17:08:14 -0400 Subject: [PATCH 04/19] change format in template --- .../owlbot/templates/java_library/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 1e1a558f69..dec9e7b186 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -71,9 +71,9 @@ If you are using Maven, add this to your pom.xml file: {{ group_id }} {{ artifact_id }} - {% if 'library_version' in metadata %} + {% if 'library_version' in metadata -%} {{ metadata['library_version'] }} - {% else %} + {% else -%} {{ metadata['latest_version'] }} {% endif %} @@ -84,7 +84,7 @@ If you are using Maven, add this to your pom.xml file: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:{{metadata['latest_bom_version']}}') +implementation platform('com.google.cloud:libraries-bom:{{metadata['libraries_bom_version']}}') implementation '{{ group_id }}:{{ artifact_id }}' ``` @@ -93,9 +93,9 @@ implementation '{{ group_id }}:{{ artifact_id }}' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -{% if 'library_version' in metadata %} +{% if 'library_version' in metadata -%} implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['library_version'] }}' -{% else %} +{% else -%} implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' {% endif %} ``` @@ -276,9 +276,9 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html [stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} [maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg -{% if 'library_version' in metadata %} +{% if 'library_version' in metadata -%} [maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['library_version'] }} -{% else %} +{% else -%} [maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} {% endif %} [authentication]: https://github.com/googleapis/google-cloud-java#authentication From b3490a4ca44def9ee784866d1a12122f007ef3e9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 17:22:25 -0400 Subject: [PATCH 05/19] change format --- .../owlbot/templates/java_library/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index dec9e7b186..0b81d721cd 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -75,7 +75,7 @@ If you are using Maven, add this to your pom.xml file: {{ metadata['library_version'] }} {% else -%} {{ metadata['latest_version'] }} - {% endif %} + {% endif -%} {% endif -%} ``` @@ -97,17 +97,17 @@ If you are using Gradle without BOM, add this to your dependencies: implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['library_version'] }}' {% else -%} implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' -{% endif %} +{% endif -%} ``` If you are using SBT, add this to your dependencies: ```Scala -{% if 'library_version' in metadata %} +{% if 'library_version' in metadata -%} libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['library_version'] }}" -{% else %} +{% else -%} libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}" -{% endif %} +{% endif -%} ``` @@ -280,7 +280,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['library_version'] }} {% else -%} [maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} -{% endif %} +{% endif -%} [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From 126446ae3bec430794773e3a5902ac5ded59fba4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 17:43:19 -0400 Subject: [PATCH 06/19] format --- library_generation/owlbot/templates/java_library/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 0b81d721cd..1a01c1d19d 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -75,7 +75,7 @@ If you are using Maven, add this to your pom.xml file: {{ metadata['library_version'] }} {% else -%} {{ metadata['latest_version'] }} - {% endif -%} + {%- endif %} {% endif -%} ``` From 208edbe1f426cc63741b8857c001706ce00f1eb1 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 18:01:42 -0400 Subject: [PATCH 07/19] format --- library_generation/owlbot/templates/java_library/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 1a01c1d19d..341bf3bb31 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -74,8 +74,7 @@ If you are using Maven, add this to your pom.xml file: {% if 'library_version' in metadata -%} {{ metadata['library_version'] }} {% else -%} - {{ metadata['latest_version'] }} - {%- endif %} + {{ metadata['latest_version'] }}{% endif %} {% endif -%} ``` From 6afffe7d95f455b66d5d465e4aa25cdc467878fc Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 18:11:11 -0400 Subject: [PATCH 08/19] format --- library_generation/owlbot/templates/java_library/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 341bf3bb31..e833051383 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -71,10 +71,11 @@ If you are using Maven, add this to your pom.xml file: {{ group_id }} {{ artifact_id }} - {% if 'library_version' in metadata -%} + {%- if 'library_version' in metadata -%} {{ metadata['library_version'] }} - {% else -%} - {{ metadata['latest_version'] }}{% endif %} + {%- else -%} + {{ metadata['latest_version'] }} + {% endif %} {% endif -%} ``` From 2d0544a1f26bafd7c8f9dac083894f04d6e499c5 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 2 Jul 2024 18:31:59 -0400 Subject: [PATCH 09/19] use variable --- .../owlbot/templates/java_library/README.md | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index e833051383..799362258f 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -1,5 +1,6 @@ {% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} {% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} +{% set version = metadata.get('library_version', metadata['latest_version']) -%} {% set repo_short = metadata['repo']['repo'].split('/')|last -%} # Google {{ metadata['repo']['name_pretty'] }} Client for Java @@ -71,11 +72,7 @@ If you are using Maven, add this to your pom.xml file: {{ group_id }} {{ artifact_id }} - {%- if 'library_version' in metadata -%} - {{ metadata['library_version'] }} - {%- else -%} - {{ metadata['latest_version'] }} - {% endif %} + {{ version }} {% endif -%} ``` @@ -93,21 +90,13 @@ implementation '{{ group_id }}:{{ artifact_id }}' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -{% if 'library_version' in metadata -%} -implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['library_version'] }}' -{% else -%} -implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' -{% endif -%} +implementation '{{ group_id }}:{{ artifact_id }}:{{ version }}' ``` If you are using SBT, add this to your dependencies: ```Scala -{% if 'library_version' in metadata -%} -libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['library_version'] }}" -{% else -%} -libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}" -{% endif -%} +libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ version }}" ``` @@ -276,11 +265,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html [stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} [maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg -{% if 'library_version' in metadata -%} -[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['library_version'] }} -{% else -%} -[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} -{% endif -%} +[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ version }} [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From 179977f135be7bb82905985f36f45a22cf552918 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 09:33:44 -0400 Subject: [PATCH 10/19] use default value for new client --- library_generation/generate_composed_library.py | 3 ++- library_generation/owlbot/bin/entrypoint.sh | 1 - library_generation/owlbot/templates/java_library/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 4d7e31f324..61a0392479 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -39,6 +39,7 @@ from library_generation.model.repo_config import RepoConfig script_dir = os.path.dirname(os.path.realpath(__file__)) +NEW_CLIENT_VERSION = "0.0.0" def generate_composed_library( @@ -103,7 +104,7 @@ def generate_composed_library( ) _, artifact_id = util.get_distribution_name(library=library) - library_version = repo_config.library_versions.get(artifact_id, None) + library_version = repo_config.library_versions.get(artifact_id, NEW_CLIENT_VERSION) # call postprocess library util.run_process_and_print_output( [ diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index a81cd5795e..b64b12bdb6 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -34,7 +34,6 @@ is_monorepo=$3 libraries_bom_version=$4 library_version=$5 - if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp rm -rd owl-bot-staging/ diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 799362258f..21091251c0 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -1,6 +1,6 @@ {% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} {% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} -{% set version = metadata.get('library_version', metadata['latest_version']) -%} +{% set version = metadata['library_version'] -%} {% set repo_short = metadata['repo']['repo'].split('/')|last -%} # Google {{ metadata['repo']['name_pretty'] }} Client for Java From 8d2b5fc3db0f05a59ef34256149bd3d42801e007 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 09:35:03 -0400 Subject: [PATCH 11/19] change synthtool commit --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 5a2232b1eb..d548325a9f 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/cloud-devrel-public-resources/python SHELL [ "/bin/bash", "-c" ] -ARG SYNTHTOOL_COMMITTISH=37772ca3ad299708a7fec2edd318e1f14c7d584d +ARG SYNTHTOOL_COMMITTISH=7b95718a2b7ecc2d302d32681517472fe75d2cac ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 ENV HOME=/home From 035f2af58c3f6d77f678d1a3aafe60d4bd3933a5 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 10:25:03 -0400 Subject: [PATCH 12/19] add unit tests --- .../test/utilities_unit_tests.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index c38016fcba..e36829a554 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -27,6 +27,7 @@ from library_generation.model.library_config import LibraryConfig from library_generation.test.test_utils import FileComparator from library_generation.test.test_utils import cleanup +from library_generation.utils.utilities import get_distribution_name script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "resources") @@ -260,6 +261,47 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual(["misc"], library_path) shutil.rmtree(repo_config.output_folder) + def test_get_distribution_name_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + ) + distribution_name, distribution_name_short = get_distribution_name(library) + self.assertEqual( + "com.google.cloud:google-cloud-baremetalsolution", distribution_name + ) + self.assertEqual("google-cloud-baremetalsolution", distribution_name_short) + + def test_get_distribution_name_non_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + cloud_api=False, + group_id="com.example", + ) + distribution_name, distribution_name_short = get_distribution_name(library) + self.assertEqual("com.example:google-baremetalsolution", distribution_name) + self.assertEqual("google-baremetalsolution", distribution_name_short) + + def test_get_distribution_name_with_distribution_name(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="com.example:baremetalsolution", + ) + distribution_name, distribution_name_short = get_distribution_name(library) + self.assertEqual("com.example:baremetalsolution", distribution_name) + self.assertEqual("baremetalsolution", distribution_name_short) + def __setup_postprocessing_prerequisite_files( self, combination: int, From 2b1ad09eb8cac38f93294a1474caa4a44be51af1 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 10:49:02 -0400 Subject: [PATCH 13/19] add unit tests --- .../generate_composed_library.py | 6 ++- library_generation/model/repo_config.py | 5 ++- .../test/model/repo_config_unit_tests.py | 37 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 library_generation/test/model/repo_config_unit_tests.py diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 61a0392479..6ebe704e82 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -104,7 +104,11 @@ def generate_composed_library( ) _, artifact_id = util.get_distribution_name(library=library) - library_version = repo_config.library_versions.get(artifact_id, NEW_CLIENT_VERSION) + # The artifact id is not in the mapping if this library is a new client. + # In this case, use `0.0.0` as the version. + library_version = repo_config.get_library_versions().get( + artifact_id, NEW_CLIENT_VERSION + ) # call postprocess library util.run_process_and_print_output( [ diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index f00f72d1dc..bdedea9e9a 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -44,7 +44,10 @@ def __init__( def get_libraries(self) -> dict[str, LibraryConfig]: return self.libraries - def get_library_versions(self): + def get_library_versions(self) -> dict[str, str]: + """ + Returns a mapping from Maven artifact ID to version. + """ return self.library_versions @staticmethod diff --git a/library_generation/test/model/repo_config_unit_tests.py b/library_generation/test/model/repo_config_unit_tests.py new file mode 100644 index 0000000000..e634b1e70c --- /dev/null +++ b/library_generation/test/model/repo_config_unit_tests.py @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.model.repo_config import RepoConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) +versions_file = os.path.join(script_dir, "..", "resources", "misc", "versions.txt") + + +class RepoConfigTest(unittest.TestCase): + def test_get_library_versions_with_existing_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) + self.assertEqual( + { + "gapic-generator-java": "2.25.0", + "api-common": "2.16.0", + "gax": "2.33.0", + "gax-grpc": "2.34.0", + "gax-httpjson": "0.118.0", + }, + repo_config.get_library_versions(), + ) From a80543d89e10eb9dd8e9e78e4b8b74731681917f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 11:44:39 -0400 Subject: [PATCH 14/19] refactor repo_config --- .../generate_composed_library.py | 7 +---- library_generation/model/repo_config.py | 12 ++++++-- .../test/model/repo_config_unit_tests.py | 30 ++++++++++++++----- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 6ebe704e82..6e489beeb4 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -39,7 +39,6 @@ from library_generation.model.repo_config import RepoConfig script_dir = os.path.dirname(os.path.realpath(__file__)) -NEW_CLIENT_VERSION = "0.0.0" def generate_composed_library( @@ -104,11 +103,7 @@ def generate_composed_library( ) _, artifact_id = util.get_distribution_name(library=library) - # The artifact id is not in the mapping if this library is a new client. - # In this case, use `0.0.0` as the version. - library_version = repo_config.get_library_versions().get( - artifact_id, NEW_CLIENT_VERSION - ) + library_version = repo_config.get_library_version(artifact_id=artifact_id) # call postprocess library util.run_process_and_print_output( [ diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index bdedea9e9a..b014f4db8c 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -17,6 +17,7 @@ GRPC_PREFIX = "grpc-" PROTO_PREFIX = "proto-" +NEW_CLIENT_VERSION = "0.0.0" class RepoConfig: @@ -32,6 +33,7 @@ def __init__( ): """ Init a RepoConfig object + :param output_folder: the path to which the generated repo goes :param libraries: a mapping from library_path to LibraryConfig object :param versions_file: the path of versions.txt used in post-processing @@ -44,11 +46,15 @@ def __init__( def get_libraries(self) -> dict[str, LibraryConfig]: return self.libraries - def get_library_versions(self) -> dict[str, str]: + def get_library_version(self, artifact_id: str) -> str: """ - Returns a mapping from Maven artifact ID to version. + Returns the version of a given artifact ID. + If the artifact ID is not managed, i.e., a new client, returns `0.0.0`. + + :param artifact_id: the Maven artifact ID. + :return: the version of the artifact. """ - return self.library_versions + return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) @staticmethod def __parse_version_from(version_file: str) -> dict[str, str]: diff --git a/library_generation/test/model/repo_config_unit_tests.py b/library_generation/test/model/repo_config_unit_tests.py index e634b1e70c..8960a5d806 100644 --- a/library_generation/test/model/repo_config_unit_tests.py +++ b/library_generation/test/model/repo_config_unit_tests.py @@ -26,12 +26,26 @@ def test_get_library_versions_with_existing_library(self): output_folder="test", libraries=dict(), versions_file=versions_file ) self.assertEqual( - { - "gapic-generator-java": "2.25.0", - "api-common": "2.16.0", - "gax": "2.33.0", - "gax-grpc": "2.34.0", - "gax-httpjson": "0.118.0", - }, - repo_config.get_library_versions(), + "2.25.0", + repo_config.get_library_version("gapic-generator-java"), + ) + self.assertEqual( + "2.16.0", + repo_config.get_library_version("api-common"), + ) + self.assertEqual( + "2.33.0", + repo_config.get_library_version("gax"), + ) + self.assertEqual( + "2.34.0", + repo_config.get_library_version("gax-grpc"), + ) + self.assertEqual( + "0.118.0", + repo_config.get_library_version("gax-httpjson"), + ) + self.assertEqual( + "0.0.0", + repo_config.get_library_version("example-artifact"), ) From 96ff23a8ecccd15dff4c9f5da2e18b3010d5caac Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 11:47:47 -0400 Subject: [PATCH 15/19] refactor tests --- library_generation/test/model/repo_config_unit_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library_generation/test/model/repo_config_unit_tests.py b/library_generation/test/model/repo_config_unit_tests.py index 8960a5d806..12d28fe254 100644 --- a/library_generation/test/model/repo_config_unit_tests.py +++ b/library_generation/test/model/repo_config_unit_tests.py @@ -45,6 +45,11 @@ def test_get_library_versions_with_existing_library(self): "0.118.0", repo_config.get_library_version("gax-httpjson"), ) + + def test_get_library_versions_with_new_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) self.assertEqual( "0.0.0", repo_config.get_library_version("example-artifact"), From 272c394667c71685077b06df2f18a71699188ebd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 13:26:00 -0400 Subject: [PATCH 16/19] change synthtool commit --- .cloudbuild/library_generation/library_generation.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index d548325a9f..9977160797 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/cloud-devrel-public-resources/python SHELL [ "/bin/bash", "-c" ] -ARG SYNTHTOOL_COMMITTISH=7b95718a2b7ecc2d302d32681517472fe75d2cac +ARG SYNTHTOOL_COMMITTISH=696c4bff721f5541cd75fdc97d413f8f39d2a2c1 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 ENV HOME=/home From 0bb4f9e07f0af0d32b263ed3320a6b304b4f443c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 17:11:09 -0400 Subject: [PATCH 17/19] refactor code --- .../generate_composed_library.py | 5 ++- library_generation/model/library_config.py | 22 +++++++++- .../templates/owlbot.yaml.monorepo.j2 | 8 ++-- .../test/model/library_config_unit_tests.py | 43 +++++++++++++++++++ .../test/utilities_unit_tests.py | 42 ------------------ library_generation/utils/utilities.py | 28 ++---------- 6 files changed, 75 insertions(+), 73 deletions(-) diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 6e489beeb4..dc094c0b11 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -102,8 +102,9 @@ def generate_composed_library( cwd=output_folder, ) - _, artifact_id = util.get_distribution_name(library=library) - library_version = repo_config.get_library_version(artifact_id=artifact_id) + library_version = repo_config.get_library_version( + artifact_id=library.get_artifact_id() + ) # call postprocess library util.run_process_and_print_output( [ diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index 52e15891b6..6e015e4a23 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re from hashlib import sha1 from typing import Optional @@ -64,7 +65,6 @@ def __init__( self.excluded_dependencies = excluded_dependencies self.excluded_poms = excluded_poms self.client_documentation = client_documentation - self.distribution_name = distribution_name self.googleapis_commitish = googleapis_commitish self.group_id = group_id self.issue_tracker = issue_tracker @@ -76,6 +76,7 @@ def __init__( self.extra_versioned_modules = extra_versioned_modules self.recommended_package = recommended_package self.min_java_version = min_java_version + self.distribution_name = self.__get_distribution_name(distribution_name) def get_library_name(self) -> str: """ @@ -87,6 +88,25 @@ def get_library_name(self) -> str: def get_sorted_gapic_configs(self) -> list[GapicConfig]: return sorted(self.gapic_configs) + def get_maven_coordinate(self): + """ + Returns the Maven coordinate (group_id:artifact_id) of the library + """ + return self.distribution_name + + def get_artifact_id(self): + """ + Returns the artifact ID of the library + """ + return re.split(r"[:/]", self.distribution_name)[-1] + + def __get_distribution_name(self, distribution_name: Optional[str]) -> str: + if distribution_name: + return distribution_name + cloud_prefix = "cloud-" if self.cloud_api else "" + library_name = self.get_library_name() + return f"{self.group_id}:google-{cloud_prefix}{library_name}" + def __eq__(self, other): return ( self.api_shortname == other.api_shortname diff --git a/library_generation/templates/owlbot.yaml.monorepo.j2 b/library_generation/templates/owlbot.yaml.monorepo.j2 index 5267a6f8a3..9ed63c4260 100644 --- a/library_generation/templates/owlbot.yaml.monorepo.j2 +++ b/library_generation/templates/owlbot.yaml.monorepo.j2 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -{% if artifact_name %} +{% if artifact_id %} deep-remove-regex: - "/{{ module_name }}/grpc-google-.*/src" - "/{{ module_name }}/proto-google-.*/src" @@ -24,11 +24,11 @@ deep-preserve-regex: deep-copy-regex: - source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_name }}-$1/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src" - source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_name }}-$1/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src" - source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_name }}/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src" - source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" {%- endif %} diff --git a/library_generation/test/model/library_config_unit_tests.py b/library_generation/test/model/library_config_unit_tests.py index 35ba5be3e4..c161cf51f2 100644 --- a/library_generation/test/model/library_config_unit_tests.py +++ b/library_generation/test/model/library_config_unit_tests.py @@ -64,3 +64,46 @@ def test_get_sorted_gapic_configs_returns_correct_order(self): ], library.get_sorted_gapic_configs(), ) + + def test_get_distribution_name_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + ) + self.assertEqual( + "com.google.cloud:google-cloud-baremetalsolution", + library.get_maven_coordinate(), + ) + self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_non_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + cloud_api=False, + group_id="com.example", + ) + self.assertEqual( + "com.example:google-baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("google-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_with_distribution_name(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="com.example:baremetalsolution", + ) + self.assertEqual( + "com.example:baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("baremetalsolution", library.get_artifact_id()) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index e36829a554..c38016fcba 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -27,7 +27,6 @@ from library_generation.model.library_config import LibraryConfig from library_generation.test.test_utils import FileComparator from library_generation.test.test_utils import cleanup -from library_generation.utils.utilities import get_distribution_name script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "resources") @@ -261,47 +260,6 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual(["misc"], library_path) shutil.rmtree(repo_config.output_folder) - def test_get_distribution_name_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - ) - distribution_name, distribution_name_short = get_distribution_name(library) - self.assertEqual( - "com.google.cloud:google-cloud-baremetalsolution", distribution_name - ) - self.assertEqual("google-cloud-baremetalsolution", distribution_name_short) - - def test_get_distribution_name_non_cloud_api(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - cloud_api=False, - group_id="com.example", - ) - distribution_name, distribution_name_short = get_distribution_name(library) - self.assertEqual("com.example:google-baremetalsolution", distribution_name) - self.assertEqual("google-baremetalsolution", distribution_name_short) - - def test_get_distribution_name_with_distribution_name(self): - library = LibraryConfig( - api_shortname="baremetalsolution", - name_pretty="Bare Metal Solution", - product_documentation="https://cloud.google.com/bare-metal/docs", - api_description="example api description", - gapic_configs=list(), - distribution_name="com.example:baremetalsolution", - ) - distribution_name, distribution_name_short = get_distribution_name(library) - self.assertEqual("com.example:baremetalsolution", distribution_name) - self.assertEqual("baremetalsolution", distribution_name_short) - def __setup_postprocessing_prerequisite_files( self, combination: int, diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 4f8208fd55..59f238aaea 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -16,7 +16,6 @@ import subprocess import os import shutil -import re from pathlib import Path from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig @@ -183,25 +182,6 @@ def pull_api_definition( ) -def get_distribution_name(library: LibraryConfig) -> tuple[str, str]: - """ - Determines the full Maven coordinate and a shortened name for a library. - - :param library: the library configuration - :return: A tuple containing: the full Maven coordinate in the format - "groupId:artifactId"; a shortened version of the artifact ID. - """ - cloud_prefix = "cloud-" if library.cloud_api else "" - library_name = library.get_library_name() - distribution_name = ( - library.distribution_name - if library.distribution_name - else f"{library.group_id}:google-{cloud_prefix}{library_name}" - ) - distribution_name_short = re.split(r"[:/]", distribution_name)[-1] - return distribution_name, distribution_name_short - - def generate_postprocessing_prerequisite_files( config: GenerationConfig, library: LibraryConfig, @@ -223,8 +203,8 @@ def generate_postprocessing_prerequisite_files( :param language: programming language of the library :return: None """ - distribution_name, distribution_name_short = get_distribution_name(library) library_name = library.get_library_name() + artifact_id = library.get_artifact_id() if config.contains_common_protos(): repo = SDK_PLATFORM_JAVA elif config.is_monorepo(): @@ -237,7 +217,7 @@ def generate_postprocessing_prerequisite_files( client_documentation = ( library.client_documentation if library.client_documentation - else f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" + else f"https://cloud.google.com/{language}/docs/reference/{artifact_id}/latest/overview" ) # The mapping is needed because transport in .repo-metadata.json @@ -260,7 +240,7 @@ def generate_postprocessing_prerequisite_files( "language": language, "repo": repo, "repo_short": f"{language}-{library_name}", - "distribution_name": distribution_name, + "distribution_name": library.get_maven_coordinate(), "api_id": api_id, "library_type": library.library_type, "requires_billing": library.requires_billing, @@ -311,7 +291,7 @@ def generate_postprocessing_prerequisite_files( render( template_name="owlbot.yaml.monorepo.j2", output_name=path_to_owlbot_yaml_file, - artifact_name=distribution_name_short, + artifact_id=artifact_id, proto_path=remove_version_from(proto_path), module_name=repo_metadata["repo_short"], api_shortname=library.api_shortname, From e5dc8626b91f56630b0ddaf0295eb02271043f77 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 3 Jul 2024 17:55:55 -0400 Subject: [PATCH 18/19] refactor --- library_generation/model/library_config.py | 20 ++++++++++++++----- library_generation/model/repo_config.py | 7 +++---- .../test/model/library_config_unit_tests.py | 13 ++++++++++++ 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index 6e015e4a23..9aef007bd8 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -12,13 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import re from hashlib import sha1 - from typing import Optional from library_generation.model.gapic_config import GapicConfig +MAVEN_COORDINATE_SEPARATOR = ":" + + class LibraryConfig: """ Class that represents a library in a generation_config.yaml file @@ -88,25 +89,34 @@ def get_library_name(self) -> str: def get_sorted_gapic_configs(self) -> list[GapicConfig]: return sorted(self.gapic_configs) - def get_maven_coordinate(self): + def get_maven_coordinate(self) -> str: """ Returns the Maven coordinate (group_id:artifact_id) of the library """ return self.distribution_name - def get_artifact_id(self): + def get_artifact_id(self) -> str: """ Returns the artifact ID of the library """ - return re.split(r"[:/]", self.distribution_name)[-1] + return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1] def __get_distribution_name(self, distribution_name: Optional[str]) -> str: + LibraryConfig.__check_distribution_name(distribution_name) if distribution_name: return distribution_name cloud_prefix = "cloud-" if self.cloud_api else "" library_name = self.get_library_name() return f"{self.group_id}:google-{cloud_prefix}{library_name}" + @staticmethod + def __check_distribution_name(distribution_name: str) -> None: + if not distribution_name: + return + sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR) + if len(sections) != 2: + raise ValueError(f"{distribution_name} is not a valid distribution name.") + def __eq__(self, other): return ( self.api_shortname == other.api_shortname diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index b014f4db8c..fb7037fb28 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -41,7 +41,7 @@ def __init__( self.output_folder = output_folder self.libraries = libraries self.versions_file = versions_file - self.library_versions = RepoConfig.__parse_version_from(self.versions_file) + self.library_versions = self.__parse_versions() def get_libraries(self) -> dict[str, LibraryConfig]: return self.libraries @@ -56,10 +56,9 @@ def get_library_version(self, artifact_id: str) -> str: """ return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) - @staticmethod - def __parse_version_from(version_file: str) -> dict[str, str]: + def __parse_versions(self) -> dict[str, str]: library_versions = dict() - with open(version_file) as f: + with open(self.versions_file) as f: for line in f.readlines(): sections = line.split(":") # skip comments and whitespace. diff --git a/library_generation/test/model/library_config_unit_tests.py b/library_generation/test/model/library_config_unit_tests.py index c161cf51f2..5d54737ced 100644 --- a/library_generation/test/model/library_config_unit_tests.py +++ b/library_generation/test/model/library_config_unit_tests.py @@ -65,6 +65,19 @@ def test_get_sorted_gapic_configs_returns_correct_order(self): library.get_sorted_gapic_configs(), ) + def test_init_invalid_distribution_name_raise_value_error(self): + self.assertRaisesRegex( + ValueError, + "fake-distribution-name is not a valid distribution name.", + LibraryConfig, + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="fake-distribution-name", + ) + def test_get_distribution_name_cloud_api(self): library = LibraryConfig( api_shortname="baremetalsolution", From 88d46663dfc272913475573ee28bc6b7b9302dee Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 4 Jul 2024 09:54:31 -0400 Subject: [PATCH 19/19] change comment --- library_generation/model/repo_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index fb7037fb28..520c505823 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -61,7 +61,7 @@ def __parse_versions(self) -> dict[str, str]: with open(self.versions_file) as f: for line in f.readlines(): sections = line.split(":") - # skip comments and whitespace. + # skip comments and empty lines. if len(sections) != 3: continue artifact_id = sections[0]