diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml index 8b1efeb01e..0eee4025d3 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml @@ -15,9 +15,10 @@ timeout: 7200s # 2 hours substitutions: _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -44,4 +45,4 @@ steps: id: native-java-kms env: - 'MODULES_UNDER_TEST=java-kms' - - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test' + - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod' diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml index 9aab5964fd..d6d0671483 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml @@ -15,9 +15,10 @@ timeout: 7200s # 2 hours substitutions: _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -44,4 +45,4 @@ steps: id: native-java-kmsinventory env: - 'MODULES_UNDER_TEST=java-kmsinventory' - - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test' + - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod' diff --git a/.cloudbuild/graalvm/cloudbuild-test-a.yaml b/.cloudbuild/graalvm/cloudbuild-test-a.yaml index 8ce2ac9a59..a101db5564 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a.yaml @@ -14,10 +14,11 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _SHARED_DEPENDENCIES_VERSION: '3.40.0' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -41,4 +42,5 @@ steps: entrypoint: bash args: [ './.kokoro/presubmit/showcase-native.sh' ] waitFor: [ "graalvm-a-build" ] - id: native-showcase \ No newline at end of file + id: native-showcase + diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml index 7d5bdc407f..9e1625da88 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml @@ -15,9 +15,10 @@ timeout: 7200s # 2 hours substitutions: _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -43,4 +44,4 @@ steps: waitFor: [ "graalvm-b-build" ] env: - 'MODULES_UNDER_TEST=java-kms' - - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test' + - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod' diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml index 4f25eeef12..c8945500b1 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml @@ -15,9 +15,10 @@ timeout: 7200s # 2 hours substitutions: _SHARED_DEPENDENCIES_VERSION: '3.30.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -44,4 +45,4 @@ steps: id: native-java-kmsinventory env: - 'MODULES_UNDER_TEST=java-kmsinventory' - - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test' + - 'GOOGLE_CLOUD_PROJECT=java-graalvm-ci-prod' diff --git a/.cloudbuild/graalvm/cloudbuild-test-b.yaml b/.cloudbuild/graalvm/cloudbuild-test-b.yaml index 36fada606d..bd0ad1ff85 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b.yaml @@ -14,10 +14,11 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _SHARED_DEPENDENCIES_VERSION: '3.40.0' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild.yaml b/.cloudbuild/graalvm/cloudbuild.yaml index 34572e8c0f..8904bf1318 100644 --- a/.cloudbuild/graalvm/cloudbuild.yaml +++ b/.cloudbuild/graalvm/cloudbuild.yaml @@ -14,8 +14,8 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} - _JAVA_SHARED_CONFIG_VERSION: '1.11.3' + _SHARED_DEPENDENCIES_VERSION: '3.40.0' # {x-version-update:google-cloud-shared-dependencies:current} + _JAVA_SHARED_CONFIG_VERSION: '1.12.0' steps: # GraalVM A build - name: gcr.io/cloud-builders/docker @@ -45,6 +45,8 @@ steps: id: graalvm-b-build waitFor: [ "-" ] +options: + logging: CLOUD_LOGGING_ONLY images: - gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION} diff --git a/.cloudbuild/graalvm/graalvm_a.Dockerfile b/.cloudbuild/graalvm/graalvm_a.Dockerfile index 40fa809aa8..ce8a048e43 100644 --- a/.cloudbuild/graalvm/graalvm_a.Dockerfile +++ b/.cloudbuild/graalvm/graalvm_a.Dockerfile @@ -15,5 +15,5 @@ ARG JAVA_SHARED_CONFIG_VERSION -FROM gcr.io/cloud-devrel-public-resources/graalvm_a:$JAVA_SHARED_CONFIG_VERSION +FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_a:$JAVA_SHARED_CONFIG_VERSION diff --git a/.cloudbuild/graalvm/graalvm_b.Dockerfile b/.cloudbuild/graalvm/graalvm_b.Dockerfile index 8bd84e53ed..51611691b1 100644 --- a/.cloudbuild/graalvm/graalvm_b.Dockerfile +++ b/.cloudbuild/graalvm/graalvm_b.Dockerfile @@ -15,4 +15,4 @@ ARG JAVA_SHARED_CONFIG_VERSION -FROM gcr.io/cloud-devrel-public-resources/graalvm_b:$JAVA_SHARED_CONFIG_VERSION \ No newline at end of file +FROM us-docker.pkg.dev/java-graalvm-ci-prod/graalvm-integration-testing/graalvm_b:$JAVA_SHARED_CONFIG_VERSION \ No newline at end of file diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml index 81691fa572..95ee665f3e 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation" - _GAPIC_GENERATOR_JAVA_VERSION: '2.46.2-SNAPSHOT' # {x-version-update:gapic-generator-java:current} + _GAPIC_GENERATOR_JAVA_VERSION: '2.50.0' # {x-version-update:gapic-generator-java:current} _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest" _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" @@ -30,6 +30,8 @@ steps: "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."] id: library-generation-build waitFor: ["-"] + env: + - 'DOCKER_BUILDKIT=1' options: logging: CLOUD_LOGGING_ONLY diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml index a45b02b9bd..db56519519 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation" - _GAPIC_GENERATOR_JAVA_VERSION: '2.49.0' # {x-version-update:gapic-generator-java:current} + _GAPIC_GENERATOR_JAVA_VERSION: '2.50.0' # {x-version-update:gapic-generator-java:current} _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest" _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" @@ -30,6 +30,8 @@ steps: "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."] id: library-generation-build waitFor: ["-"] + env: + - 'DOCKER_BUILDKIT=1' options: logging: CLOUD_LOGGING_ONLY diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 13fd53dd18..6f7b4ef1a4 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -14,35 +14,72 @@ # install gapic-generator-java in a separate layer so we don't overload the image # with the transferred source code and jars -FROM gcr.io/cloud-devrel-public-resources/java21@sha256:2ceff5eeea72260258df56d42e44ed413e52ee421c1b77393c5f2c9c4d7c41da AS ggj-build + +# 3.9.9-eclipse-temurin-11-alpine +FROM docker.io/library/maven@sha256:006d25558f9d5244ed55b5d2bd8eaf34d883e447d0c4b940e67b9f44d21167bf AS ggj-build WORKDIR /sdk-platform-java COPY . . # {x-version-update-start:gapic-generator-java:current} -ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.0" +ENV DOCKER_GAPIC_GENERATOR_VERSION="2.50.0" # {x-version-update-end} RUN mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ "./gapic-generator-java.jar" -# build from the root of this repo: -FROM gcr.io/cloud-devrel-public-resources/python@sha256:9c5ea427632f195ad164054831968389d86fdde4a15abca651f3fcb2a71268cb +# alpine:3.20.3 +FROM docker.io/library/alpine@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d as glibc-compat + +RUN apk add git sudo +# This SHA is the latest known-to-work version of this binary compatibility tool +ARG GLIB_MUS_SHA=7717dd4dc26377dd9cedcc92b72ebf35f9e68a2d +WORKDIR /home + +# Install compatibility layer to run glibc-based programs (such as the +# grpc plugin). +# Alpine, by default, only supports musl-based binaries, and there is no public +# downloadable distribution of the grpc plugin that is Alpine (musl) compatible. +# This is one of the recommended approaches to ensure glibc-compatibility +# as per https://wiki.alpinelinux.org/wiki/Running_glibc_programs +RUN git clone https://gitlab.com/manoel-linux1/GlibMus-HQ.git +WORKDIR /home/GlibMus-HQ +# We lock the tool to the latest known-to-work version +RUN git checkout "${GLIB_MUS_SHA}" +RUN chmod a+x compile-x86_64-alpine-linux.sh +RUN sh compile-x86_64-alpine-linux.sh + +# python:3.12.7-alpine3.20 +FROM docker.io/library/python@sha256:38e179a0f0436c97ecc76bcd378d7293ab3ee79e4b8c440fdc7113670cb6e204 as final -SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3 ARG PROTOC_VERSION=25.5 -ARG GRPC_VERSION=1.67.1 +ARG GRPC_VERSION=1.68.1 ARG JAVA_FORMAT_VERSION=1.7 ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq \ - && apt-get clean +RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git + +SHELL [ "/bin/bash", "-c" ] + +# Copy glibc shared objects to enable execution of the grpc plugin. +# This list was obtained via `libtree -pvvv /grpc/*` in the final container as +# well as inspecting the modifications done by compile-x86_64-alpine-linux.sh +# in the glibc-compat stage using the `dive` command. +COPY --from=glibc-compat /etc/libgcc* /etc/ +COPY --from=glibc-compat /lib64/ld-linux-x86-64.so.2 /lib64/ +COPY --from=glibc-compat /lib/GLIBCFAKE.so.0 /lib/ +COPY --from=glibc-compat /lib/ld-linux-x86-64.so.2 /lib/ +COPY --from=glibc-compat /lib/libpthread* /lib/ +COPY --from=glibc-compat /lib/libucontext* /lib/ +COPY --from=glibc-compat /lib/libc.* /lib/ +COPY --from=glibc-compat /usr/lib/libgcc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libstdc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libobstack* /usr/lib/ # copy source code COPY hermetic_build/common /src/common @@ -72,10 +109,6 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -# use python 3.12 (the base image has several python versions; here we define the default one) -RUN rm $(which python3) -RUN ln -s $(which python3.12) /usr/local/bin/python -RUN ln -s $(which python3.12) /usr/local/bin/python3 RUN python -m pip install --upgrade pip # install main scripts as a python package @@ -85,16 +118,6 @@ RUN python -m pip install src/common RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt RUN python -m pip install src/library_generation -# Install nvm with node and npm -ENV NODE_VERSION 20.12.0 -WORKDIR /home -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash -RUN chmod o+rx /home/.nvm -ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin -ENV PATH=${PATH}:${NODE_PATH} -RUN node --version -RUN npm --version - # install the owl-bot CLI WORKDIR /tools RUN git clone https://github.com/googleapis/repo-automation-bots @@ -102,8 +125,7 @@ WORKDIR /tools/repo-automation-bots/packages/owl-bot RUN git checkout "${OWLBOT_CLI_COMMITTISH}" RUN npm i && npm run compile && npm link RUN owl-bot copy-code --version -RUN chmod -R o+rx ${NODE_PATH} -RUN ln -sf ${NODE_PATH}/* /usr/local/bin +RUN chmod o+rx $(which owl-bot) # download the Java formatter ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \ @@ -120,7 +142,6 @@ RUN git config --system user.name "Cloud Java Bot" # allow read-write for /home and execution for binaries in /home/.nvm RUN chmod -R a+rw /home -RUN chmod -R a+rx /home/.nvm WORKDIR /workspace ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 7c4ae22f4e..5e8b55660f 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -60,8 +60,6 @@ runs: cd "${GITHUB_WORKSPACE}" pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt pip install hermetic_build/release_note_generation - name: Generate changed libraries diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 59fc82fc12..3c2a7d0e47 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -81,9 +81,6 @@ git checkout "${current_branch}" # copy generation configuration from target branch to current branch. git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" -# get .m2 folder so it's mapped into the docker container -m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)") - # download api definitions from googleapis repository googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs) api_def_dir=$(mktemp -d) @@ -92,18 +89,23 @@ pushd "${api_def_dir}" git checkout "${googleapis_commitish}" popd +# get changed library list. +changed_libraries=$(python hermetic_build/common/cli/get_changed_libraries.py create \ + --baseline-generation-config-path="${baseline_generation_config}" \ + --current-generation-config-path="${generation_config}") +echo "Changed libraries are: ${changed_libraries:-"No changed library"}." + # run hermetic code generation docker image. docker run \ --rm \ --quiet \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ - -v "${m2_folder}":/home/.m2 \ -v "${api_def_dir}:${workspace_name}/googleapis" \ -e GENERATOR_VERSION="${image_tag}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ - --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ - --current-generation-config-path="${workspace_name}/${generation_config}" \ + --generation-config-path="${workspace_name}/${generation_config}" \ + --library-names="${changed_libraries}" \ --api-definitions-path="${workspace_name}/googleapis" python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 32f4a9e84e..de3babb001 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -14,8 +14,12 @@ branchProtectionRules: - build (17) - cla/google - compatibility - - graalvm-presubmit-sdk-platform-java-a (cloud-devrel-kokoro-resources) - - graalvm-presubmit-sdk-platform-java-b (cloud-devrel-kokoro-resources) + - graalvm-presubmit-sdk-platform-java-a (java-graalvm-ci-prod) + - graalvm-presubmit-sdk-platform-java-b (java-graalvm-ci-prod) + - graalvm-presubmit-sdk-platform-java-a-downstream-kms (java-graalvm-ci-prod) + - graalvm-presubmit-sdk-platform-java-b-downstream-kms (java-graalvm-ci-prod) + - graalvm-presubmit-sdk-platform-java-a-downstream-kmsinventory (java-graalvm-ci-prod) + - graalvm-presubmit-sdk-platform-java-b-downstream-kmsinventory (java-graalvm-ci-prod) - library_generation - library-generation-integration-tests - library-generation-lint-python diff --git a/.gitignore b/.gitignore index 6768e8cd69..b1b8440011 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ target/ **/build/ **/dist/ library_generation/**/*.jar - +**/google-java-format.jar diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 841dcc60c0..11ee7212d7 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "2.49.0" + ".": "2.50.0" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 662e2894c8..bedb3aa241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,37 @@ # Changelog +## [2.50.0](https://github.com/googleapis/sdk-platform-java/compare/v2.49.0...v2.50.0) (2024-11-14) + + +### Features + +* Add experimental S2A integration in client libraries grpc transport ([#3326](https://github.com/googleapis/sdk-platform-java/issues/3326)) ([1138ca6](https://github.com/googleapis/sdk-platform-java/commit/1138ca682cd47d6164ceaa47803bfe2f68b1bc14)) +* enable selective generation based on service config include list ([#3323](https://github.com/googleapis/sdk-platform-java/issues/3323)) ([0cddadb](https://github.com/googleapis/sdk-platform-java/commit/0cddadb8ad3eddfffa356a479964d8a720937503)) +* introduce `java.time` to java-core ([#3330](https://github.com/googleapis/sdk-platform-java/issues/3330)) ([f202c3b](https://github.com/googleapis/sdk-platform-java/commit/f202c3b550936168b9876860853876aa6d51c6a1)) +* Update Gapic-Generator to generate libraries using `java.time` methods ([#3321](https://github.com/googleapis/sdk-platform-java/issues/3321)) ([b21c9a4](https://github.com/googleapis/sdk-platform-java/commit/b21c9a42121c22a1ab229d2d485265c271305110)) + + +### Bug Fixes + +* Fix flaky test ScheduledRetryingExecutorTest.testCancelOuterFutureAfterStart ([#3335](https://github.com/googleapis/sdk-platform-java/issues/3335)) ([e73740d](https://github.com/googleapis/sdk-platform-java/commit/e73740dbdb21d7c28908554fe3725504dc8ce84b)) +* httpjson callables to trace attempts (started, failed) ([#3300](https://github.com/googleapis/sdk-platform-java/issues/3300)) ([15a64ee](https://github.com/googleapis/sdk-platform-java/commit/15a64ee2e63165e50fd07b2b3a40f0d2ef2edfe2)) +* instantiate GaxProperties at build time to ensure we get the protobuf version ([#3365](https://github.com/googleapis/sdk-platform-java/issues/3365)) ([bb2a3be](https://github.com/googleapis/sdk-platform-java/commit/bb2a3be87291ae718ac0e8538025a0867a6b6ff6)) +* protobuf version not always getting set in headers ([#3322](https://github.com/googleapis/sdk-platform-java/issues/3322)) ([7f6e470](https://github.com/googleapis/sdk-platform-java/commit/7f6e470fea1673a5cf50fe3b49263615a172afde)) +* use BuildKit instead of legacy builder to build the Hermetic Build images ([#3338](https://github.com/googleapis/sdk-platform-java/issues/3338)) ([222fb45](https://github.com/googleapis/sdk-platform-java/commit/222fb452e00bd195ad51389ea308993a7e1bc956)) + + +### Dependencies + +* update google auth library dependencies to v1.30.0 ([#3367](https://github.com/googleapis/sdk-platform-java/issues/3367)) ([a31c682](https://github.com/googleapis/sdk-platform-java/commit/a31c68232584bf90bc00ace8310adeab8fa26add)) +* update grpc dependencies to v1.68.1 ([#3240](https://github.com/googleapis/sdk-platform-java/issues/3240)) ([c8e3941](https://github.com/googleapis/sdk-platform-java/commit/c8e3941ef6f5bd1236f5ceedfd488e5113928471)) + + +### Documentation + +* fix list num ([#3356](https://github.com/googleapis/sdk-platform-java/issues/3356)) ([b7d6296](https://github.com/googleapis/sdk-platform-java/commit/b7d62968cd837a7addc06da8b9bc2131c36c7fbc)) +* **hermetic-build:** indicate usage of Docker Buildkit in development guide ([#3337](https://github.com/googleapis/sdk-platform-java/issues/3337)) ([01e742d](https://github.com/googleapis/sdk-platform-java/commit/01e742de49e151efbd903808a859f595f99bc8de)) +* modify hermetic build docs ([#3331](https://github.com/googleapis/sdk-platform-java/issues/3331)) ([25023af](https://github.com/googleapis/sdk-platform-java/commit/25023afad8f483a3eae8846c4bc1fbe2c7a260c5)) + ## [2.49.0](https://github.com/googleapis/sdk-platform-java/compare/v2.48.0...v2.49.0) (2024-10-25) diff --git a/WORKSPACE b/WORKSPACE index 93d210c127..71e2af2758 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -58,14 +58,13 @@ load("@rules_jvm_external//:defs.bzl", "maven_install") load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS") load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS") -_gapic_generator_java_version = "2.49.0" # {x-version-update:gapic-generator-java:current} +_gapic_generator_java_version = "2.50.0" # {x-version-update:gapic-generator-java:current} maven_install( artifacts = [ "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, ] + PROTOBUF_MAVEN_ARTIFACTS + IO_GRPC_GRPC_JAVA_ARTIFACTS, fail_on_missing_checksum = False, - override_targets = IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS, repositories = [ "m2Local", "https://repo.maven.apache.org/maven2/", diff --git a/api-common-java/pom.xml b/api-common-java/pom.xml index a7bf98f6c7..07f447df37 100644 --- a/api-common-java/pom.xml +++ b/api-common-java/pom.xml @@ -5,14 +5,14 @@ com.google.api api-common jar - 2.40.0 + 2.41.0 API Common Common utilities for Google APIs in Java com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 ../gapic-generator-java-pom-parent diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index 65e30ed999..65269d7aa0 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -31,22 +31,22 @@ com.google.api gax - 2.57.0 + 2.58.0 com.google.api gax-grpc - 2.57.0 + 2.58.0 com.google.api gax-httpjson - 2.57.0 + 2.58.0 com.google.api api-common - 2.40.0 + 2.41.0 diff --git a/gapic-generator-java-bom/pom.xml b/gapic-generator-java-bom/pom.xml index 5899626b81..5d70b96524 100644 --- a/gapic-generator-java-bom/pom.xml +++ b/gapic-generator-java-bom/pom.xml @@ -4,7 +4,7 @@ com.google.api gapic-generator-java-bom pom - 2.49.0 + 2.50.0 GAPIC Generator Java BOM BOM for the libraries in gapic-generator-java repository. Users should not @@ -15,7 +15,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 ../gapic-generator-java-pom-parent @@ -75,61 +75,61 @@ com.google.api api-common - 2.40.0 + 2.41.0 com.google.api gax-bom - 2.57.0 + 2.58.0 pom import com.google.api gapic-generator-java - 2.49.0 + 2.50.0 com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.49.0 com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.49.0 com.google.api.grpc proto-google-iam-v1 - 1.43.0 + 1.44.0 com.google.api.grpc proto-google-iam-v2 - 1.43.0 + 1.44.0 com.google.api.grpc proto-google-iam-v2beta - 1.43.0 + 1.44.0 com.google.api.grpc grpc-google-iam-v1 - 1.43.0 + 1.44.0 com.google.api.grpc grpc-google-iam-v2 - 1.43.0 + 1.44.0 com.google.api.grpc grpc-google-iam-v2beta - 1.43.0 + 1.44.0 diff --git a/gapic-generator-java-pom-parent/pom.xml b/gapic-generator-java-pom-parent/pom.xml index d0c1ddc413..2dc1174f94 100644 --- a/gapic-generator-java-pom-parent/pom.xml +++ b/gapic-generator-java-pom-parent/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 pom GAPIC Generator Java POM Parent https://github.com/googleapis/sdk-platform-java @@ -15,7 +15,7 @@ com.google.cloud google-cloud-shared-config - 1.11.3 + 1.12.0 @@ -26,8 +26,8 @@ 1.3.2 - 1.67.1 - 1.29.0 + 1.68.1 + 1.30.0 1.45.0 2.11.0 33.3.1-jre diff --git a/gapic-generator-java/pom.xml b/gapic-generator-java/pom.xml index 9c163e79d4..050a393f7c 100644 --- a/gapic-generator-java/pom.xml +++ b/gapic-generator-java/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.google.api gapic-generator-java - 2.49.0 + 2.50.0 GAPIC Generator Java GAPIC generator Java @@ -22,7 +22,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 ../gapic-generator-java-pom-parent @@ -31,7 +31,7 @@ com.google.api gapic-generator-java-bom - 2.49.0 + 2.50.0 pom import diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java index ae23a0aedd..500d330d30 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java @@ -104,6 +104,7 @@ import com.google.longrunning.Operation; import com.google.protobuf.Empty; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -120,7 +121,6 @@ import java.util.stream.Collectors; import javax.annotation.Generated; import javax.annotation.Nullable; -import org.threeten.bp.Duration; public abstract class AbstractServiceStubSettingsClassComposer implements ClassComposer { private static final Statement EMPTY_LINE_STATEMENT = EmptyLineStatement.create(); diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java index e5e4ad4195..6690c57674 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java @@ -390,7 +390,7 @@ public static Expr createBatchingBuilderSettingsExpr( batchingSettingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(batchingSettingsBuilderExpr) - .setMethodName("setDelayThreshold") + .setMethodName("setDelayThresholdDuration") .setArguments( createDurationOfMillisExpr(toValExpr(batchingSettings.delayThresholdMillis()))) .build(); @@ -511,7 +511,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setInitialRetryDelay") + .setMethodName("setInitialRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(retryPolicy.getInitialBackoff()))) .build(); @@ -528,7 +528,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(retryPolicy.getMaxBackoff()))) .build(); } @@ -537,7 +537,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setInitialRpcTimeout") + .setMethodName("setInitialRpcTimeoutDuration") .setArguments(createDurationOfMillisExpr(toValExpr(settings.timeout()))) .build(); } @@ -553,7 +553,8 @@ private static List createRetrySettingsExprs( .build(); if (!settings.kind().equals(GapicRetrySettings.Kind.NONE)) { - for (String setterMethodName : Arrays.asList("setMaxRpcTimeout", "setTotalTimeout")) { + for (String setterMethodName : + Arrays.asList("setMaxRpcTimeoutDuration", "setTotalTimeoutDuration")) { settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) @@ -614,7 +615,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setInitialRetryDelay") + .setMethodName("setInitialRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(initialPollDelayMillis))) .build(); @@ -628,7 +629,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(maxPollDelayMillis))) .build(); @@ -638,7 +639,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setInitialRpcTimeout") + .setMethodName("setInitialRpcTimeoutDuration") .setArguments(zeroDurationExpr) .build(); @@ -654,14 +655,14 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setMaxRpcTimeout") + .setMethodName("setMaxRpcTimeoutDuration") .setArguments(zeroDurationExpr) .build(); lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setTotalTimeout") + .setMethodName("setTotalTimeoutDuration") .setArguments(createDurationOfMillisExpr(toValExpr(totalPollTimeoutMillis))) .build(); @@ -714,7 +715,7 @@ private static TypeStore createStaticTypes() { List> concreteClazzes = Arrays.asList( BatchingSettings.class, - org.threeten.bp.Duration.class, + java.time.Duration.class, FlowControlSettings.class, FlowController.LimitExceededBehavior.class, ImmutableMap.class, diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java index 460c8a443a..57d396847e 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java @@ -70,15 +70,15 @@ public static Optional composeSettingsSample( .build(); // Builder with set value method - // e.g foobarSettingBuilder.fooSetting().setRetrySettings( - // echoSettingsBuilder.echoSettings().getRetrySettings().toBuilder().setTotalTimeout(Duration.ofSeconds(30)).build()); + // e.g. foobarSettingBuilder.fooSetting().setRetrySettings( + // echoSettingsBuilder.echoSettings().getRetrySettings().toBuilder() + // .setTotalTimeoutDuration(Duration.ofSeconds(30)).build()); MethodInvocationExpr settingBuilderMethodInvocationExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(localSettingsVarExpr) .setMethodName( JavaStyle.toLowerCamelCase(String.format("%sSettings", methodNameOpt.get()))) .build(); - String disambiguation = "Settings"; MethodInvocationExpr retrySettingsArgExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingBuilderMethodInvocationExpr) @@ -364,7 +364,7 @@ public static Optional composeLroSettingsSample( retrySettingsArgExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(retrySettingsArgExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(ofFiveThousandMillisMethodInvocationExpr) .build(); retrySettingsArgExpr = diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java index 9a2da74747..2e17b9026b 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java @@ -14,6 +14,7 @@ package com.google.api.generator.gapic.protoparser; +import com.google.api.ClientLibrarySettings; import com.google.api.ClientProto; import com.google.api.DocumentationRule; import com.google.api.FieldBehavior; @@ -84,6 +85,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -160,11 +162,11 @@ public static GapicContext parse(CodeGeneratorRequest request) { messages = updateResourceNamesInMessages(messages, resourceNames.values()); // Contains only resource names that are actually used. Usage refers to the presence of a - // request message's field in an RPC's method_signature annotation. That is, resource name - // definitions - // or references that are simply defined, but not used in such a manner, will not have - // corresponding Java helper - // classes generated. + // request message's field in an RPC's method_signature annotation. That is, resource name + // definitions or references that are simply defined, but not used in such a manner, + // will not have corresponding Java helper classes generated. + // If selective api generation is configured via service yaml, Java helper classes are only + // generated if resource names are actually used by methods selected to generate. Set outputArgResourceNames = new HashSet<>(); List mixinServices = new ArrayList<>(); Transport transport = Transport.parse(transportOpt.orElse(Transport.GRPC.toString())); @@ -425,6 +427,71 @@ public static List parseService( Transport.GRPC); } + static boolean shouldIncludeMethodInGeneration( + MethodDescriptor method, + Optional serviceYamlProtoOpt, + String protoPackage) { + // default to include all when no service yaml or no library setting section. + if (!serviceYamlProtoOpt.isPresent() + || serviceYamlProtoOpt.get().getPublishing().getLibrarySettingsCount() == 0) { + return true; + } + List librarySettingsList = + serviceYamlProtoOpt.get().getPublishing().getLibrarySettingsList(); + // Validate for logging purpose, this should be validated upstream. + // If library_settings.version does not match with proto package name + // Give warnings and disregard this config. default to include all. + if (!librarySettingsList.get(0).getVersion().isEmpty() + && !protoPackage.equals(librarySettingsList.get(0).getVersion())) { + if (LOGGER.isLoggable(Level.WARNING)) { + LOGGER.warning( + String.format( + "Service yaml config is misconfigured. Version in " + + "publishing.library_settings (%s) does not match proto package (%s)." + + "Disregarding selective generation settings.", + librarySettingsList.get(0).getVersion(), protoPackage)); + } + return true; + } + // librarySettingsList is technically a list, but is processed upstream and + // only leave with 1 element. Otherwise, it is a misconfiguration and + // should be caught upstream. + List includeMethodsList = + librarySettingsList + .get(0) + .getJavaSettings() + .getCommon() + .getSelectiveGapicGeneration() + .getMethodsList(); + // default to include all when nothing specified, this could be no java section + // specified in library setting, or the method list is empty + if (includeMethodsList.isEmpty()) { + return true; + } + + return includeMethodsList.contains(method.getFullName()); + } + + private static boolean isEmptyService( + ServiceDescriptor serviceDescriptor, + Optional serviceYamlProtoOpt, + String protoPackage) { + List methodsList = serviceDescriptor.getMethods(); + List methodListSelected = + methodsList.stream() + .filter( + method -> + shouldIncludeMethodInGeneration(method, serviceYamlProtoOpt, protoPackage)) + .collect(Collectors.toList()); + if (methodListSelected.isEmpty()) { + LOGGER.log( + Level.WARNING, + "Service {0} has no RPC methods and will not be generated", + serviceDescriptor.getName()); + } + return methodListSelected.isEmpty(); + } + public static List parseService( FileDescriptor fileDescriptor, Map messageTypes, @@ -433,19 +500,11 @@ public static List parseService( Optional serviceConfigOpt, Set outputArgResourceNames, Transport transport) { - + String protoPackage = fileDescriptor.getPackage(); return fileDescriptor.getServices().stream() .filter( - serviceDescriptor -> { - List methodsList = serviceDescriptor.getMethods(); - if (methodsList.isEmpty()) { - LOGGER.warning( - String.format( - "Service %s has no RPC methods and will not be generated", - serviceDescriptor.getName())); - } - return !methodsList.isEmpty(); - }) + serviceDescriptor -> + !isEmptyService(serviceDescriptor, serviceYamlProtoOpt, protoPackage)) .map( s -> { // Workaround for a missing default_host and oauth_scopes annotation from a service @@ -498,6 +557,8 @@ public static List parseService( String pakkage = TypeParser.getPackage(fileDescriptor); String originalJavaPackage = pakkage; // Override Java package with that specified in gapic.yaml. + // this override is deprecated and legacy support only + // see go/client-user-guide#configure-long-running-operation-polling-timeouts-optional if (serviceConfigOpt.isPresent() && serviceConfigOpt.get().getLanguageSettingsOpt().isPresent()) { GapicLanguageSettings languageSettings = @@ -518,6 +579,7 @@ public static List parseService( .setMethods( parseMethods( s, + protoPackage, pakkage, messageTypes, resourceNames, @@ -709,6 +771,7 @@ public static Map parseResourceNames( @VisibleForTesting static List parseMethods( ServiceDescriptor serviceDescriptor, + String protoPackage, String servicePackage, Map messageTypes, Map resourceNames, @@ -721,8 +784,10 @@ static List parseMethods( // Parse the serviceYaml for autopopulated methods and fields once and put into a map Map> autoPopulatedMethodsWithFields = parseAutoPopulatedMethodsAndFields(serviceYamlProtoOpt); - for (MethodDescriptor protoMethod : serviceDescriptor.getMethods()) { + if (!shouldIncludeMethodInGeneration(protoMethod, serviceYamlProtoOpt, protoPackage)) { + continue; + } // Parse the method. TypeNode inputType = TypeParser.parseType(protoMethod.getInputType()); Method.Builder methodBuilder = Method.builder(); diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java index 717191842a..775a0b1d09 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java @@ -119,20 +119,20 @@ void paramDefinitionsBlock_basic() { "ImmutableMap.Builder definitions = ImmutableMap.builder();\n", "RetrySettings settings = null;\n", "settings =" - + " RetrySettings.newBuilder().setInitialRetryDelay(" + + " RetrySettings.newBuilder().setInitialRetryDelayDuration(" + "Duration.ofMillis(100L)).setRetryDelayMultiplier(2.0)" - + ".setMaxRetryDelay(Duration.ofMillis(3000L))" - + ".setInitialRpcTimeout(Duration.ofMillis(10000L))" + + ".setMaxRetryDelayDuration(Duration.ofMillis(3000L))" + + ".setInitialRpcTimeoutDuration(Duration.ofMillis(10000L))" + ".setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ofMillis(10000L))" - + ".setTotalTimeout(Duration.ofMillis(10000L)).build();\n", + + ".setMaxRpcTimeoutDuration(Duration.ofMillis(10000L))" + + ".setTotalTimeoutDuration(Duration.ofMillis(10000L)).build();\n", "definitions.put(\"retry_policy_1_params\", settings);\n", "settings =" + " RetrySettings.newBuilder()" - + ".setInitialRpcTimeout(Duration.ofMillis(5000L))" + + ".setInitialRpcTimeoutDuration(Duration.ofMillis(5000L))" + ".setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ofMillis(5000L))" - + ".setTotalTimeout(Duration.ofMillis(5000L)).build();\n", + + ".setMaxRpcTimeoutDuration(Duration.ofMillis(5000L))" + + ".setTotalTimeoutDuration(Duration.ofMillis(5000L)).build();\n", "definitions.put(\"no_retry_0_params\", settings);\n", "RETRY_PARAM_DEFINITIONS = definitions.build();\n", "}\n"); @@ -341,10 +341,10 @@ void lroBuilderExpr() { + "WaitResponse.class))" + ".setMetadataTransformer(ProtoOperationTransformers.MetadataTransformer.create(" + "WaitMetadata.class)).setPollingAlgorithm(OperationTimedPollAlgorithm.create(" - + "RetrySettings.newBuilder().setInitialRetryDelay(Duration.ofMillis(5000L))" - + ".setRetryDelayMultiplier(1.5).setMaxRetryDelay(Duration.ofMillis(45000L))" - + ".setInitialRpcTimeout(Duration.ZERO).setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ZERO).setTotalTimeout(Duration.ofMillis(300000L))" + + "RetrySettings.newBuilder().setInitialRetryDelayDuration(Duration.ofMillis(5000L))" + + ".setRetryDelayMultiplier(1.5).setMaxRetryDelayDuration(Duration.ofMillis(45000L))" + + ".setInitialRpcTimeoutDuration(Duration.ZERO).setRpcTimeoutMultiplier(1.0)" + + ".setMaxRpcTimeoutDuration(Duration.ZERO).setTotalTimeoutDuration(Duration.ofMillis(300000L))" + ".build()))"); assertEquals(expected, writerVisitor.write()); } @@ -394,7 +394,7 @@ void batchingSettings_minimalFlowControlSettings() { + "BatchingSettings.newBuilder()" + ".setElementCountThreshold(100L)" + ".setRequestByteThreshold(1048576L)" - + ".setDelayThreshold(Duration.ofMillis(10L))" + + ".setDelayThresholdDuration(Duration.ofMillis(10L))" + ".setFlowControlSettings(" + "FlowControlSettings.newBuilder()" + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore)" @@ -451,7 +451,7 @@ void batchingSettings_fullFlowControlSettings() { + "BatchingSettings.newBuilder()" + ".setElementCountThreshold(1000L)" + ".setRequestByteThreshold(1048576L)" - + ".setDelayThreshold(Duration.ofMillis(50L))" + + ".setDelayThresholdDuration(Duration.ofMillis(50L))" + ".setFlowControlSettings(" + "FlowControlSettings.newBuilder()" + ".setMaxOutstandingElementCount(100000L)" diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden index 329be3111f..6d81887aed 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden @@ -22,9 +22,9 @@ import com.google.common.collect.Lists; import com.google.protobuf.Empty; import com.google.testdata.v1.FibonacciRequest; import java.io.IOException; +import java.time.Duration; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -206,10 +206,10 @@ public class DeprecatedServiceStubSettings extends StubSettings { RetrySettings settings = null; settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(2.0) - .setMaxRetryDelay(Duration.ofMillis(3000L)) - .setInitialRpcTimeout(Duration.ofMillis(10000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(3000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(10000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(10000L)) - .setTotalTimeout(Duration.ofMillis(10000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(10000L)) + .setTotalTimeoutDuration(Duration.ofMillis(10000L)) .build(); definitions.put("retry_policy_1_params", settings); settings = RetrySettings.newBuilder() - .setInitialRpcTimeout(Duration.ofMillis(5000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(5000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(5000L)) - .setTotalTimeout(Duration.ofMillis(5000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(5000L)) + .setTotalTimeoutDuration(Duration.ofMillis(5000L)) .build(); definitions.put("no_retry_0_params", settings); RETRY_PARAM_DEFINITIONS = definitions.build(); @@ -575,13 +575,13 @@ public class EchoStubSettings extends StubSettings { .setPollingAlgorithm( OperationTimedPollAlgorithm.create( RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(5000L)) + .setInitialRetryDelayDuration(Duration.ofMillis(5000L)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(45000L)) - .setInitialRpcTimeout(Duration.ZERO) + .setMaxRetryDelayDuration(Duration.ofMillis(45000L)) + .setInitialRpcTimeoutDuration(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ZERO) - .setTotalTimeout(Duration.ofMillis(300000L)) + .setMaxRpcTimeoutDuration(Duration.ZERO) + .setTotalTimeoutDuration(Duration.ofMillis(300000L)) .build())); return builder; diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden index 3b10c57af5..c199db3fee 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden @@ -54,10 +54,10 @@ import com.google.logging.v2.WriteLogEntriesRequest; import com.google.logging.v2.WriteLogEntriesResponse; import com.google.protobuf.Empty; import java.io.IOException; +import java.time.Duration; import java.util.Collection; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -557,24 +557,24 @@ public class LoggingServiceV2StubSettings extends StubSettings { RetrySettings settings = null; settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_0_params", settings); settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_1_params", settings); settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_2_params", settings); RETRY_PARAM_DEFINITIONS = definitions.build(); @@ -697,7 +697,7 @@ public class PublisherStubSettings extends StubSettings { BatchingSettings.newBuilder() .setElementCountThreshold(100L) .setRequestByteThreshold(1048576L) - .setDelayThreshold(Duration.ofMillis(10L)) + .setDelayThresholdDuration(Duration.ofMillis(10L)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore) diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden index 71cf9529ac..cd31f6581a 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden @@ -41,7 +41,7 @@ public class SyncWait { RetrySettings.newBuilder() .setInitialRetryDelayDuration(Duration.ofMillis(500)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(5000)) + .setMaxRetryDelayDuration(Duration.ofMillis(5000)) .setTotalTimeoutDuration(Duration.ofHours(24)) .build()); echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden index ee1c010647..bd2264892e 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden @@ -41,7 +41,7 @@ public class SyncWait { RetrySettings.newBuilder() .setInitialRetryDelayDuration(Duration.ofMillis(500)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(5000)) + .setMaxRetryDelayDuration(Duration.ofMillis(5000)) .setTotalTimeoutDuration(Duration.ofHours(24)) .build()); echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden index c94624b679..8ef290e405 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden @@ -90,7 +90,7 @@ import javax.annotation.Generated; * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden index 41cfa15e00..4f2603041f 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden @@ -54,9 +54,9 @@ import com.google.showcase.grpcrest.v1beta1.WaitMetadata; import com.google.showcase.grpcrest.v1beta1.WaitRequest; import com.google.showcase.grpcrest.v1beta1.WaitResponse; import java.io.IOException; +import java.time.Duration; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -123,7 +123,7 @@ import org.threeten.bp.Duration; * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder @@ -630,13 +630,13 @@ public class EchoStubSettings extends StubSettings { .setPollingAlgorithm( OperationTimedPollAlgorithm.create( RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(5000L)) + .setInitialRetryDelayDuration(Duration.ofMillis(5000L)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(45000L)) - .setInitialRpcTimeout(Duration.ZERO) + .setMaxRetryDelayDuration(Duration.ofMillis(45000L)) + .setInitialRpcTimeoutDuration(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ZERO) - .setTotalTimeout(Duration.ofMillis(300000L)) + .setMaxRpcTimeoutDuration(Duration.ZERO) + .setTotalTimeoutDuration(Duration.ofMillis(300000L)) .build())); return builder; diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java index 7745f3aff8..eb81a73a8e 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java @@ -129,7 +129,7 @@ void composeSettingsSample_serviceSettingsClass_LroMethod() { " RetrySettings.newBuilder()\n", " .setInitialRetryDelayDuration(Duration.ofMillis(500))\n", " .setRetryDelayMultiplier(1.5)\n", - " .setMaxRetryDelay(Duration.ofMillis(5000))\n", + " .setMaxRetryDelayDuration(Duration.ofMillis(5000))\n", " .setTotalTimeoutDuration(Duration.ofHours(24))\n", " .build());\n", "waitSettingsBuilder\n", @@ -159,7 +159,7 @@ void composeSettingsSample_serviceStubClass_LroMethod() { " RetrySettings.newBuilder()\n", " .setInitialRetryDelayDuration(Duration.ofMillis(500))\n", " .setRetryDelayMultiplier(1.5)\n", - " .setMaxRetryDelay(Duration.ofMillis(5000))\n", + " .setMaxRetryDelayDuration(Duration.ofMillis(5000))\n", " .setTotalTimeoutDuration(Duration.ofHours(24))\n", " .build());\n", "waitSettingsBuilder\n", diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java index 2776fec687..6e8ffa7232 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java @@ -21,9 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.api.ClientLibrarySettings; import com.google.api.FieldInfo.Format; import com.google.api.MethodSettings; import com.google.api.Publishing; +import com.google.api.PythonSettings; import com.google.api.Service; import com.google.api.generator.engine.ast.ConcreteReference; import com.google.api.generator.engine.ast.Reference; @@ -46,6 +48,7 @@ import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Descriptors.ServiceDescriptor; import com.google.protobuf.compiler.PluginProtos.CodeGeneratorRequest; +import com.google.selective.generate.v1beta1.SelectiveApiGenerationOuterClass; import com.google.showcase.v1beta1.EchoOuterClass; import com.google.showcase.v1beta1.TestingOuterClass; import com.google.testgapic.v1beta1.LockerProto; @@ -58,6 +61,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -137,6 +142,7 @@ void parseMethods_basic() { Parser.parseMethods( echoService, ECHO_PACKAGE, + ECHO_PACKAGE, messageTypes, resourceNames, Optional.empty(), @@ -200,6 +206,7 @@ void parseMethods_basicLro() { Parser.parseMethods( echoService, ECHO_PACKAGE, + ECHO_PACKAGE, messageTypes, resourceNames, Optional.empty(), @@ -705,6 +712,128 @@ void parseServiceWithNoMethodsTest() { assertEquals("EchoWithMethods", services.get(0).overriddenName()); } + @Test + void selectiveGenerationTest_shouldExcludeUnusedResourceNames() { + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + Map messageTypes = Parser.parseMessages(fileDescriptor); + Map resourceNames = Parser.parseResourceNames(fileDescriptor); + + String serviceYamlFilename = "selective_api_generation_v1beta1.yaml"; + String testFilesDirectory = "src/test/resources/"; + Path serviceYamlPath = Paths.get(testFilesDirectory, serviceYamlFilename); + Optional serviceYamlOpt = + ServiceYamlParser.parse(serviceYamlPath.toString()); + Assert.assertTrue(serviceYamlOpt.isPresent()); + + Set helperResourceNames = new HashSet<>(); + Parser.parseService( + fileDescriptor, messageTypes, resourceNames, serviceYamlOpt, helperResourceNames); + // resource Name Foobarbaz is not present + assertEquals(2, helperResourceNames.size()); + assertTrue( + helperResourceNames.stream() + .map(ResourceName::variableName) + .collect(Collectors.toSet()) + .containsAll(ImmutableList.of("foobar", "anythingGoes"))); + } + + @Test + void selectiveGenerationTest_shouldGenerateOnlySelectiveMethods() { + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + Map messageTypes = Parser.parseMessages(fileDescriptor); + Map resourceNames = Parser.parseResourceNames(fileDescriptor); + + // test with service yaml file to show usage of this feature, test itself + // can be done without this file and build a Service object from code. + String serviceYamlFilename = "selective_api_generation_v1beta1.yaml"; + String testFilesDirectory = "src/test/resources/"; + Path serviceYamlPath = Paths.get(testFilesDirectory, serviceYamlFilename); + Optional serviceYamlOpt = + ServiceYamlParser.parse(serviceYamlPath.toString()); + Assert.assertTrue(serviceYamlOpt.isPresent()); + + List services = + Parser.parseService( + fileDescriptor, messageTypes, resourceNames, serviceYamlOpt, new HashSet<>()); + assertEquals(1, services.size()); + assertEquals("EchoServiceShouldGeneratePartial", services.get(0).overriddenName()); + assertEquals(3, services.get(0).methods().size()); + for (Method method : services.get(0).methods()) { + assertTrue(method.name().contains("ShouldInclude")); + } + } + + @Test + void selectiveGenerationTest_shouldGenerateAllIfNoPublishingSectionInServiceYaml() { + Service service = + Service.newBuilder() + .setTitle("Selective generation testing with no publishing section") + .build(); + Publishing publishing = service.getPublishing(); + Assert.assertEquals(0, publishing.getLibrarySettingsCount()); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + String protoPackage = "google.selective.generate.v1beta1"; + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + + @Test + void selectiveGenerationTest_shouldIncludeMethodInGenerationWhenProtoPackageMismatch() { + String protoPackage = "google.selective.generate.v1beta1"; + + // situation where service yaml has different version stated + ClientLibrarySettings clientLibrarySettings = + ClientLibrarySettings.newBuilder().setVersion("google.selective.generate.v1").build(); + Publishing publishing = + Publishing.newBuilder().addLibrarySettings(clientLibrarySettings).build(); + Service service = + Service.newBuilder() + .setTitle( + "Selective generation test when proto package " + + "does not match library_settings version from service yaml") + .setPublishing(publishing) + .build(); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + + @Test + void selectiveGenerationTest_shouldGenerateAllIfNoJavaSectionInServiceYaml() { + String protoPackage = "google.selective.generate.v1beta1"; + + // situation where service yaml has other language settings but no + // java settings in library_settings. + ClientLibrarySettings clientLibrarySettings = + ClientLibrarySettings.newBuilder() + .setVersion(protoPackage) + .setPythonSettings(PythonSettings.newBuilder().build()) + .build(); + Publishing publishing = + Publishing.newBuilder().addLibrarySettings(clientLibrarySettings).build(); + Service service = + Service.newBuilder() + .setTitle( + "Selective generation test when no java section in " + + "library_settings from service yaml") + .setPublishing(publishing) + .build(); + + Assert.assertEquals(1, publishing.getLibrarySettingsCount()); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + private void assertMethodArgumentEquals( String name, TypeNode type, List nestedFields, MethodArgument argument) { assertEquals(name, argument.name()); diff --git a/gapic-generator-java/src/test/proto/selective_api_generation.proto b/gapic-generator-java/src/test/proto/selective_api_generation.proto new file mode 100644 index 0000000000..06da2c2e41 --- /dev/null +++ b/gapic-generator-java/src/test/proto/selective_api_generation.proto @@ -0,0 +1,163 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/field_info.proto"; +import "google/api/resource.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "google/rpc/status.proto"; + +package google.selective.generate.v1beta1; + +option java_package = "com.google.selective.generate.v1beta1"; +option java_multiple_files = true; +option java_outer_classname = "SelectiveApiGenerationOuterClass"; + +// resource not tied to message +option (google.api.resource_definition) = { + type: "showcase.googleapis.com/AnythingGoes" + pattern: "*" +}; + +// This proto is used to test selective api generation +// covered scenarios: +// - A service with several rpcs, part of them should be generated +// - A service with several rpcs, none of them should be generated +// This proto should be tested side-by-side with yaml file: +// - selective_api_generation_v1beta1.yaml + +service EchoServiceShouldGeneratePartial { + option (google.api.default_host) = "localhost:7469"; + + rpc EchoShouldInclude(EchoRequest) returns (EchoResponse) { + option (google.api.http) = { + post: "/v1beta1/echo:echo" + body: "*" + }; + option (google.api.method_signature) = "name"; + option (google.api.method_signature) = ""; + } + + rpc ChatShouldInclude(stream EchoRequest) returns (stream EchoResponse); + + rpc ChatAgainShouldInclude(stream EchoRequest) returns (stream EchoResponse) { + option (google.api.method_signature) = "content"; + } + + rpc AnExcludedMethod(stream EchoRequestWithFoobarbaz) returns (stream EchoResponse); + + rpc AnotherExcludedMethod(stream EchoRequest) returns (stream EchoResponse); + +} + +service EchoServiceShouldGenerateNone { + option (google.api.default_host) = "localhost:7469"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; + + rpc Echo(EchoRequest) returns (EchoResponse) { + option (google.api.method_signature) = "content"; + } + + rpc ChatAgain(stream EchoRequest) returns (stream EchoResponse) { + option (google.api.method_signature) = "content"; + } +} + +// resource name used for message EchoRequest. +message Foobar { + option (google.api.resource) = { + type: "showcase.googleapis.com/Foobar" + pattern: "projects/{project}/foobars/{foobar}" + }; + + string name = 1; + string info = 2; +} + +// resource name used only for message EchoRequestWithFoobarbaz. +// should not be generated with selective generation when +// AnExcludedMethod is not config as included. +message Foobarbaz { + option (google.api.resource) = { + type: "showcase.googleapis.com/Foobarbaz" + pattern: "projects/{project}/foobarsbaz/{foobarbaz}" + pattern: "projects/{project}/chocolate/variants/{variant}/foobars/{foobar}" + }; + + string name = 1; + string info = 2; +} + +// RPCs in inclusion list and not in the list both relies on this request message. +message EchoRequest { + string name = 5 [ + (google.api.resource_reference).type = "showcase.googleapis.com/Foobar", + (google.api.field_behavior) = REQUIRED + ]; + + string parent = 6 [ + (google.api.resource_reference).child_type = + "showcase.googleapis.com/AnythingGoes", + (google.api.field_behavior) = REQUIRED + ]; + + oneof response { + // The content to be echoed by the server. + string content = 1; + + // The error to be thrown by the server. + google.rpc.Status error = 2; + } + + Foobar foobar = 4; +} + +// This request message is used by AnExcludedMethod rpc. +// To demonstrate that if AnExcludedMethod is not included in generation, +// then the resource name Foobarbaz, which is only used by this method, +// should not be generated. +message EchoRequestWithFoobarbaz { + string name = 5 [ + (google.api.resource_reference).type = "showcase.googleapis.com/Foobarbaz", + (google.api.field_behavior) = REQUIRED + ]; + + string parent = 6 [ + (google.api.resource_reference).child_type = + "showcase.googleapis.com/AnythingGoes", + (google.api.field_behavior) = REQUIRED + ]; + + oneof response { + // The content to be echoed by the server. + string content = 1; + + // The error to be thrown by the server. + google.rpc.Status error = 2; + } + + Foobarbaz foobar = 4; +} + +message EchoResponse { + // The content specified in the request. + string content = 1; +} diff --git a/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml b/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml new file mode 100644 index 0000000000..021e257c50 --- /dev/null +++ b/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml @@ -0,0 +1,23 @@ +type: google.api.Service +config_version: 3 +name: selective_api_generation_testing.googleapis.com +title: Selective Generation Testing API + +publishing: + # ... + library_settings: + - version: google.selective.generate.v1beta1 + java_settings: + common: + selective_gapic_generation: + methods: + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.EchoShouldInclude + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.ChatShouldInclude + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.ChatAgainShouldInclude + reference_docs_uri: www.abc.net + destinations: + - PACKAGE_MANAGER + python_settings: + common: + destinations: + - PACKAGE_MANAGER diff --git a/gax-java/README.md b/gax-java/README.md index 8734bfd0fe..d219315e5e 100644 --- a/gax-java/README.md +++ b/gax-java/README.md @@ -34,27 +34,27 @@ If you are using Maven, add this to your pom.xml file com.google.api gax - 2.57.0 + 2.58.0 com.google.api gax-grpc - 2.57.0 + 2.58.0 ``` If you are using Gradle, add this to your dependencies ```Groovy -compile 'com.google.api:gax:2.57.0', - 'com.google.api:gax-grpc:2.57.0' +compile 'com.google.api:gax:2.58.0', + 'com.google.api:gax-grpc:2.58.0' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.api" % "gax" % "2.57.0" -libraryDependencies += "com.google.api" % "gax-grpc" % "2.57.0" +libraryDependencies += "com.google.api" % "gax" % "2.58.0" +libraryDependencies += "com.google.api" % "gax-grpc" % "2.58.0" ``` [//]: # ({x-version-update-end}) diff --git a/gax-java/dependencies.properties b/gax-java/dependencies.properties index 144965463b..77aeab9fe2 100644 --- a/gax-java/dependencies.properties +++ b/gax-java/dependencies.properties @@ -8,16 +8,16 @@ # Versions of oneself # {x-version-update-start:gax:current} -version.gax=2.57.0 +version.gax=2.58.0 # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_grpc=2.57.0 +version.gax_grpc=2.58.0 # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_bom=2.57.0 +version.gax_bom=2.58.0 # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_httpjson=2.57.0 +version.gax_httpjson=2.58.0 # {x-version-update-end} # Versions for dependencies which actual artifacts differ between Bazel and Gradle. @@ -28,7 +28,7 @@ version.gax_httpjson=2.57.0 version.com_google_protobuf=3.25.5 version.google_java_format=1.15.0 -version.io_grpc=1.67.1 +version.io_grpc=1.68.1 # Maven artifacts. # Note, the actual name of each property matters (bazel build scripts depend on it). @@ -37,8 +37,8 @@ version.io_grpc=1.67.1 # 2) Replace all characters which are neither alphabetic nor digits with the underscore ('_') character maven.com_google_api_grpc_proto_google_common_protos=com.google.api.grpc:proto-google-common-protos:2.46.0 maven.com_google_api_grpc_grpc_google_common_protos=com.google.api.grpc:grpc-google-common-protos:2.46.0 -maven.com_google_auth_google_auth_library_oauth2_http=com.google.auth:google-auth-library-oauth2-http:1.29.0 -maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.29.0 +maven.com_google_auth_google_auth_library_oauth2_http=com.google.auth:google-auth-library-oauth2-http:1.30.0 +maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.30.0 maven.io_opentelemetry_opentelemetry_api=io.opentelemetry:opentelemetry-api:1.42.1 maven.io_opencensus_opencensus_api=io.opencensus:opencensus-api:0.31.1 maven.io_opencensus_opencensus_contrib_grpc_metrics=io.opencensus:opencensus-contrib-grpc-metrics:0.31.1 diff --git a/gax-java/gax-bom/pom.xml b/gax-java/gax-bom/pom.xml index 08c2149ac8..011e42b586 100644 --- a/gax-java/gax-bom/pom.xml +++ b/gax-java/gax-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.api gax-bom - 2.57.0 + 2.58.0 pom GAX (Google Api eXtensions) for Java (BOM) Google Api eXtensions for Java (BOM) @@ -11,7 +11,7 @@ com.google.cloud google-cloud-shared-config - 1.11.3 + 1.12.0 @@ -43,55 +43,55 @@ com.google.api gax - 2.57.0 + 2.58.0 com.google.api gax - 2.57.0 + 2.58.0 test-jar testlib com.google.api gax - 2.57.0 + 2.58.0 testlib com.google.api gax-grpc - 2.57.0 + 2.58.0 com.google.api gax-grpc - 2.57.0 + 2.58.0 test-jar testlib com.google.api gax-grpc - 2.57.0 + 2.58.0 testlib com.google.api gax-httpjson - 2.57.0 + 2.58.0 com.google.api gax-httpjson - 2.57.0 + 2.58.0 test-jar testlib com.google.api gax-httpjson - 2.57.0 + 2.58.0 testlib diff --git a/gax-java/gax-grpc/BUILD.bazel b/gax-java/gax-grpc/BUILD.bazel index be224ff3f8..99e4aba500 100644 --- a/gax-java/gax-grpc/BUILD.bazel +++ b/gax-java/gax-grpc/BUILD.bazel @@ -28,6 +28,7 @@ _COMPILE_DEPS = [ "@io_grpc_grpc_netty_shaded//jar", "@io_grpc_grpc_grpclb//jar", "@io_grpc_grpc_java//alts:alts", + "@io_grpc_grpc_java//s2a:s2av2_credentials", "@io_netty_netty_tcnative_boringssl_static//jar", "@javax_annotation_javax_annotation_api//jar", "//gax:gax", diff --git a/gax-java/gax-grpc/pom.xml b/gax-java/gax-grpc/pom.xml index 6992eea261..927415b594 100644 --- a/gax-java/gax-grpc/pom.xml +++ b/gax-java/gax-grpc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-grpc - 2.57.0 + 2.58.0 jar GAX (Google Api eXtensions) for Java (gRPC) Google Api eXtensions for Java (gRPC) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.58.0 @@ -63,6 +63,10 @@ io.grpc grpc-protobuf + + io.grpc + grpc-s2a + io.grpc grpc-stub diff --git a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java index ae4d7f9e51..8cad9f0383 100644 --- a/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java +++ b/gax-java/gax-grpc/src/main/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProvider.java @@ -46,19 +46,24 @@ import com.google.auth.ApiKeyCredentials; import com.google.auth.Credentials; import com.google.auth.oauth2.ComputeEngineCredentials; +import com.google.auth.oauth2.SecureSessionAgent; +import com.google.auth.oauth2.SecureSessionAgentConfig; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.Files; import io.grpc.CallCredentials; import io.grpc.ChannelCredentials; import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.TlsChannelCredentials; import io.grpc.alts.GoogleDefaultChannelCredentials; import io.grpc.auth.MoreCallCredentials; +import io.grpc.s2a.S2AChannelCredentials; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -99,6 +104,15 @@ public final class InstantiatingGrpcChannelProvider implements TransportChannelP @VisibleForTesting static final String DIRECT_PATH_ENV_ENABLE_XDS = "GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS"; + // The public portion of the mTLS MDS root certificate is stored for performing + // cert verification when establishing an mTLS connection with the MDS. See + // https://cloud.google.com/compute/docs/metadata/overview#https-mds-root-certs + private static final String MTLS_MDS_ROOT_PATH = "/run/google-mds-mtls/root.crt"; + // The mTLS MDS credentials are formatted as the concatenation of a PEM-encoded certificate chain + // followed by a PEM-encoded private key. See + // https://cloud.google.com/compute/docs/metadata/overview#https-mds-client-certs + private static final String MTLS_MDS_CERT_CHAIN_AND_KEY_PATH = "/run/google-mds-mtls/client.key"; + static final long DIRECT_PATH_KEEP_ALIVE_TIME_SECONDS = 3600; static final long DIRECT_PATH_KEEP_ALIVE_TIMEOUT_SECONDS = 20; static final String GCE_PRODUCTION_NAME_PRIOR_2016 = "Google"; @@ -107,6 +121,7 @@ public final class InstantiatingGrpcChannelProvider implements TransportChannelP private final int processorCount; private final Executor executor; private final HeaderProvider headerProvider; + private final boolean useS2A; private final String endpoint; // TODO: remove. envProvider currently provides DirectPath environment variable, and is only used // during initial rollout for DirectPath. This provider will be removed once the DirectPath @@ -126,6 +141,7 @@ public final class InstantiatingGrpcChannelProvider implements TransportChannelP @Nullable private final Boolean allowNonDefaultServiceAccount; @VisibleForTesting final ImmutableMap directPathServiceConfig; @Nullable private final MtlsProvider mtlsProvider; + @Nullable private final SecureSessionAgent s2aConfigProvider; @VisibleForTesting final Map headersWithDuplicatesRemoved = new HashMap<>(); @Nullable @@ -136,7 +152,9 @@ private InstantiatingGrpcChannelProvider(Builder builder) { this.executor = builder.executor; this.headerProvider = builder.headerProvider; this.endpoint = builder.endpoint; + this.useS2A = builder.useS2A; this.mtlsProvider = builder.mtlsProvider; + this.s2aConfigProvider = builder.s2aConfigProvider; this.envProvider = builder.envProvider; this.interceptorProvider = builder.interceptorProvider; this.maxInboundMessageSize = builder.maxInboundMessageSize; @@ -225,6 +243,17 @@ public TransportChannelProvider withEndpoint(String endpoint) { return toBuilder().setEndpoint(endpoint).build(); } + /** + * Specify whether or not to use S2A. + * + * @param useS2A + * @return A new {@link InstantiatingGrpcChannelProvider} with useS2A set. + */ + @Override + public TransportChannelProvider withUseS2A(boolean useS2A) { + return toBuilder().setUseS2A(useS2A).build(); + } + /** @deprecated Please modify pool settings via {@link #toBuilder()} */ @Deprecated @Override @@ -410,6 +439,101 @@ ChannelCredentials createMtlsChannelCredentials() throws IOException, GeneralSec return null; } + /** + * This method creates {@link TlsChannelCredentials} to be used by the client to establish an mTLS + * connection to S2A. Returns null if any of {@param trustBundle}, {@param privateKey} or {@param + * certChain} are missing. + * + * @param trustBundle the trust bundle to be used to establish the client -> S2A mTLS connection + * @param privateKey the client's private key to be used to establish the client -> S2A mtls + * connection + * @param certChain the client's cert chain to be used to establish the client -> S2A mtls + * connection + * @return {@link ChannelCredentials} to use to create an mtls connection between client and S2A + * @throws IOException on error + */ + @VisibleForTesting + ChannelCredentials createMtlsToS2AChannelCredentials( + File trustBundle, File privateKey, File certChain) throws IOException { + if (trustBundle == null || privateKey == null || certChain == null) { + return null; + } + return TlsChannelCredentials.newBuilder() + .keyManager(privateKey, certChain) + .trustManager(trustBundle) + .build(); + } + + /** + * This method creates {@link ChannelCredentials} to be used by client to establish a plaintext + * connection to S2A. if {@param plaintextAddress} is not present, returns null. + * + * @param plaintextAddress the address to reach S2A which accepts plaintext connections + * @return {@link ChannelCredentials} to use to create a plaintext connection between client and + * S2A + */ + ChannelCredentials createPlaintextToS2AChannelCredentials(String plaintextAddress) { + if (Strings.isNullOrEmpty(plaintextAddress)) { + return null; + } + return S2AChannelCredentials.newBuilder(plaintextAddress, InsecureChannelCredentials.create()) + .build(); + } + + /** + * This method creates gRPC {@link ChannelCredentials} configured to use S2A to estbalish a mTLS + * connection. First, the address of S2A is discovered by using the {@link S2A} utility to learn + * the {@code mtlsAddress} to reach S2A and the {@code plaintextAddress} to reach S2A. Prefer to + * use the {@code mtlsAddress} address to reach S2A if it is non-empty and the MTLS-MDS + * credentials can successfully be discovered and used to create {@link TlsChannelCredentials}. If + * there is any failure using mTLS-to-S2A, fallback to using a plaintext connection to S2A using + * the {@code plaintextAddress}. If {@code plaintextAddress} is not available, this function + * returns null; in this case S2A will not be used, and a TLS connection to the service will be + * established. + * + * @return {@link ChannelCredentials} configured to use S2A to create mTLS connection to + * mtlsEndpoint. + */ + ChannelCredentials createS2ASecuredChannelCredentials() { + SecureSessionAgentConfig config = s2aConfigProvider.getConfig(); + String plaintextAddress = config.getPlaintextAddress(); + String mtlsAddress = config.getMtlsAddress(); + if (Strings.isNullOrEmpty(mtlsAddress)) { + // Fallback to plaintext connection to S2A. + LOG.log( + Level.INFO, + "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not return a mtls address to reach S2A."); + return createPlaintextToS2AChannelCredentials(plaintextAddress); + } + // Currently, MTLS to MDS is only available on GCE. See: + // https://cloud.google.com/compute/docs/metadata/overview#https-mds + // Try to load MTLS-MDS creds. + File rootFile = new File(MTLS_MDS_ROOT_PATH); + File certKeyFile = new File(MTLS_MDS_CERT_CHAIN_AND_KEY_PATH); + if (rootFile.isFile() && certKeyFile.isFile()) { + // Try to connect to S2A using mTLS. + ChannelCredentials mtlsToS2AChannelCredentials = null; + try { + mtlsToS2AChannelCredentials = + createMtlsToS2AChannelCredentials(rootFile, certKeyFile, certKeyFile); + } catch (IOException ignore) { + // Fallback to plaintext-to-S2A connection on error. + LOG.log( + Level.WARNING, + "Cannot establish an mTLS connection to S2A due to error creating MTLS to MDS TlsChannelCredentials credentials, falling back to plaintext connection to S2A: " + + ignore.getMessage()); + return createPlaintextToS2AChannelCredentials(plaintextAddress); + } + return S2AChannelCredentials.newBuilder(mtlsAddress, mtlsToS2AChannelCredentials).build(); + } else { + // Fallback to plaintext-to-S2A connection if MTLS-MDS creds do not exist. + LOG.log( + Level.INFO, + "Cannot establish an mTLS connection to S2A because MTLS to MDS credentials do not exist on filesystem, falling back to plaintext connection to S2A"); + return createPlaintextToS2AChannelCredentials(plaintextAddress); + } + } + private ManagedChannel createSingleChannel() throws IOException { GrpcHeaderInterceptor headerInterceptor = new GrpcHeaderInterceptor(headersWithDuplicatesRemoved); @@ -447,6 +571,7 @@ private ManagedChannel createSingleChannel() throws IOException { builder.keepAliveTime(DIRECT_PATH_KEEP_ALIVE_TIME_SECONDS, TimeUnit.SECONDS); builder.keepAliveTimeout(DIRECT_PATH_KEEP_ALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS); } else { + // Try and create credentials via DCA. See https://google.aip.dev/auth/4114. ChannelCredentials channelCredentials; try { channelCredentials = createMtlsChannelCredentials(); @@ -454,9 +579,23 @@ private ManagedChannel createSingleChannel() throws IOException { throw new IOException(e); } if (channelCredentials != null) { + // Create the channel using channel credentials created via DCA. builder = Grpc.newChannelBuilder(endpoint, channelCredentials); } else { - builder = ManagedChannelBuilder.forAddress(serviceAddress, port); + // Could not create channel credentials via DCA. In accordance with + // https://google.aip.dev/auth/4115, if credentials not available through + // DCA, try mTLS with credentials held by the S2A (Secure Session Agent). + if (useS2A) { + channelCredentials = createS2ASecuredChannelCredentials(); + } + if (channelCredentials != null) { + // Create the channel using S2A-secured channel credentials. + // {@code endpoint} is set to mtlsEndpoint in {@link EndpointContext} when useS2A is true. + builder = Grpc.newChannelBuilder(endpoint, channelCredentials); + } else { + // Use default if we cannot initialize channel credentials via DCA or S2A. + builder = ManagedChannelBuilder.forAddress(serviceAddress, port); + } } } // google-c2p resolver requires service config lookup @@ -604,7 +743,9 @@ public static final class Builder { private Executor executor; private HeaderProvider headerProvider; private String endpoint; + private boolean useS2A; private EnvironmentProvider envProvider; + private SecureSessionAgent s2aConfigProvider = SecureSessionAgent.create(); private MtlsProvider mtlsProvider = new MtlsProvider(); @Nullable private GrpcInterceptorProvider interceptorProvider; @Nullable private Integer maxInboundMessageSize; @@ -632,6 +773,7 @@ private Builder(InstantiatingGrpcChannelProvider provider) { this.executor = provider.executor; this.headerProvider = provider.headerProvider; this.endpoint = provider.endpoint; + this.useS2A = provider.useS2A; this.envProvider = provider.envProvider; this.interceptorProvider = provider.interceptorProvider; this.maxInboundMessageSize = provider.maxInboundMessageSize; @@ -648,6 +790,7 @@ private Builder(InstantiatingGrpcChannelProvider provider) { this.allowNonDefaultServiceAccount = provider.allowNonDefaultServiceAccount; this.directPathServiceConfig = provider.directPathServiceConfig; this.mtlsProvider = provider.mtlsProvider; + this.s2aConfigProvider = provider.s2aConfigProvider; } /** @@ -700,12 +843,23 @@ public Builder setEndpoint(String endpoint) { return this; } + Builder setUseS2A(boolean useS2A) { + this.useS2A = useS2A; + return this; + } + @VisibleForTesting Builder setMtlsProvider(MtlsProvider mtlsProvider) { this.mtlsProvider = mtlsProvider; return this; } + @VisibleForTesting + Builder setS2AConfigProvider(SecureSessionAgent s2aConfigProvider) { + this.s2aConfigProvider = s2aConfigProvider; + return this; + } + /** * Sets the GrpcInterceptorProvider for this TransportChannelProvider. * diff --git a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcLongRunningTest.java b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcLongRunningTest.java index 241f90b08a..ac88e4acec 100644 --- a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcLongRunningTest.java +++ b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/GrpcLongRunningTest.java @@ -101,6 +101,8 @@ void setUp() throws IOException { TransportChannel transportChannel = GrpcTransportChannel.newBuilder().setManagedChannel(channel).build(); when(operationsChannelProvider.getTransportChannel()).thenReturn(transportChannel); + when(operationsChannelProvider.withUseS2A(Mockito.any(boolean.class))) + .thenReturn(operationsChannelProvider); clock = new FakeApiClock(0L); executor = RecordingScheduler.create(clock); diff --git a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java index a58f9b8173..049c34dd96 100644 --- a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java +++ b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/InstantiatingGrpcChannelProviderTest.java @@ -51,12 +51,16 @@ import com.google.auth.http.AuthHttpConstants; import com.google.auth.oauth2.CloudShellCredentials; import com.google.auth.oauth2.ComputeEngineCredentials; +import com.google.auth.oauth2.SecureSessionAgent; +import com.google.auth.oauth2.SecureSessionAgentConfig; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.truth.Truth; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.TlsChannelCredentials; import io.grpc.alts.ComputeEngineChannelBuilder; +import java.io.File; import java.io.IOException; import java.security.GeneralSecurityException; import java.time.Duration; @@ -980,6 +984,120 @@ private FixedHeaderProvider getHeaderProviderWithApiKeyHeader() { return FixedHeaderProvider.create(header); } + @Test + void createPlaintextToS2AChannelCredentials_emptyPlaintextAddress_returnsNull() { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + assertThat(provider.createPlaintextToS2AChannelCredentials("")).isNull(); + } + + @Test + void createPlaintextToS2AChannelCredentials_success() { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + assertThat(provider.createPlaintextToS2AChannelCredentials("localhost:8080")).isNotNull(); + } + + @Test + void createMtlsToS2AChannelCredentials_missingAllFiles_throws() throws IOException { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + assertThat(provider.createMtlsToS2AChannelCredentials(null, null, null)).isNull(); + } + + @Test + void createMtlsToS2AChannelCredentials_missingRootFile_throws() throws IOException { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + File privateKey = new File("src/test/resources/client_key.pem"); + File certChain = new File("src/test/resources/client_cert.pem"); + assertThat(provider.createMtlsToS2AChannelCredentials(null, privateKey, certChain)).isNull(); + } + + @Test + void createMtlsToS2AChannelCredentials_missingKeyFile_throws() throws IOException { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + File trustBundle = new File("src/test/resources/root_cert.pem"); + File certChain = new File("src/test/resources/client_cert.pem"); + assertThat(provider.createMtlsToS2AChannelCredentials(trustBundle, null, certChain)).isNull(); + } + + @Test + void createMtlsToS2AChannelCredentials_missingCertChainFile_throws() throws IOException { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + File trustBundle = new File("src/test/resources/root_cert.pem"); + File privateKey = new File("src/test/resources/client_key.pem"); + assertThat(provider.createMtlsToS2AChannelCredentials(trustBundle, privateKey, null)).isNull(); + } + + @Test + void createMtlsToS2AChannelCredentials_success() throws IOException { + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder().build(); + File trustBundle = new File("src/test/resources/root_cert.pem"); + File privateKey = new File("src/test/resources/client_key.pem"); + File certChain = new File("src/test/resources/client_cert.pem"); + assertEquals( + provider.createMtlsToS2AChannelCredentials(trustBundle, privateKey, certChain).getClass(), + TlsChannelCredentials.class); + } + + @Test + void createS2ASecuredChannelCredentials_bothS2AAddressesNull_returnsNull() { + SecureSessionAgent s2aConfigProvider = Mockito.mock(SecureSessionAgent.class); + SecureSessionAgentConfig config = SecureSessionAgentConfig.createBuilder().build(); + Mockito.when(s2aConfigProvider.getConfig()).thenReturn(config); + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setS2AConfigProvider(s2aConfigProvider) + .build(); + assertThat(provider.createS2ASecuredChannelCredentials()).isNull(); + } + + @Test + void + createS2ASecuredChannelCredentials_mtlsS2AAddressNull_returnsPlaintextToS2AS2AChannelCredentials() { + SecureSessionAgent s2aConfigProvider = Mockito.mock(SecureSessionAgent.class); + SecureSessionAgentConfig config = + SecureSessionAgentConfig.createBuilder().setPlaintextAddress("localhost:8080").build(); + Mockito.when(s2aConfigProvider.getConfig()).thenReturn(config); + FakeLogHandler logHandler = new FakeLogHandler(); + InstantiatingGrpcChannelProvider.LOG.addHandler(logHandler); + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setS2AConfigProvider(s2aConfigProvider) + .build(); + assertThat(provider.createS2ASecuredChannelCredentials()).isNotNull(); + assertThat(logHandler.getAllMessages()) + .contains( + "Cannot establish an mTLS connection to S2A because autoconfig endpoint did not return a mtls address to reach S2A."); + InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); + } + + @Test + void createS2ASecuredChannelCredentials_returnsPlaintextToS2AS2AChannelCredentials() { + SecureSessionAgent s2aConfigProvider = Mockito.mock(SecureSessionAgent.class); + SecureSessionAgentConfig config = + SecureSessionAgentConfig.createBuilder() + .setMtlsAddress("localhost:8080") + .setPlaintextAddress("localhost:8080") + .build(); + Mockito.when(s2aConfigProvider.getConfig()).thenReturn(config); + FakeLogHandler logHandler = new FakeLogHandler(); + InstantiatingGrpcChannelProvider.LOG.addHandler(logHandler); + InstantiatingGrpcChannelProvider provider = + InstantiatingGrpcChannelProvider.newBuilder() + .setS2AConfigProvider(s2aConfigProvider) + .build(); + assertThat(provider.createS2ASecuredChannelCredentials()).isNotNull(); + assertThat(logHandler.getAllMessages()) + .contains( + "Cannot establish an mTLS connection to S2A because MTLS to MDS credentials do not exist on filesystem, falling back to plaintext connection to S2A"); + InstantiatingGrpcChannelProvider.LOG.removeHandler(logHandler); + } + private static class FakeLogHandler extends Handler { List records = new ArrayList<>(); diff --git a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/testing/LocalChannelProvider.java b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/testing/LocalChannelProvider.java index 5e538a06c2..856a2850bb 100644 --- a/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/testing/LocalChannelProvider.java +++ b/gax-java/gax-grpc/src/test/java/com/google/api/gax/grpc/testing/LocalChannelProvider.java @@ -106,6 +106,12 @@ public TransportChannelProvider withEndpoint(String endpoint) { throw new UnsupportedOperationException("LocalChannelProvider doesn't need an endpoint"); } + @Override + public TransportChannelProvider withUseS2A(boolean useS2A) { + // Overriden for technical reasons. This method is a no-op for LocalChannelProvider. + return this; + } + @Override @BetaApi("The surface for customizing pool size is not stable yet and may change in the future.") public boolean acceptsPoolSize() { diff --git a/gax-java/gax-grpc/src/test/resources/README.md b/gax-java/gax-grpc/src/test/resources/README.md new file mode 100644 index 0000000000..a9a9b0efe9 --- /dev/null +++ b/gax-java/gax-grpc/src/test/resources/README.md @@ -0,0 +1,29 @@ +# Regenerate certificates and keys for testing mTLS-S2A +Below are the commands which can be used to regenerate the certs used in tests. This is the same process +used to generate test certs for S2A client in grpc-java: https://github.com/grpc/grpc-java/blob/master/s2a/src/test/resources/README.md + +Create root CA + +``` +openssl req -x509 -sha256 -days 7305 -newkey rsa:2048 -keyout root_key.pem -out +root_cert.pem +``` + +Generate private key + +``` +openssl genrsa -out client_key.pem 2048 +``` + +Generate CSR (set Common Name to localhost, leave all +other fields blank) + +``` +openssl req -key client_key.pem -new -out client.csr -config config.cnf +``` + +Sign CSR for client + +``` +openssl x509 -req -CA root_cert.pem -CAkey root_key.pem -in client.csr -out client_cert.pem -days 7305 +``` diff --git a/gax-java/gax-grpc/src/test/resources/client_cert.pem b/gax-java/gax-grpc/src/test/resources/client_cert.pem new file mode 100644 index 0000000000..837f8bb501 --- /dev/null +++ b/gax-java/gax-grpc/src/test/resources/client_cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPTCCAiWgAwIBAgIUaarddwSWeE4jDC9kwxEr446ehqUwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTI0MTAwMTIxNTk1NFoXDTQ0MTAwMTIxNTk1NFowFDESMBAGA1UEAwwJbG9jYWxo +b3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxlNsldt7yAU4KRuS +2D2/FjNIE1US5olBm4HteTr++41WaELZJqNLRPPp052jEQU3aKSYNGZvUUO6buu7 +eFpz2SBNUVMyvmzzocjVAyyf4NQvDazYHWOb+/YCeUppTRWriz4V5sn47qJTQ8cd +CGrTFeLHxUjx4nh/OiqVXP/KnF3EqPEuqph0ky7+GirnJgPRe+C5ERuGkJye8dmP +yWGA2lSS6MeDe7JZTAMi08bAn7BuNpeBkOzz1msGGI9PnUanUs7GOPWTDdcQAVY8 +KMvHCuGaNMGpb4rOR2mm8LlbAbpTPz8Pkw4QtMCLkgsrz2CzXpVwnLsU7nDXJAIO +B155lQIDAQABo0IwQDAdBgNVHQ4EFgQUSZEyIHLzkIw7AwkBaUjYfIrGVR4wHwYD +VR0jBBgwFoAUcq3dtxAVA410YWyM0B4e+4umbiwwDQYJKoZIhvcNAQELBQADggEB +AAz0bZ4ayrZLhA45xn0yvdpdqiCtiWikCRtxgE7VXHg/ziZJVMpBpAhbIGO5tIyd +lttnRXHwz5DUwKiba4/bCEFe229BshQEql5qaqcbGbFfSly11WeqqnwR1N7c8Gpv +pD9sVrx22seN0rTUk87MY/S7mzCxHqAx35zm/LTW3pWcgCTMKFHy4Gt4mpTnXkNA +WkhP2OhW5RLiu6Whi0BEdb2TGG1+ctamgijKXb+gJeef5ehlHXG8eU862KF5UlEA +NeQKBm/PpQxOMe0NdpatjN8QRoczku0Itiodng+OZ1o+2iSNG988uFRb3CUSnjtE +R/HL6ULAFzo59EpIYxruU/w= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/gax-java/gax-grpc/src/test/resources/client_key.pem b/gax-java/gax-grpc/src/test/resources/client_key.pem new file mode 100644 index 0000000000..38b93eb65c --- /dev/null +++ b/gax-java/gax-grpc/src/test/resources/client_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGU2yV23vIBTgp +G5LYPb8WM0gTVRLmiUGbge15Ov77jVZoQtkmo0tE8+nTnaMRBTdopJg0Zm9RQ7pu +67t4WnPZIE1RUzK+bPOhyNUDLJ/g1C8NrNgdY5v79gJ5SmlNFauLPhXmyfjuolND +xx0IatMV4sfFSPHieH86KpVc/8qcXcSo8S6qmHSTLv4aKucmA9F74LkRG4aQnJ7x +2Y/JYYDaVJLox4N7sllMAyLTxsCfsG42l4GQ7PPWawYYj0+dRqdSzsY49ZMN1xAB +Vjwoy8cK4Zo0walvis5HaabwuVsBulM/Pw+TDhC0wIuSCyvPYLNelXCcuxTucNck +Ag4HXnmVAgMBAAECggEAKuW9jXaBgiS63o1jyFkmvWcPNntG0M2sfrXuRzQfFgse +vwOCk8xrSflWQNsOe+58ayp6746ekl3LdBWSIbiy6SqG/sm3pp/LXNmjVYHv/QH4 +QYV643R5t1ihdVnGiBFhXwdpVleme/tpdjYZzgnJKak5W69o/nrgzhSK5ShAy2xM +j0XXbgdqG+4JxPb5BZmjHHfXAXUfgSORMdfArkbgFBRc9wL/6JVTXjeAMy5WX9qe +5UQsSOYkwc9P2snifC/jdIhjHQOkkx59O0FgukJEFZPoagVG1duWQbnNDr7QVHCJ +jV6dg9tIT4SXD3uPSPbgNGlRUseIakCzrhHARJuA2wKBgQD/h8zoh0KaqKyViCYw +XKOFpm1pAFnp2GiDOblxNubNFAXEWnC+FlkvO/z1s0zVuYELUqfxcYMSXJFEVelK +rfjZtoC5oxqWGqLo9iCj7pa8t+ipulYcLt2SWc7eZPD4T4lzeEf1Qz77aKcz34sa +dv9lzQkDvhR/Mv1VeEGFHiq2VwKBgQDGsLcTGH5Yxs//LRSY8TigBkQEDrH5NvXu +2jtAzZhy1Yhsoa5eiZkhnnzM6+n05ovfZLcy6s7dnwP1Y+C79vs+DKMBsodtDG5z +YpsB0VrXYa6P6pCqkcz0Bz9xdo5sOhAK3AKnX6jd29XBDdeYsw/lxHLG24wProTD +cCYFqtaj8wKBgQCaqKT68DL9zK14a8lBaDCIyexaqx3AjXzkP+Hfhi03XrEG4P5v +7rLYBeTbCUSt7vMN2V9QoTWFvYUm6SCkVJvTmcRblz6WL1T+z0l+LwAJBP7LC77m +m+77j2PH8yxt/iXhP6G97o+GNxdMLDbTM8bs5KZaH4fkXQY73uc5HMMZTQKBgEZS +7blYhf+t/ph2wD+RwVUCYrh86wkmJs2veCFro3WhlnO8lhbn5Mc9bTaqmVgQ8ZjT +8POYoDdYvPHxs+1TcYF4v4kuQziZmc5FLE/sZZauADb38tQsXrpQhmgGakpsEpmF +XXsYJJDB6lo2KATn+8x7R5SSyHQUdPEnlI2U9ft5AoGBAJw0NJiM1EzRS8xq0DmO +AvQaPjo01o2hH6wghws8gDQwrj0eHraHgVi7zo0VkaHJbO7ahKPudset3N7owJhA +CUAPPRtv5wn0amAyNz77f1dz4Gys3AkcchflqhbEaQpzKYx4kX0adclur4WJ/DVm +P7DI977SHCVB4FVMbXMEkBjN +-----END PRIVATE KEY----- \ No newline at end of file diff --git a/gax-java/gax-grpc/src/test/resources/root_cert.pem b/gax-java/gax-grpc/src/test/resources/root_cert.pem new file mode 100644 index 0000000000..ccd0a46bc2 --- /dev/null +++ b/gax-java/gax-grpc/src/test/resources/root_cert.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUWemeXZdfqcqkP8/Eyj74oTJtoNQwDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTI0MTAwMTIxNTkxMVoXDTQ0MTAwMTIxNTkxMVowWTELMAkGA1UEBhMCQVUxEzAR +BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 +IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAt3A04hy5lljv86Nu0LLQZ2hA+fcImHjt1p1Mxgcta/5oxfVLcerE +ZH+DAQLDtWzp9Up/vI57MM419GIL8Iszk7hnZRS/HWJ+2jewZJtz4i/g15dLr6+1 +uabMdPOWos60BwcLMxKEe6lJO1mV4z9d4NH4mAuMIHyM+ty0Klp9MfeDJtYEh0+z +AxJUHCixDTsnKJro7My7A3ZT7bvaMfXxS7XN6qlRgBfiCmXo/GKTFfmfBW/EZGkG +XOCxE2D79wYNhC41Q/ix0kwjEeOj2vgGFoiyblSdHdzvRXzsoQTEiZSM8lJDR2IT +ZbpgbBlknMU6efNWlS8P5damB9ZWXg3x4wIDAQABo1MwUTAdBgNVHQ4EFgQUcq3d +txAVA410YWyM0B4e+4umbiwwHwYDVR0jBBgwFoAUcq3dtxAVA410YWyM0B4e+4um +biwwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEApZvaI9y7vjX/ +RRdvwf2Db9KlTE9nuVQ3AsrmG9Ml0p2X6U5aTetxdYBo2PuaaYHheF03JOH8zjpL +UfFzvbi52DPbfFAaDw/6NIAenXlg492leNvUFNjGGRyJO9R5/aDfv40/fT3Em5G5 +DnR8SeGQ9tI1t6xBBT+d+/MilSiEKVu8IIF/p0SwvEyR4pKo6wFVZR0ZiIj2v/FZ +P5Qk0Xhb+slpmaR3Wtx/mPl9Wb3kpPD4CAwhWDqFkKJql9/n9FvMjdwlCQKQGB26 +ZDXY3C0UTdktK5biNWRgAUVJEWBX6Q2amrxQHIn2d9RJ8uxCME/KBAntK+VxZE78 +w0JOvQ4Dpw== +-----END CERTIFICATE----- \ No newline at end of file diff --git a/gax-java/gax-httpjson/pom.xml b/gax-java/gax-httpjson/pom.xml index 6ab196acce..7d91f4771c 100644 --- a/gax-java/gax-httpjson/pom.xml +++ b/gax-java/gax-httpjson/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-httpjson - 2.57.0 + 2.58.0 jar GAX (Google Api eXtensions) for Java (HTTP JSON) Google Api eXtensions for Java (HTTP JSON) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.58.0 diff --git a/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallableFactory.java b/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallableFactory.java index 33e2ff886e..2a160d5d0a 100644 --- a/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallableFactory.java +++ b/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/HttpJsonCallableFactory.java @@ -84,6 +84,11 @@ static UnaryCallable createUnaryCalla callable = Callables.retrying( callable, callSettings, clientContext, httpJsonCallSettings.getRequestMutator()); + callable = + new TracedUnaryCallable<>( + callable, + clientContext.getTracerFactory(), + getSpanName(httpJsonCallSettings.getMethodDescriptor())); return callable.withDefaultCallContext(clientContext.getDefaultCallContext()); } @@ -136,12 +141,6 @@ public static UnaryCallable createUna UnaryCallable innerCallable = createDirectUnaryCallable(httpJsonCallSettings); - innerCallable = - new TracedUnaryCallable<>( - innerCallable, - clientContext.getTracerFactory(), - getSpanName(httpJsonCallSettings.getMethodDescriptor())); - return createUnaryCallable(innerCallable, callSettings, httpJsonCallSettings, clientContext); } diff --git a/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java b/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java index f92bdf299c..170b955c2a 100644 --- a/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java +++ b/gax-java/gax-httpjson/src/main/java/com/google/api/gax/httpjson/InstantiatingHttpJsonChannelProvider.java @@ -124,6 +124,11 @@ public TransportChannelProvider withEndpoint(String endpoint) { return toBuilder().setEndpoint(endpoint).build(); } + @Override + public TransportChannelProvider withUseS2A(boolean useS2A) { + return this; + } + /** @deprecated REST transport channel doesn't support channel pooling */ @Deprecated @Override diff --git a/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RetryingTest.java b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RetryingTest.java index 892fc9b8f8..63f826bdbe 100644 --- a/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RetryingTest.java +++ b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/RetryingTest.java @@ -43,6 +43,7 @@ import com.google.api.core.ApiFutures; import com.google.api.gax.core.FakeApiClock; import com.google.api.gax.core.RecordingScheduler; +import com.google.api.gax.httpjson.testing.TestApiTracerFactory; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; @@ -80,6 +81,7 @@ class RetryingTest { private final Integer initialRequest = 1; private final Integer modifiedRequest = 0; + private TestApiTracerFactory tracerFactory; private final HttpJsonCallSettings httpJsonCallSettings = HttpJsonCallSettings.newBuilder() @@ -115,8 +117,11 @@ class RetryingTest { void resetClock() { fakeClock = new FakeApiClock(System.nanoTime()); executor = RecordingScheduler.create(fakeClock); + tracerFactory = new TestApiTracerFactory(); clientContext = ClientContext.newBuilder() + // we use a custom tracer to confirm whether the retrials are being recorded. + .setTracerFactory(tracerFactory) .setExecutor(executor) .setClock(fakeClock) .setDefaultCallContext(HttpJsonCallContext.createDefault()) @@ -130,6 +135,7 @@ void teardown() { @Test void retry() { + // set a retriable that will fail 3 times before returning "2" ImmutableSet retryable = ImmutableSet.of(Code.UNAVAILABLE); Mockito.when(callInt.futureCall((Integer) any(), (ApiCallContext) any())) .thenReturn(ApiFutures.immediateFailedFuture(HTTP_SERVICE_UNAVAILABLE_EXCEPTION)) @@ -143,6 +149,9 @@ void retry() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); assertThat(callable.call(initialRequest)).isEqualTo(2); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(3); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(4); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); @@ -180,6 +189,9 @@ void retryTotalTimeoutExceeded() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); assertThrows(ApiException.class, () -> callable.call(initialRequest)); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(1); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(0); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); verify(callInt, atLeastOnce()).futureCall(argumentCaptor.capture(), any(ApiCallContext.class)); @@ -200,6 +212,9 @@ void retryMaxAttemptsExceeded() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); assertThrows(ApiException.class, () -> callable.call(initialRequest)); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(2); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(2); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isTrue(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); verify(callInt, atLeastOnce()).futureCall(argumentCaptor.capture(), any(ApiCallContext.class)); @@ -220,6 +235,9 @@ void retryWithinMaxAttempts() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); assertThat(callable.call(initialRequest)).isEqualTo(2); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(3); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(2); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); verify(callInt, atLeastOnce()).futureCall(argumentCaptor.capture(), any(ApiCallContext.class)); @@ -246,6 +264,9 @@ void retryOnStatusUnknown() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); assertThat(callable.call(initialRequest)).isEqualTo(2); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(4); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(3); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); verify(callInt, atLeastOnce()).futureCall(argumentCaptor.capture(), any(ApiCallContext.class)); @@ -264,6 +285,9 @@ void retryOnUnexpectedException() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); ApiException exception = assertThrows(ApiException.class, () -> callable.call(initialRequest)); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(1); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(0); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); assertThat(exception).hasCauseThat().isSameInstanceAs(throwable); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); @@ -293,6 +317,9 @@ void retryNoRecover() { HttpJsonCallableFactory.createUnaryCallable( callInt, callSettings, httpJsonCallSettings, clientContext); ApiException exception = assertThrows(ApiException.class, () -> callable.call(initialRequest)); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(1); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(0); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); assertThat(exception).isSameInstanceAs(apiException); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); @@ -319,6 +346,10 @@ void retryKeepFailing() { UncheckedExecutionException exception = assertThrows(UncheckedExecutionException.class, () -> Futures.getUnchecked(future)); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isGreaterThan(0); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()) + .isEqualTo(tracerFactory.getInstance().getAttemptsStarted().get()); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isTrue(); assertThat(exception).hasCauseThat().isInstanceOf(ApiException.class); assertThat(exception).hasCauseThat().hasMessageThat().contains("Unavailable"); // Capture the argument passed to futureCall @@ -359,6 +390,9 @@ void testKnownStatusCode() { callInt, callSettings, httpJsonCallSettings, clientContext); ApiException exception = assertThrows(FailedPreconditionException.class, () -> callable.call(initialRequest)); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(1); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(0); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); assertThat(exception.getStatusCode().getTransportCode()) .isEqualTo(HTTP_CODE_PRECONDITION_FAILED); assertThat(exception).hasMessageThat().contains("precondition failed"); @@ -383,6 +417,9 @@ void testUnknownStatusCode() { UnknownException exception = assertThrows(UnknownException.class, () -> callable.call(initialRequest)); assertThat(exception).hasMessageThat().isEqualTo("java.lang.RuntimeException: unknown"); + assertThat(tracerFactory.getInstance().getAttemptsStarted().get()).isEqualTo(1); + assertThat(tracerFactory.getInstance().getAttemptsFailed().get()).isEqualTo(0); + assertThat(tracerFactory.getInstance().getRetriesExhausted().get()).isFalse(); // Capture the argument passed to futureCall ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Integer.class); verify(callInt, atLeastOnce()).futureCall(argumentCaptor.capture(), any(ApiCallContext.class)); diff --git a/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracer.java b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracer.java new file mode 100644 index 0000000000..7a95e94949 --- /dev/null +++ b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracer.java @@ -0,0 +1,81 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.httpjson.testing; + +import com.google.api.gax.tracing.ApiTracer; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.threeten.bp.Duration; + +/** + * Test tracer that keeps count of different events. See {@link TestApiTracerFactory} for more + * details. + */ +public class TestApiTracer implements ApiTracer { + + private final AtomicInteger attemptsStarted = new AtomicInteger(); + private final AtomicInteger attemptsFailed = new AtomicInteger(); + private final AtomicBoolean retriesExhausted = new AtomicBoolean(false); + + public TestApiTracer() {} + + public AtomicInteger getAttemptsStarted() { + return attemptsStarted; + } + + public AtomicInteger getAttemptsFailed() { + return attemptsFailed; + } + + public AtomicBoolean getRetriesExhausted() { + return retriesExhausted; + } + + @Override + public void attemptStarted(int attemptNumber) { + attemptsStarted.incrementAndGet(); + } + + @Override + public void attemptStarted(Object request, int attemptNumber) { + attemptsStarted.incrementAndGet(); + } + + @Override + public void attemptFailed(Throwable error, Duration delay) { + attemptsFailed.incrementAndGet(); + } + + @Override + public void attemptFailedRetriesExhausted(Throwable error) { + attemptsFailed.incrementAndGet(); + retriesExhausted.set(true); + } +}; diff --git a/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracerFactory.java b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracerFactory.java new file mode 100644 index 0000000000..77f05a6cd2 --- /dev/null +++ b/gax-java/gax-httpjson/src/test/java/com/google/api/gax/httpjson/testing/TestApiTracerFactory.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.httpjson.testing; + +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.SpanName; + +/** + * Produces a {@link TestApiTracer}, which keeps count of the attempts made and attempts + * made-and-failed. It also keeps count of the operations failed and when the retries have been + * exhausted. + */ +public class TestApiTracerFactory implements ApiTracerFactory { + private final TestApiTracer instance = new TestApiTracer(); + + public TestApiTracer getInstance() { + return instance; + } + + @Override + public ApiTracer newTracer(ApiTracer parent, SpanName spanName, OperationType operationType) { + return instance; + } +} diff --git a/gax-java/gax/clirr-ignored-differences.xml b/gax-java/gax/clirr-ignored-differences.xml index af5c5f26a1..6e3b3953ac 100644 --- a/gax-java/gax/clirr-ignored-differences.xml +++ b/gax-java/gax/clirr-ignored-differences.xml @@ -106,4 +106,16 @@ com/google/api/gax/batching/Batcher * + + + 7013 + com/google/api/gax/rpc/EndpointContext + * useS2A() + + + + 7012 + com/google/api/gax/rpc/TransportChannelProvider + * withUseS2A(*) + diff --git a/gax-java/gax/pom.xml b/gax-java/gax/pom.xml index 7aa543eaf0..b74d38adb2 100644 --- a/gax-java/gax/pom.xml +++ b/gax-java/gax/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax - 2.57.0 + 2.58.0 jar GAX (Google Api eXtensions) for Java (Core) Google Api eXtensions for Java (Core) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.58.0 diff --git a/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java b/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java index f15046afcb..994ba2eb82 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java @@ -49,7 +49,7 @@ public class GaxProperties { private static final String GAX_VERSION = getLibraryVersion(GaxProperties.class, "version.gax"); private static final String JAVA_VERSION = getRuntimeVersion(); private static final String PROTOBUF_VERSION = - getBundleVersion(Any.class).orElse(DEFAULT_VERSION); + getProtobufVersion(Any.class, "com.google.protobuf.RuntimeVersion");; private GaxProperties() {} @@ -148,4 +148,29 @@ static Optional getBundleVersion(Class clazz) { return Optional.empty(); } } + + /** + * Returns the Protobuf runtime version as reported by com.google.protobuf.RuntimeVersion, if + * class is available, otherwise by reading from MANIFEST file. If niether option is available + * defaults to protobuf version 3 as RuntimeVersion class is available in protobuf version 4+ + */ + @VisibleForTesting + static String getProtobufVersion(Class clazz, String protobufRuntimeVersionClassName) { + try { + Class protobufRuntimeVersionClass = Class.forName(protobufRuntimeVersionClassName); + return protobufRuntimeVersionClass.getField("MAJOR").get(null) + + "." + + protobufRuntimeVersionClass.getField("MINOR").get(null) + + "." + + protobufRuntimeVersionClass.getField("PATCH").get(null); + } catch (ClassNotFoundException + | NoSuchFieldException + | IllegalAccessException + | SecurityException + | NullPointerException e) { + // If manifest file is not available default to protobuf generic version 3 as we know + // RuntimeVersion class is available in protobuf jar 4+. + return getBundleVersion(clazz).orElse("3"); + } + } } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java index 35307764d2..52f923e111 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java @@ -88,7 +88,7 @@ private static String checkAndAppendProtobufVersionIfNecessary( // TODO(b/366417603): appending protobuf version to existing client library token until resolved Pattern pattern = Pattern.compile("(gccl|gapic)\\S*"); Matcher matcher = pattern.matcher(apiClientHeaderValue); - if (matcher.find()) { + if (matcher.find() && GaxProperties.getProtobufVersion() != null) { return apiClientHeaderValue.substring(0, matcher.end()) + "--" + PROTOBUF_HEADER_VERSION_KEY diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientContext.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientContext.java index 5bce1ac6bb..8e7c9a3090 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientContext.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ClientContext.java @@ -222,6 +222,7 @@ public static ClientContext create(StubSettings settings) throws IOException { if (transportChannelProvider.needsEndpoint()) { transportChannelProvider = transportChannelProvider.withEndpoint(endpoint); } + transportChannelProvider = transportChannelProvider.withUseS2A(endpointContext.useS2A()); TransportChannel transportChannel = transportChannelProvider.getTransportChannel(); ApiCallContext defaultCallContext = diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java index dd6c199b35..0148c07a01 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/EndpointContext.java @@ -30,6 +30,7 @@ package com.google.api.gax.rpc; import com.google.api.core.InternalApi; +import com.google.api.gax.rpc.internal.EnvironmentProvider; import com.google.api.gax.rpc.mtls.MtlsProvider; import com.google.auth.Credentials; import com.google.auth.oauth2.ComputeEngineCredentials; @@ -65,6 +66,9 @@ public abstract class EndpointContext { "The configured universe domain (%s) does not match the universe domain found in the credentials (%s). If you haven't configured the universe domain explicitly, `googleapis.com` is the default."; public static final String UNABLE_TO_RETRIEVE_CREDENTIALS_ERROR_MESSAGE = "Unable to retrieve the Universe Domain from the Credentials."; + // This environment variable is a temporary measure. It will be removed when the feature is + // non-experimental. + static final String S2A_ENV_ENABLE_USE_S2A = "EXPERIMENTAL_GOOGLE_API_USE_S2A"; public static EndpointContext getDefaultInstance() { return INSTANCE; @@ -100,6 +104,11 @@ public static EndpointContext getDefaultInstance() { @Nullable public abstract String transportChannelProviderEndpoint(); + abstract boolean useS2A(); + + @Nullable + abstract EnvironmentProvider envProvider(); + @Nullable public abstract String mtlsEndpoint(); @@ -119,7 +128,8 @@ public static EndpointContext getDefaultInstance() { public static Builder newBuilder() { return new AutoValue_EndpointContext.Builder() .setSwitchToMtlsEndpointAllowed(false) - .setUsingGDCH(false); + .setUsingGDCH(false) + .setEnvProvider(System::getenv); } /** Configure the existing EndpointContext to be using GDC-H */ @@ -208,6 +218,10 @@ public abstract static class Builder { public abstract Builder setResolvedUniverseDomain(String resolvedUniverseDomain); + abstract Builder setUseS2A(boolean useS2A); + + abstract Builder setEnvProvider(EnvironmentProvider envProvider); + abstract String serviceName(); abstract String universeDomain(); @@ -216,6 +230,10 @@ public abstract static class Builder { abstract String transportChannelProviderEndpoint(); + abstract boolean useS2A(); + + abstract EnvironmentProvider envProvider(); + abstract String mtlsEndpoint(); abstract boolean switchToMtlsEndpointAllowed(); @@ -254,6 +272,10 @@ private String determineUniverseDomain() { /** Determines the fully resolved endpoint and universe domain values */ private String determineEndpoint() throws IOException { + if (shouldUseS2A()) { + return mtlsEndpoint(); + } + MtlsProvider mtlsProvider = mtlsProvider() == null ? new MtlsProvider() : mtlsProvider(); // TransportChannelProvider's endpoint will override the ClientSettings' endpoint String customEndpoint = @@ -288,6 +310,32 @@ private String determineEndpoint() throws IOException { return endpoint; } + /** Determine if S2A can be used */ + @VisibleForTesting + boolean shouldUseS2A() { + // If EXPERIMENTAL_GOOGLE_API_USE_S2A is not set to true, skip S2A. + String s2AEnv; + s2AEnv = envProvider().getenv(S2A_ENV_ENABLE_USE_S2A); + boolean s2AEnabled = Boolean.parseBoolean(s2AEnv); + if (!s2AEnabled) { + return false; + } + + // Skip S2A when using GDC-H + if (usingGDCH()) { + return false; + } + + // If a custom endpoint is being used, skip S2A. + if (!Strings.isNullOrEmpty(clientSettingsEndpoint()) + || !Strings.isNullOrEmpty(transportChannelProviderEndpoint())) { + return false; + } + + // mTLS via S2A is not supported in any universe other than googleapis.com. + return mtlsEndpoint().contains(Credentials.GOOGLE_DEFAULT_UNIVERSE); + } + // Default to port 443 for HTTPS. Using HTTP requires explicitly setting the endpoint private String buildEndpointTemplate(String serviceName, String resolvedUniverseDomain) { return serviceName + "." + resolvedUniverseDomain + ":443"; @@ -321,6 +369,7 @@ public EndpointContext build() throws IOException { // The Universe Domain is used to resolve the Endpoint. It should be resolved first setResolvedUniverseDomain(determineUniverseDomain()); setResolvedEndpoint(determineEndpoint()); + setUseS2A(shouldUseS2A()); return autoBuild(); } } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/FixedTransportChannelProvider.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/FixedTransportChannelProvider.java index 0bf6205dd9..2f70c06b5f 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/FixedTransportChannelProvider.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/FixedTransportChannelProvider.java @@ -89,6 +89,12 @@ public TransportChannelProvider withEndpoint(String endpoint) { "FixedTransportChannelProvider doesn't need an endpoint"); } + @Override + public TransportChannelProvider withUseS2A(boolean useS2A) throws UnsupportedOperationException { + // Overriden for technical reasons. This method is a no-op for FixedTransportChannelProvider. + return this; + } + /** @deprecated FixedTransportChannelProvider doesn't support ChannelPool configuration */ @Deprecated @Override diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannelProvider.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannelProvider.java index 21f3c31f63..f58acffc54 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannelProvider.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/TransportChannelProvider.java @@ -97,6 +97,11 @@ public interface TransportChannelProvider { */ TransportChannelProvider withEndpoint(String endpoint); + /** Sets whether to use S2A when constructing a new {@link TransportChannel}. */ + default TransportChannelProvider withUseS2A(boolean useS2A) { + throw new UnsupportedOperationException("S2A is not supported"); + } + /** * Reports whether this provider allows pool size customization. * diff --git a/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/native-image.properties b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/native-image.properties index afad5895db..75b882d8a4 100644 --- a/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/native-image.properties +++ b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/native-image.properties @@ -1,6 +1,9 @@ Args = --enable-url-protocols=https,http \ --initialize-at-build-time=org.conscrypt,\ - org.junit.platform.engine.TestTag \ + org.junit.platform.engine.TestTag,\ + com.google.api.gax.core.GaxProperties,\ + com.google.common.base.Platform,\ + com.google.common.base.Platform$JdkPatternCompiler \ --features=com.google.api.gax.nativeimage.OpenCensusFeature,\ com.google.api.gax.nativeimage.GoogleJsonClientFeature \ --add-modules=jdk.httpserver diff --git a/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json new file mode 100644 index 0000000000..3a38d53361 --- /dev/null +++ b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json @@ -0,0 +1,10 @@ +[ + { + "name": "com.google.protobuf.RuntimeVersion", + "fields" : [ + { "name" : "MAJOR" }, + { "name" : "MINOR" }, + { "name" : "PATCH" } + ] + } +] \ No newline at end of file diff --git a/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java b/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java index 1369ec35ae..6560df4bc1 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java @@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.base.Strings; +import com.google.protobuf.Any; import java.io.IOException; import java.util.Optional; import java.util.regex.Pattern; @@ -160,12 +161,8 @@ void testGetJavaRuntimeInfo_nullJavaVersion() { @Test public void testGetProtobufVersion() throws IOException { - Version version = readVersion(GaxProperties.getProtobufVersion()); - - assertTrue(version.major >= 3); - if (version.major == 3) { - assertTrue(version.minor >= 25); - } + assertTrue( + Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(GaxProperties.getProtobufVersion()).find()); } @Test @@ -175,6 +172,36 @@ public void testGetBundleVersion_noManifestFile() throws IOException { assertFalse(version.isPresent()); } + @Test + void testGetProtobufVersion_success() { + String version = + GaxProperties.getProtobufVersion( + Any.class, "com.google.api.gax.core.GaxPropertiesTest$RuntimeVersion"); + + assertEquals("3.13.6", version); + } + + @Test + void testGetProtobufVersion_classNotFoundException() throws Exception { + String version = GaxProperties.getProtobufVersion(Any.class, "foo.NonExistantClass"); + + assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); + } + + @Test + void testgetProtobufVersion_noSuchFieldException() throws Exception { + String version = GaxProperties.getProtobufVersion(Any.class, "java.lang.Class"); + + assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); + } + + @Test + void testGetProtobufVersion_noManifest() throws Exception { + String version = GaxProperties.getProtobufVersion(GaxProperties.class, "foo.NonExistantClass"); + + assertEquals("3", version); + } + private Version readVersion(String version) { assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); String[] versionComponents = version.split("\\."); @@ -194,4 +221,11 @@ public Version(int major, int minor) { this.minor = minor; } } + + // Test class that emulates com.google.protobuf.RuntimeVersion for reflection lookup of fields + class RuntimeVersion { + public static final int MAJOR = 3; + public static final int MINOR = 13; + public static final int PATCH = 6; + } } diff --git a/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java b/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java index 53c1707290..5a65895ef0 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java @@ -195,13 +195,13 @@ void testCancelGetAttempt(boolean withCustomRetrySettings) throws Exception { setUp(withCustomRetrySettings); for (int executionsCount = 0; executionsCount < EXECUTIONS_COUNT; executionsCount++) { ScheduledExecutorService localExecutor = Executors.newSingleThreadScheduledExecutor(); - final int maxRetries = 100; + final int maxRetries = 20; FailingCallable callable = new FailingCallable(maxRetries - 1, "request", "SUCCESS", tracer); RetrySettings retrySettings = FAST_RETRY_SETTINGS .toBuilder() - .setTotalTimeoutDuration(java.time.Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(java.time.Duration.ofMillis(5000L)) .setMaxAttempts(maxRetries) .build(); @@ -259,10 +259,11 @@ void testCancelOuterFutureAfterStart() throws Exception { .toBuilder() // These params were selected to ensure that future tries to run and fail (at least // once) but does not complete before it is cancelled. Assuming no computation time, - // it would take 25 + 100 + 400 + 1000 = 1525ms for the future to complete, which should + // it would take 2500 + 10000 + 10000 + 10000 = 32500ms for the future to complete, + // which should // be more than enough time to cancel the future. - .setInitialRetryDelayDuration(java.time.Duration.ofMillis(25L)) - .setMaxRetryDelayDuration(java.time.Duration.ofMillis(1000L)) + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(2500L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(10000L)) .setRetryDelayMultiplier(4.0) .setTotalTimeoutDuration(java.time.Duration.ofMillis(60000L)) // Set this test to not use jitter as the randomized retry delay (RRD) may introduce diff --git a/gax-java/gax/src/test/java/com/google/api/gax/rpc/ClientContextTest.java b/gax-java/gax/src/test/java/com/google/api/gax/rpc/ClientContextTest.java index 826864a49c..facc93ed86 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/rpc/ClientContextTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/rpc/ClientContextTest.java @@ -195,6 +195,17 @@ public TransportChannelProvider withEndpoint(String endpoint) { endpoint); } + @Override + public TransportChannelProvider withUseS2A(boolean useS2A) { + return new FakeTransportProvider( + this.transport, + this.executor, + this.shouldAutoClose, + this.headers, + this.credentials, + this.endpoint); + } + @Override public boolean acceptsPoolSize() { return false; diff --git a/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java b/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java index 3276e4a73e..5561427dde 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/rpc/EndpointContextTest.java @@ -33,6 +33,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.google.api.gax.core.NoCredentialsProvider; +import com.google.api.gax.rpc.internal.EnvironmentProvider; import com.google.api.gax.rpc.mtls.MtlsProvider; import com.google.api.gax.rpc.testing.FakeMtlsProvider; import com.google.auth.Credentials; @@ -454,4 +455,97 @@ void hasValidUniverseDomain_computeEngineCredentials_noValidationOnUniverseDomai .build(); assertDoesNotThrow(() -> endpointContext.validateUniverseDomain(credentials, statusCode)); } + + @Test + void shouldUseS2A_envVarNotSet_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("false"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_UsingGDCH_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("") + .setUsingGDCH(true); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_customEndpointSetViaClientSettings_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("test.endpoint.com:443") + .setTransportChannelProviderEndpoint("") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_customEndpointSetViaTransportChannelProvider_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("test.endpoint.com:443") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_mtlsEndpointEmpty_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("") + .setMtlsEndpoint("") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_mtlsEndpointNotGoogleDefaultUniverse_returnsFalse() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("") + .setMtlsEndpoint("test.mtls.abcd.com:443") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isFalse(); + } + + @Test + void shouldUseS2A_success() throws IOException { + EnvironmentProvider envProvider = Mockito.mock(EnvironmentProvider.class); + Mockito.when(envProvider.getenv(EndpointContext.S2A_ENV_ENABLE_USE_S2A)).thenReturn("true"); + defaultEndpointContextBuilder = + defaultEndpointContextBuilder + .setEnvProvider(envProvider) + .setClientSettingsEndpoint("") + .setTransportChannelProviderEndpoint("") + .setUsingGDCH(false); + Truth.assertThat(defaultEndpointContextBuilder.shouldUseS2A()).isTrue(); + } } diff --git a/gax-java/pom.xml b/gax-java/pom.xml index a5d9a10384..7dc3f5b209 100644 --- a/gax-java/pom.xml +++ b/gax-java/pom.xml @@ -4,14 +4,14 @@ com.google.api gax-parent pom - 2.57.0 + 2.58.0 GAX (Google Api eXtensions) for Java (Parent) Google Api eXtensions for Java (Parent) com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 ../gapic-generator-java-pom-parent @@ -50,7 +50,7 @@ com.google.api api-common - 2.40.0 + 2.41.0 com.google.auth @@ -98,24 +98,24 @@ com.google.api gax - 2.57.0 + 2.58.0 com.google.api gax - 2.57.0 + 2.58.0 test-jar testlib com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.49.0 com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.49.0 io.grpc diff --git a/generation_config.yaml b/generation_config.yaml index 484c904e7a..b6b681dc07 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,4 @@ -googleapis_commitish: 537fd482f6bb8afb3a146d9b21673a8eb27958bd +googleapis_commitish: 6b5d85c66e0885b1665040f6f80b2401f60c1068 # the libraries are ordered with respect to library name, which is # java-{library.library_name} or java-{library.api-shortname} when # library.library_name is not defined. diff --git a/hermetic_build/DEVELOPMENT.md b/hermetic_build/DEVELOPMENT.md new file mode 100644 index 0000000000..1969bbcdfb --- /dev/null +++ b/hermetic_build/DEVELOPMENT.md @@ -0,0 +1,230 @@ +> [!IMPORTANT] +> All examples assume you are inside the repository root folder. + + +# Linting + +When contributing, ensure your changes to python code have a valid format. + +``` +python -m pip install black +black {source_file_or_directory} +``` + +# Install package dependencies + +```shell +python -m pip install --require-hashes -r hermetic_build/common/requirements.txt +python -m pip install hermetic_build/common +python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt +python -m pip install hermetic_build/library_generation +python -m pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt +python -m pip install hermetic_build/release_note_generation +``` + +# Run the integration tests + +The integration tests build the docker image declared in +`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC +repositories, generate the libraries and compare the results with the source +code declared in a "golden branch" of the repo. + +It requires docker and python (>= 3.12.0) to be installed. + +```shell +python -m unittest hermetic_build/library_generation/tests/integration_tests.py +``` + +# Run the unit tests + +There is one unit test file per component. +Every unit test script ends with `unit_tests.py`. +To avoid specifying them individually, we can use the following command: + +```shell +python -m unittest discover -s hermetic_build -p "*unit_tests.py" +``` + +> [!NOTE] +> The output of this command may look erratic during the first 30 seconds. +> This is normal. After the tests are done, an "OK" message should be shown. + +# Run the library generation scripts in your local environment + +Although the scripts are designed to run in a Docker container, you can also +run them directly. +This section explains how to run the entrypoint script +(`hermetic_build/library_generation/cli/entry_point.py`). + +## Assumptions made by the scripts + +### The Hermetic Build's well-known folder +Located in `${HOME}/.library_generation`, this folder is assumed by the scripts +to contain certain tools. + +Developers must make sure this folder is properly configured before running the +scripts locally. +Note that this relies on the `HOME` environment variable which is always defined +as per [POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). + +#### Put the gapic-generator-java jar in its well-known location + +1. Run the following command to install gapic-generator-java. + + ```shell + mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip + ``` + This will generate a jar located in `~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` + +2. Move the jar into its well-known location. + + ```shell + mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" + ``` + +#### Put the java formatter jar in its well-known location + +1. Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) +or [GitHub releases](https://github.com/google/google-java-format/releases). +2. Move the jar into its well-known location. + + ```shell + mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" + ``` + +## Installing prerequisites + +In order to run the generation scripts directly, there are a few tools we +need to install beforehand. + +### Install the owl-bot CLI + +This requires node.js to be installed. +Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) +for NVM, Node.js's version manager. + +After you install it, you can install the owl-bot CLI with the following +commands: + +```shell +git clone https://github.com/googleapis/repo-automation-bots +cd repo-automation-bots/packages/owl-bot +npm i && npm run compile && npm link +owl-bot copy-code --version +``` + +The key step is `npm link`, which will make the command available in you current +shell session. + +## Run the script +The entrypoint script (`hermetic_build/library_generation/cli/entry_point.py`) +allows you to generate a GAPIC repository with a given api definition (proto, +service yaml). + +### Download the api definition +For example, from googleapis + +```shell +git clone https://github.com/googleapis/googleapis +export api_definitions_path="$(pwd)/googleapis" +``` + +### Download the repo +For example, google-cloud-java +```shell +git clone https://github.com/googleapis/google-cloud-java +export path_to_repo="$(pwd)/google-cloud-java" +``` + +### Install the scripts + +You can skip this step if you've installed the packages in [Install package dependencies](#install-package-dependencies). + +```shell +python -m pip install --require-hashes -r hermetic_build/common/requirements.txt +python -m pip install hermetic_build/common +python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt +python -m pip install hermetic_build/library_generation +``` + +### Run the script + +```shell +python hermetic_build/library_generation/cli/entry_point.py generate \ + --repository-path="${path_to_repo}" \ + --api-definitions-path="${api_definitions_path}" +``` + +# Build the image from source + +1. Run the following command to build the image from source + + ```shell + DOCKER_BUILDKIT=1 docker build \ + -f .cloudbuild/library_generation/library_generation.Dockerfile \ + -t local:image-tag \ + . + ``` + Please note that the build only works when using the new [Docker BuildKit](https://docs.docker.com/build/buildkit/) + (enabled through the `DOCKER_BUILDKIT` variable). + +2. Set the version of gapic-generator-java + + ```shell + LOCAL_GENERATOR_VERSION=$(mvn \ + org.apache.maven.plugins:maven-help-plugin:evaluate \ + -Dexpression=project.version \ + -pl gapic-generator-java \ + -DforceStdout \ + -q) + ``` + +3. Run the image + + ```shell + # Assume you want to generate the library in the current working directory + # and the generation configuration is in the same directory. + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api-definitions:/workspace/apis \ + -e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}" \ + local:image-tag \ + --generation-config-path=/workspace/generation_config_file \ + --library-names=apigee-connect,asset \ + --repository-path=/workspace \ + --api-definitions-path=/workspace/apis + ``` + Note that if you specify the generator version using environment variable, + `-e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}"` in the above example, + you should not set `gapic_generator_version` and `protoc_version` in the + generation configuration because values in the generation configuration will + take precedence. + +# Debug the library generation container +If you are working on changing the way the containers are created, you may want +to inspect the containers to check the setup. +It would be convenient in such case to have a text editor/viewer available. +You can achieve this by modifying the Dockerfile as follows: + +```dockerfile +# install OS tools +RUN apk update && apk add \ + unzip curl rsync openjdk11 jq bash nodejs npm git less vim +``` + +We add `less` and `vim` as text tools for further inspection. + +You can also run a shell in a new container by running: + +```shell +docker run \ + --rm \ + -it \ + -u $(id -u):$(id -g) \ + -v /path/to/google-cloud-java:/workspace \ + --entrypoint="bash" \ + $(cat image-id) +``` diff --git a/hermetic_build/library_generation/README.md b/hermetic_build/README.md similarity index 67% rename from hermetic_build/library_generation/README.md rename to hermetic_build/README.md index 0b4208ac3e..583e766bdf 100644 --- a/hermetic_build/library_generation/README.md +++ b/hermetic_build/README.md @@ -1,44 +1,43 @@ +> [!IMPORTANT] +> All scripts/examples assume you are inside the repository root folder. + # Generate a repository containing GAPIC Client Libraries -The script, `entry_point.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. +Running the docker image built from `hermetic_build/library_generation` +directory, you can generate a repository containing GAPIC client libraries (a +monorepo, for example, google-cloud-java) from a configuration file. + +Instead of running the docker image, if you prefer running the underlying python +scripts directly, please refer to the [development guide](DEVELOPMENT.md#run-the-script) +for additional instructions. ## Environment - OS: Linux -- Java runtime environment (8 or above) -- Python (3.12 or above) -- Docker -- Git +- Docker -## Prerequisite +## Prerequisites In order to generate a version for each library, a versions.txt has to exist -in `repository_path`. -Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. - -## Parameters to generate a repository using `entry_point.py` - -### Baseline generation configuration yaml (`baseline_generation_config`) +in `repository-path`. +Please refer to [Repository path](#repository-path--repositorypath---optional) +for more information. -An absolute or relative path to a generation_config.yaml. -This config file is used for computing changed libraries, not library -generation. +## Parameters to generate a repository using the docker image -### Current generation configuration yaml (`current_generation_config`) +### Generation configuration yaml (`generation-config-path`) An absolute or relative path to a configuration file containing parameters to generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) for more information. -### Repository path (`repository_path`), optional +### Repository path (`repository-path`), optional The path to where the generated repository goes. The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python entry_point.py ...` without +For example, `cd google-cloud-java && python /path/to/entry_point.py ...` without specifying the `--repository_path` option will modify the `google-cloud-java` repository the user `cd`'d into. @@ -47,28 +46,37 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. -### Api definitions path (`api_definitions_path`), optional +### A list of library names (`library-names`), optional + +A list of library names that will be generated, separated by comma. +The library name of a library is the value of `library_name` or `api_shortname`, +if `library_name` is not specified, in the generation configuration. + +If not specified, all libraries in the generation +configuration will be generated. + +### Api definitions path (`api-definitions-path`), optional The path to where the api definition (proto, service yaml) resides. The default value is the current working directory when running the script. -Note that you need not only the protos defined the service, but also the transitive -dependencies of those protos. +**Note that you need not only the protos defined the service, but also the +transitive dependencies of those protos.** Any missing dependencies will cause `File not found` error. -For example, if your service is defined in `example_service.proto` and it imports -`google/api/annotations.proto`, you need the `annotations.proto` resides in a -folder that has the exact structure of the import statement (`google/api` in this -case), and set `api_definitions_path` to the path contains the root folder (`google` -in this case). +For example, if your service is defined in `example_service.proto` and it +imports `google/api/annotations.proto`, you need the `annotations.proto` resides +in a folder that has the exact structure of the import statement (`google/api` +in this case), and set `api_definitions_path` to the path contains the root +folder (`google` in this case). -## Output of `entry_point.py` +## Output ### GAPIC libraries -For each module (e.g. `google-cloud-java/java-asset`), the following files/folders -will be created/modified: +For each module (e.g. `google-cloud-java/java-asset`), the following +files/folders will be created/modified: | Name | Notes | |:------------------------------------|:-------------------------------------------------------------------------| @@ -185,32 +193,58 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -# Local Environment Setup before running `entry_point.py` - -1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. -The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). -2. Assuming the virtual environment is installed under `sdk-platform-java`. -Run the following command under the root folder of `sdk-platform-java` to install `library_generation` and its dependencies. +# Run the library generation image + +1. Download the API definitions to a local directory, e.g., from [googleapis](https://github.com/googleapis/googleapis). + +2. Run the docker image. + ```shell + # Assume you want to generate the library in the current working directory + # and the generation configuration is in the same directory. + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api_definition:/workspace \ + gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag + ``` - ```bash - python -m pip install --require-hashes -r hermetic_build/common/requirements.txt - python -m pip install hermetic_build/common - python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - python -m pip install hermetic_build/library_generation + * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating + yourself. + This avoids folder ownership changes since it runs as root by default. + * `-v "$(pwd):/workspace"` maps the host machine's current working directory + to the /workspace folder. + The image is configured to perform changes in this directory. + * `-v /path/to/api_definition:/workspace` maps the host machine's API + definitions folder to `/workspace/apis` folder. + +3. An advanced example: + ```shell + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api_definition:/workspace/apis \ + gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag \ + --generation-config-path=/workspace/generation_config_file \ + --library-names=apigee-connect,asset \ + --repository-path=/workspace \ + --api-definitions-path=/workspace/apis ``` + + * `--generation-config-path=/workspace/generation_config_file` set the + generation configuration to `/workspace/generation_config_file`. + * `--api-definitions-path=/workspace/apis` set the API definition path to + `/workspace/apis`. -3. Download api definition to a local directory +To debug the image, please refer to [development guide](DEVELOPMENT.md#debug-the-library-generation-container) +for more info. -## An example to generate a repository using `entry_point.py` +## An example to generate a repository using the docker image -```bash -python hermetic_build/library_generation/cli/entry_point.py generate \ - --baseline-generation-config-path=/path/to/baseline_config_file \ - --current-generation-config-path=/path/to/current_config_file \ - --repository-path=path/to/repository \ - --api-definitions-path=path/to/api_definition -``` -If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) +If you run the docker image with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` $repository_path @@ -284,7 +318,70 @@ $repository_path |_versions.txt ``` -# Owlbot Java Postprocessor +# Generate release notes from API definition changes + +The script, `hermetic_build/release_note_generation/cli/generate_release_note.py` +allows you to generate a file containing release notes from API definition +changes in [googleapis](https://github.com/googleapis/googleapis) GitHub +repository. + +## Environment + +- OS: Linux +- Python (3.12.0 or above) + +## Parameters to generate a release note + +### Baseline generation configuration path (`baseline-generation-config-path`) + +Absolute or relative path to a generation configuration. +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. + +Note that the `googleapis_commitish` in this configuration is used to retrieve +the first commit, exclusively, to generate the release notes. + +### Current generation configuration path (`current-generation-config-path`) + +Absolute or relative path to a generation configuration. +The release notes will be generated from commits that are related to the +libraries specified in this configuration. +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. + +Note that the `googleapis_commitish` entry in this configuration is used to +retrieve the last commit, inclusively, to generate the release notes. + +### Repository path (`repository-path`), optional + +The path to which the file, `pr_description.txt` containing the release notes +will be sent. +If not specified, the file will be generated to the current working directory. + +## Generate a release notes file in a local environment + +1. Install python (>= 3.12.0). +It is recommended to create a python virtual environment through the +[official guide](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments). + +2. Run the following commands to install python packages + ```shell + cd /path/to/sdk-platform-java + pip install --require-hashes -r hermetic_build/common/requirements.txt + pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt + pip install hermetic_build/release_note_generation + ``` +3. Run the following commands to generate a release note + ```shell + cd /path/to/sdk-platform-java + python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ + --baseline-generation-config-path=/path/to/baseline_generation_config \ + --current-generation-config-path=/path/to/current_generation_config \ + --repository-path=/path/to/send/release_note + ``` + +# OwlBot Java Postprocessor We have transferred the [implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) diff --git a/hermetic_build/common/cli/get_changed_libraries.py b/hermetic_build/common/cli/get_changed_libraries.py new file mode 100644 index 0000000000..cf92cf0853 --- /dev/null +++ b/hermetic_build/common/cli/get_changed_libraries.py @@ -0,0 +1,83 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click + +from common.model.generation_config import from_yaml +from common.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config-path", + required=True, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for computing changed library list. + """, +) +@click.option( + "--current-generation-config-path", + required=True, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +def create( + baseline_generation_config_path: str, + current_generation_config_path: str, +) -> None: + """ + Compares baseline generation config with current generation config and + generates changed library names (a comma separated string) based on current + generation config. + """ + baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) + if not os.path.isfile(baseline_generation_config_path): + raise FileNotFoundError( + f"{baseline_generation_config_path} does not exist. " + "A valid generation config has to be passed in as " + "baseline-generation-config-path." + ) + current_generation_config_path = os.path.abspath(current_generation_config_path) + if not os.path.isfile(current_generation_config_path): + raise FileNotFoundError( + f"{current_generation_config_path} does not exist. " + "A valid generation config has to be passed in as " + "current-generation-config-path." + ) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config_path), + current_config=from_yaml(current_generation_config_path), + ) + changed_libraries = config_change.get_changed_libraries() + if changed_libraries is None: + print("No changed library.") + return + click.echo(",".join(config_change.get_changed_libraries())) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/model/config_change.py b/hermetic_build/common/model/config_change.py similarity index 86% rename from hermetic_build/library_generation/model/config_change.py rename to hermetic_build/common/model/config_change.py index 018aee8ccd..7ddc338448 100644 --- a/hermetic_build/library_generation/model/config_change.py +++ b/hermetic_build/common/model/config_change.py @@ -11,17 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import shutil +import tempfile from enum import Enum from typing import Optional from git import Commit, Repo - from common.model.gapic_inputs import parse_build_str from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig -from library_generation.utils.utilities import sh_util -from library_generation.utils.proto_path_utils import find_versioned_proto_path +from common.utils.proto_path_utils import find_versioned_proto_path INSERTIONS = "insertions" LINES = "lines" @@ -109,25 +106,22 @@ def get_qualified_commits( :param repo_url: the repository contains the commit history. :return: QualifiedCommit objects. """ - tmp_dir = sh_util("get_output_folder") - shutil.rmtree(tmp_dir, ignore_errors=True) - os.mkdir(tmp_dir) - # we only need commit history, thus shadow clone is enough. - repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.current_config.googleapis_commitish) - proto_paths = self.current_config.get_proto_path_to_library_name() - qualified_commits = [] - while str(commit.hexsha) != self.baseline_config.googleapis_commitish: - qualified_commit = ConfigChange.__create_qualified_commit( - proto_paths=proto_paths, commit=commit - ) - if qualified_commit is not None: - qualified_commits.append(qualified_commit) - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - shutil.rmtree(tmp_dir, ignore_errors=True) + with tempfile.TemporaryDirectory() as tmp_dir: + # we only need commit history, thus a shadow clone is enough. + repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() + qualified_commits = [] + while str(commit.hexsha) != self.baseline_config.googleapis_commitish: + qualified_commit = ConfigChange.__create_qualified_commit( + proto_paths=proto_paths, commit=commit + ) + if qualified_commit is not None: + qualified_commits.append(qualified_commit) + commit_parents = commit.parents + if len(commit_parents) == 0: + break + commit = commit_parents[0] return qualified_commits def __get_library_names_from_qualified_commits(self) -> list[str]: diff --git a/hermetic_build/common/requirements.in b/hermetic_build/common/requirements.in index a34205e5fa..21607220f6 100644 --- a/hermetic_build/common/requirements.in +++ b/hermetic_build/common/requirements.in @@ -1,3 +1,4 @@ black==24.8.0 +GitPython==3.1.43 parameterized==0.9.0 PyYAML==6.0.2 \ No newline at end of file diff --git a/hermetic_build/common/requirements.txt b/hermetic_build/common/requirements.txt index d952506bb5..9b79817c1b 100644 --- a/hermetic_build/common/requirements.txt +++ b/hermetic_build/common/requirements.txt @@ -32,6 +32,14 @@ click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via black +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff + # via -r hermetic_build/common/requirements.in mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 @@ -107,3 +115,7 @@ pyyaml==6.0.2 \ --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 # via -r hermetic_build/common/requirements.in +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb diff --git a/hermetic_build/common/tests/cli/__init__.py b/hermetic_build/common/tests/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/common/tests/cli/config_change_unit_tests.py b/hermetic_build/common/tests/cli/config_change_unit_tests.py new file mode 100644 index 0000000000..e3bdd753de --- /dev/null +++ b/hermetic_build/common/tests/cli/config_change_unit_tests.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from click.testing import CliRunner +import unittest + +from common.cli.get_changed_libraries import create + +script_dir = os.path.dirname(os.path.realpath(__file__)) +test_resource_dir = os.path.join(script_dir, "..", "resources", "cli") + + +class GetChangedLibrariesTest(unittest.TestCase): + def test_entry_point_without_baseline_config_raise_system_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(create) + self.assertEqual(2, result.exit_code) + self.assertEqual(SystemExit, result.exc_info[0]) + + def test_entry_point_without_current_config_raise_system_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, ["--baseline-generation-config-path=/invalid/path/file"] + ) + self.assertEqual(2, result.exit_code) + self.assertEqual(SystemExit, result.exc_info[0]) + + def test_entry_point_with_invalid_baseline_config_raise_file_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, + [ + "--baseline-generation-config-path=/invalid/path/file", + "--current-generation-config-path=/invalid/path/file", + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex(result.exception.args[0], "baseline-generation-config-path") + + def test_entry_point_with_invalid_current_config_raise_file_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, + [ + f"--baseline-generation-config-path={test_resource_dir}/empty_config.yaml", + "--current-generation-config-path=/invalid/path/file", + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex(result.exception.args[0], "current-generation-config-path") diff --git a/hermetic_build/library_generation/tests/model/config_change_unit_tests.py b/hermetic_build/common/tests/model/config_change_unit_tests.py similarity index 98% rename from hermetic_build/library_generation/tests/model/config_change_unit_tests.py rename to hermetic_build/common/tests/model/config_change_unit_tests.py index 6e0a088e75..3abc603141 100644 --- a/hermetic_build/library_generation/tests/model/config_change_unit_tests.py +++ b/hermetic_build/common/tests/model/config_change_unit_tests.py @@ -13,9 +13,9 @@ # limitations under the License. import unittest -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange +from common.model.config_change import ChangeType +from common.model.config_change import ConfigChange +from common.model.config_change import LibraryChange from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig diff --git a/hermetic_build/common/tests/resources/cli/empty_config.yaml b/hermetic_build/common/tests/resources/cli/empty_config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/common/tests/utils/__init__.py b/hermetic_build/common/tests/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py b/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py similarity index 99% rename from hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py rename to hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py index f88f71d40e..00edb511eb 100644 --- a/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py +++ b/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py @@ -16,8 +16,8 @@ from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig -from library_generation.utils.generation_config_comparator import ChangeType -from library_generation.utils.generation_config_comparator import compare_config +from common.utils.generation_config_comparator import ChangeType +from common.utils.generation_config_comparator import compare_config class GenerationConfigComparatorTest(unittest.TestCase): diff --git a/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py new file mode 100644 index 0000000000..90b3dd3f55 --- /dev/null +++ b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path +from common.utils.proto_path_utils import find_versioned_proto_path + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + + +class ProtoPathsUtilsTest(unittest.TestCase): + def test_find_versioned_proto_path_nested_version_success(self): + proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" + expected = "google/cloud/aiplatform/v1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_path_success(self): + proto_path = "google/cloud/asset/v1p2beta1/assets.proto" + expected = "google/cloud/asset/v1p2beta1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_without_version_return_itself(self): + proto_path = "google/type/color.proto" + expected = "google/type/color.proto" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) diff --git a/hermetic_build/library_generation/utils/generation_config_comparator.py b/hermetic_build/common/utils/generation_config_comparator.py similarity index 97% rename from hermetic_build/library_generation/utils/generation_config_comparator.py rename to hermetic_build/common/utils/generation_config_comparator.py index d0851c7f31..f41299ddc2 100644 --- a/hermetic_build/library_generation/utils/generation_config_comparator.py +++ b/hermetic_build/common/utils/generation_config_comparator.py @@ -15,10 +15,10 @@ from typing import Any from typing import Dict from typing import List -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange -from library_generation.model.config_change import HashLibrary +from common.model.config_change import ChangeType +from common.model.config_change import ConfigChange +from common.model.config_change import LibraryChange +from common.model.config_change import HashLibrary from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig diff --git a/hermetic_build/common/utils/proto_path_utils.py b/hermetic_build/common/utils/proto_path_utils.py new file mode 100644 index 0000000000..49a86dcbd2 --- /dev/null +++ b/hermetic_build/common/utils/proto_path_utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + + +def find_versioned_proto_path(proto_path: str) -> str: + """ + Returns a versioned proto_path from a given proto_path; or proto_path itself + if it doesn't contain a versioned proto_path. + :param proto_path: a proto file path + :return: the versioned proto_path + """ + version_regex = re.compile(r"^v[1-9].*") + directories = proto_path.split("/") + for directory in directories: + result = version_regex.search(directory) + if result: + version = result[0] + idx = proto_path.find(version) + return proto_path[:idx] + version + return proto_path diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md deleted file mode 100644 index df7843aeaa..0000000000 --- a/hermetic_build/library_generation/DEVELOPMENT.md +++ /dev/null @@ -1,207 +0,0 @@ -> [!IMPORTANT] -> All examples assume you are inside the `hermetic_build` folder. - - -# Linting - -When contributing, ensure your changes to python code have a valid format. - -``` -python -m pip install black -black . -``` - -# Running the integration tests - -The integration tests build the docker image declared in -`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC -repositories, generate the libraries and compares the results with the source -code declared in a "golden branch" of the repo. - -It requires docker and python 3.x to be installed. - -``` -python -m pip install --require-hashes -r library_generation/requirements.txt -python -m pip install library_generation -python -m unittest library_generation/tests/integration_tests.py -``` - -# Running the unit tests - -The unit tests of the hermetic build scripts are contained in several scripts, -corresponding to a specific component. -Every unit test script ends with `unit_tests.py`. -To avoid them specifying them individually, we can use the following command: - -```bash -python -m unittest discover -s library_generation/tests/ -p "*unit_tests.py" -``` - -> [!NOTE] -> The output of this command may look erratic during the first 30 seconds. -> This is normal. After the tests are done, an "OK" message should be shown. - -# Running the scripts in your local environment - -Although the scripts are designed to be run in a Docker container, you can also -run them directly. -This section explains how to run the entrypoint script -(`library_generation/cli/entry_point.py`). - -## Assumptions made by the scripts -### The Hermetic Build's well-known folder -Located in `${HOME}/.library_generation`, this folder is assumed by the scripts -to contain certain tools. - -Developers must make sure this folder is properly configured before running the -scripts locally. -Note that this relies on the `HOME` en var which is always defined as per -[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). - -#### Put the gapic-generator-java jar in its well-known location - -Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip --Dcheckstyle.skip`. -This will generate a jar located in -`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` - -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" -``` - -#### Put the java formatter jar in its well-known location - -Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) -or [GitHub releases](https://github.com/google/google-java-format/releases). -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" -``` - -## Installing prerequisites - -In order to run the generation scripts directly, there are a few tools we -need to install beforehand. - -### Install the owl-bot CLI - -Requires node.js to be installed. -Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) -for NVM, Node.js's version manager. - -After you install it, you can install the owl-bot CLI with the following -commands: -```bash -git clone https://github.com/googleapis/repo-automation-bots -cd repo-automation-bots/packages/owl-bot -npm i && npm run compile && npm link -owl-bot copy-code --version -``` - -The key step is `npm link`, which will make the command available in you current -shell session. - - -## Running the script -The entrypoint script (`library_generation/cli/entry_point.py`) allows you to -generate a GAPIC repository with a given api definition (proto, service yaml). - -### Download the api definition -For example, googleapis -``` -git clone https://github.com/googleapis/googleapis -export api_definitions_path="$(pwd)/googleapis" -``` - -### Download the repo -For example, google-cloud-java -``` -git clone https://github.com/googleapis/google-cloud-java -export path_to_repo="$(pwd)/google-cloud-java" -``` - -### Install the scripts -``` -python -m pip install . -``` - -### Run the script -``` -python library_generation/cli/entry_point.py generate \ - --repository-path="${path_to_repo}" \ - --api-definitions-path="${api_definitions_path}" -``` - - -# Running the scripts using the docker container image -This is convenient in order to avoid installing the dependencies manually. - -> [!IMPORTANT] -> From now, the examples assume you are in the root of your sdk-platform-java -> folder. - -## Build the docker image -```bash -docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id . -``` - -This will create an `image-id` file at the root of the repo with the hash ID of -the image. - -## Run the docker image -The docker image will perform changes on its internal `/workspace` folder, -to which you need to map a folder on your host machine (i.e. map your downloaded -repo to this folder). - -To run the docker container on the google-cloud-java repo, you must run: -```bash -docker run \ - -u "$(id -u)":"$(id -g)" \ - -v /path/to/google-cloud-java:/workspace \ - -v /path/to/api-definition:/workspace/apis \ - $(cat image-id) \ - --api-definitions-path=/workspace/apis -``` - - * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating - yourself. This avoids folder ownership changes since it runs as root by - default. - * `-v /path/to/google-cloud-java:/workspace` maps the host machine's - google-cloud-java folder to the /workspace folder. - The image is configured to perform changes in this directory. - * `-v /path/to/api-definition:/workspace/apis` maps the host machine's - api-definition folder to /workspace/apis folder. - * `$(cat image-id)` obtains the image ID created in the build step. - * `--api-definitions-path=/workspace/apis` set the API definition path to - `/workspace/apis`. - -## Debug the created containers -If you are working on changing the way the containers are created, you may want -to inspect the containers to check the setup. -It would be convenient in such case to have a text editor/viewer available. -You can achieve this by modifying the Dockerfile as follows: - -```docker -# install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq less vim \ - && apt-get clean -``` - -We add `less` and `vim` as text tools for further inspection. - -You can also run a shell in a new container by running: - -```bash -docker run \ - --rm -it \ - -u $(id -u):$(id -g) \ - -v /path/to/google-cloud-java:/workspace \ - --entrypoint="bash" \ - $(cat image-id) -``` diff --git a/hermetic_build/library_generation/cli/entry_point.py b/hermetic_build/library_generation/cli/entry_point.py index b15880f06d..e568f831ab 100644 --- a/hermetic_build/library_generation/cli/entry_point.py +++ b/hermetic_build/library_generation/cli/entry_point.py @@ -16,9 +16,7 @@ from typing import Optional import click as click from library_generation.generate_repo import generate_from_yaml -from library_generation.model.config_change import ConfigChange -from common.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config +from common.model.generation_config import from_yaml, GenerationConfig @click.group(invoke_without_command=False) @@ -30,18 +28,7 @@ def main(ctx): @main.command() @click.option( - "--baseline-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for commit history generation, not library - generation. - """, -) -@click.option( - "--current-generation-config-path", + "--generation-config-path", required=False, default=None, type=str, @@ -85,8 +72,7 @@ def main(ctx): """, ) def generate( - baseline_generation_config_path: str, - current_generation_config_path: str, + generation_config_path: Optional[str], library_names: Optional[str], repository_path: str, api_definitions_path: str, @@ -115,103 +101,63 @@ def generate( Raise FileNotFoundError if the default config does not exist. """ - __generate_repo_and_pr_description_impl( - baseline_generation_config_path=baseline_generation_config_path, - current_generation_config_path=current_generation_config_path, + __generate_repo_impl( + generation_config_path=generation_config_path, library_names=library_names, repository_path=repository_path, api_definitions_path=api_definitions_path, ) -def __generate_repo_and_pr_description_impl( - baseline_generation_config_path: str, - current_generation_config_path: str, +def __generate_repo_impl( + generation_config_path: Optional[str], library_names: Optional[str], repository_path: str, api_definitions_path: str, ): """ Implementation method for generate(). - The decoupling of generate and __generate_repo_and_pr_description_impl is + The decoupling of generate and __generate_repo_impl is meant to allow testing of this implementation function. """ default_generation_config_path = f"{os.getcwd()}/generation_config.yaml" - - if ( - baseline_generation_config_path is None - and current_generation_config_path is None - ): - if not os.path.isfile(default_generation_config_path): - raise FileNotFoundError( - f"{default_generation_config_path} does not exist. " - "A valid generation config has to be passed in as " - "current_generation_config or exist in the current working " - "directory." - ) - current_generation_config_path = default_generation_config_path - elif current_generation_config_path is None: + if generation_config_path is None: + generation_config_path = default_generation_config_path + generation_config_path = os.path.abspath(generation_config_path) + if not os.path.isfile(generation_config_path): raise FileNotFoundError( - "current_generation_config is not specified when " - "baseline_generation_config is specified. " - "current_generation_config should be the source of truth of " - "library generation." + f"Generation config {generation_config_path} does not exist." ) - - current_generation_config_path = os.path.abspath(current_generation_config_path) repository_path = os.path.abspath(repository_path) api_definitions_path = os.path.abspath(api_definitions_path) - include_library_names = _parse_library_name_from(library_names) - - if not baseline_generation_config_path: - # Execute selective generation based on current_generation_config if - # baseline_generation_config is not specified. - generate_from_yaml( - config=from_yaml(current_generation_config_path), - repository_path=repository_path, - api_definitions_path=api_definitions_path, - target_library_names=include_library_names, - ) - return - - # Compare two generation configs to get changed libraries. - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), - ) - # Pass None if we want to fully generate the repository. - changed_library_names = ( - config_change.get_changed_libraries() - if not _needs_full_repo_generation(config_change=config_change) - else None - ) - # Include library names takes preference if specified. - target_library_names = ( - include_library_names - if include_library_names is not None - else changed_library_names + generation_config = from_yaml(generation_config_path) + include_library_names = _parse_library_name_from( + includes=library_names, generation_config=generation_config ) generate_from_yaml( - config=config_change.current_config, + config=generation_config, repository_path=repository_path, api_definitions_path=api_definitions_path, - target_library_names=target_library_names, + target_library_names=include_library_names, ) -def _needs_full_repo_generation(config_change: ConfigChange) -> bool: +def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool: """ Whether you should need a full repo generation, i.e., generate all libraries in the generation configuration. """ - current_config = config_change.current_config - return not current_config.is_monorepo() or current_config.contains_common_protos() + return ( + not generation_config.is_monorepo() + or generation_config.contains_common_protos() + ) -def _parse_library_name_from(includes: str) -> Optional[list[str]]: - if includes is None: +def _parse_library_name_from( + includes: Optional[str], generation_config: GenerationConfig +) -> Optional[list[str]]: + if includes is None or _needs_full_repo_generation(generation_config): return None return [library_name.strip() for library_name in includes.split(",")] diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index b8d22aca54..f5cae1ba5a 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -115,11 +115,11 @@ if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -temp_destination_path="${output_folder}/temp_preprocessed" +temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run - rm -rd "${temp_destination_path}" + rm -r "${temp_destination_path}" fi mkdir -p "${temp_destination_path}" ##################### Section 0 ##################### @@ -274,5 +274,5 @@ rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_pro popd # destination path cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" -rm -rdf "${temp_destination_path}" +rm -rf "${temp_destination_path}" exit 0 diff --git a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh index 3d38677678..3118f32a2b 100755 --- a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh +++ b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -36,7 +36,7 @@ library_version=$5 if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp - rm -rd owl-bot-staging/ + rm -rf owl-bot-staging/ mv temp owl-bot-staging fi diff --git a/hermetic_build/library_generation/owlbot/bin/format_source.sh b/hermetic_build/library_generation/owlbot/bin/format_source.sh index 9402be778b..efc51b8940 100755 --- a/hermetic_build/library_generation/owlbot/bin/format_source.sh +++ b/hermetic_build/library_generation/owlbot/bin/format_source.sh @@ -36,6 +36,9 @@ do elif [[ $file =~ .*/samples/snippets/src/.*/java/com/example/spanner/.*.java ]]; then echo "File skipped formatting: $file" + elif [[ $file =~ .*/test-jdk17/java/com/google/cloud/firestore/.*java ]]; + then + echo "File skipped formatting: $file" else echo $file >> $tmp_file fi diff --git a/hermetic_build/library_generation/requirements.in b/hermetic_build/library_generation/requirements.in index 4f8ad9709b..7ab992ea62 100644 --- a/hermetic_build/library_generation/requirements.in +++ b/hermetic_build/library_generation/requirements.in @@ -1,6 +1,5 @@ attrs==24.2.0 click==8.1.7 -GitPython==3.1.43 jinja2==3.1.4 lxml==5.3.0 PyYAML==6.0.2 diff --git a/hermetic_build/library_generation/requirements.txt b/hermetic_build/library_generation/requirements.txt index 87ac0ba921..ef3d97bedc 100644 --- a/hermetic_build/library_generation/requirements.txt +++ b/hermetic_build/library_generation/requirements.txt @@ -123,14 +123,6 @@ click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via -r hermetic_build/library_generation/requirements.in -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via gitpython -gitpython==3.1.43 \ - --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ - --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff - # via -r hermetic_build/library_generation/requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 @@ -407,10 +399,6 @@ requests-mock==1.12.1 \ --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 # via -r hermetic_build/library_generation/requirements.in -smmap==5.0.1 \ - --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ - --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da - # via gitdb urllib3==2.2.3 \ --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 diff --git a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py index b7e7088cf9..82f6ec1c13 100644 --- a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py +++ b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py @@ -18,7 +18,7 @@ from library_generation.cli.entry_point import ( generate, validate_generation_config, - __generate_repo_and_pr_description_impl as generate_impl, + __generate_repo_impl as generate_impl, ) from common.model.generation_config import from_yaml @@ -27,34 +27,27 @@ class EntryPointTest(unittest.TestCase): - def test_entry_point_without_config_raise_file_exception(self): + def test_entry_point_without_default_config_raise_file_exception(self): os.chdir(script_dir) runner = CliRunner() # noinspection PyTypeChecker - result = runner.invoke(generate, ["--repository-path=."]) + result = runner.invoke(generate) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) self.assertRegex( result.exception.args[0], "generation_config.yaml does not exist." ) - def test_entry_point_with_baseline_without_current_raise_file_exception(self): + def test_entry_point_with_invalid_config_raise_file_exception(self): + os.chdir(script_dir) runner = CliRunner() # noinspection PyTypeChecker result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=path/to/config.yaml", - "--repository-path=.", - ], + generate, ["--generation-config-path=/non-existent/file"] ) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], - "current_generation_config is not specified when " - "baseline_generation_config is specified.", - ) + self.assertRegex(result.exception.args[0], "/non-existent/file does not exist.") def test_validate_generation_config_succeeds( self, @@ -86,7 +79,7 @@ def test_validate_generation_config_with_duplicate_library_name_raise_file_excep ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_triggers_full_generation( + def test_generate_non_monorepo_without_library_names_full_generation( self, generate_from_yaml, ): @@ -100,8 +93,7 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -114,7 +106,7 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( + def test_generate_non_monorepo_with_library_names_full_generation( self, generate_from_yaml, ): @@ -129,41 +121,8 @@ def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_ # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_triggers_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). generate() should call generate_from_yaml() - with target_library_names=None in order to trigger the full generation - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, + generation_config_path=config_path, + library_names="non-existent-library", repository_path=".", api_definitions_path=".", ) @@ -175,40 +134,7 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). - generate() should call generate_from_yaml() with - target_library_names equals includes - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_triggers_full_generation( + def test_generate_monorepo_with_common_protos_without_library_names_triggers_full_generation( self, generate_from_yaml, ): @@ -223,8 +149,7 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -237,7 +162,7 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( + def test_generate_monorepo_with_common_protos_with_library_names_triggers_full_generation( self, generate_from_yaml, ): @@ -251,8 +176,7 @@ def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_g # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names="iam,non-existent-library", repository_path=".", api_definitions_path=".", @@ -261,11 +185,11 @@ def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_g config=ANY, repository_path=ANY, api_definitions_path=ANY, - target_library_names=["iam", "non-existent-library"], + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_does_not_trigger_generation( + def test_generate_monorepo_without_library_names_trigger_full_generation( self, generate_from_yaml, ): @@ -281,8 +205,7 @@ def test_generate_monorepo_without_change_does_not_trigger_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -291,11 +214,11 @@ def test_generate_monorepo_without_change_does_not_trigger_generation( config=ANY, repository_path=ANY, api_definitions_path=ANY, - target_library_names=[], + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( + def test_generate_monorepo_with_library_names_trigger_selective_generation( self, generate_from_yaml, ): @@ -311,8 +234,7 @@ def test_generate_monorepo_without_change_with_includes_trigger_selective_genera # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names="asset", repository_path=".", api_definitions_path=".", @@ -323,94 +245,3 @@ def test_generate_monorepo_without_change_with_includes_trigger_selective_genera api_definitions_path=ANY, target_library_names=["asset"], ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["asset"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the same as include libraries, regardless - the library exists or not. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudbuild,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudbuild", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=[], - ) diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.py b/hermetic_build/library_generation/tests/generate_library_unit_tests.py index 7bc14e3e20..eb2249908c 100644 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.py +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.py @@ -53,7 +53,7 @@ def setUp(self): bash_call(f"mkdir {self.output_folder}") def tearDown(self): - bash_call(f"rm -rdf {self.simulated_home}") + bash_call(f"rm -rf {self.simulated_home}") def _run_command(self, command, **kwargs): env = os.environ.copy() diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh index 639abd8677..a3d95c1be7 100755 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh @@ -138,7 +138,7 @@ download_tools_succeed_with_baked_protoc() { download_tools "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_PROTOC_LOCATION unset DOCKER_PROTOC_VERSION unset output_folder @@ -159,7 +159,7 @@ download_tools_succeed_with_baked_grpc() { download_tools "${test_protoc_version}" "99.99" "linux-x86_64" assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_GRPC_LOCATION unset DOCKER_GRPC_VERSION unset output_folder @@ -243,7 +243,7 @@ copy_directory_if_exists_valid_folder_succeeds() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 1 ${n_matching_folders} } @@ -253,7 +253,7 @@ copy_directory_if_exists_invalid_folder_does_not_copy() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 0 ${n_matching_folders} } diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index fd534ff207..a940069baa 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -43,7 +43,6 @@ "google-cloud-java": "chore/test-hermetic-build", "java-bigtable": "chore/test-hermetic-build", } -baseline_config_name = "baseline_generation_config.yaml" current_config_name = "current_generation_config.yaml" googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" # This variable is used to override the jar created by building the image @@ -88,8 +87,7 @@ def test_entry_point_running_in_container(self): self.__run_entry_point_in_docker_container( repo_location=repo_location, config_location=config_location, - baseline_config=baseline_config_name, - current_config=current_config_name, + generation_config=current_config_name, api_definition=api_definitions_path, ) # 4. compare generation result with golden files @@ -204,6 +202,7 @@ def __build_image(cls, docker_file: str, cwd: str): subprocess.check_call( ["docker", "build", "-f", docker_file, "-t", image_tag, "."], cwd=cwd, + env=dict(os.environ, DOCKER_BUILDKIT="1"), ) @classmethod @@ -222,6 +221,8 @@ def __download_generator_jar(cls, coordinates_file: str) -> None: [ "mvn", "dependency:copy", + "-B", + "-ntp", f"-Dartifact={coordinates}", f"-DoutputDirectory={config_dir}", ] @@ -287,8 +288,7 @@ def __run_entry_point_in_docker_container( cls, repo_location: str, config_location: str, - baseline_config: str, - current_config: str, + generation_config: str, api_definition: str, ): # we use the calling user to prevent the mapped volumes from changing @@ -314,8 +314,7 @@ def __run_entry_point_in_docker_container( "-w", "/workspace/repo", image_tag, - f"--baseline-generation-config-path=/workspace/config/{baseline_config}", - f"--current-generation-config-path=/workspace/config/{current_config}", + f"--generation-config-path=/workspace/config/{generation_config}", f"--api-definitions-path=/workspace/api", ], ) diff --git a/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py index 2e23c2b403..76b663c6cf 100644 --- a/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py +++ b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py @@ -15,10 +15,7 @@ import os import unittest from pathlib import Path -from library_generation.utils.proto_path_utils import ( - find_versioned_proto_path, - remove_version_from, -) +from library_generation.utils.proto_path_utils import remove_version_from script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "..", "resources") @@ -26,21 +23,6 @@ class ProtoPathsUtilsTest(unittest.TestCase): - def test_find_versioned_proto_path_nested_version_success(self): - proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" - expected = "google/cloud/aiplatform/v1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_path_success(self): - proto_path = "google/cloud/asset/v1p2beta1/assets.proto" - expected = "google/cloud/asset/v1p2beta1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_without_version_return_itself(self): - proto_path = "google/type/color.proto" - expected = "google/type/color.proto" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - def test_remove_version_from_returns_non_versioned_path(self): proto_path = "google/cloud/aiplatform/v1" self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/hermetic_build/library_generation/utils/proto_path_utils.py b/hermetic_build/library_generation/utils/proto_path_utils.py index d2ae25f602..27e3f8aa38 100644 --- a/hermetic_build/library_generation/utils/proto_path_utils.py +++ b/hermetic_build/library_generation/utils/proto_path_utils.py @@ -27,21 +27,3 @@ def remove_version_from(proto_path: str) -> str: if re.match(version_pattern, version): return proto_path[:index] return proto_path - - -def find_versioned_proto_path(proto_path: str) -> str: - """ - Returns a versioned proto_path from a given proto_path; or proto_path itself - if it doesn't contain a versioned proto_path. - :param proto_path: a proto file path - :return: the versioned proto_path - """ - version_regex = re.compile(r"^v[1-9].*") - directories = proto_path.split("/") - for directory in directories: - result = version_regex.search(directory) - if result: - version = result[0] - idx = proto_path.find(version) - return proto_path[:idx] + version - return proto_path diff --git a/hermetic_build/release_note_generation/cli/generate_release_note.py b/hermetic_build/release_note_generation/cli/generate_release_note.py index adc93d9ea7..a54e228f62 100644 --- a/hermetic_build/release_note_generation/cli/generate_release_note.py +++ b/hermetic_build/release_note_generation/cli/generate_release_note.py @@ -16,7 +16,7 @@ import click as click from release_note_generation.generate_pr_description import generate_pr_descriptions from common.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config +from common.utils.generation_config_comparator import compare_config @click.group(invoke_without_command=False) @@ -54,8 +54,9 @@ def main(ctx): default=".", show_default=True, help=""" - The repository path to which the generated files will be sent. - If not specified, the repository will be generated to the current working + The path where the file `pr_description.txt`, which contains the release + notes, will be sent. + If not specified, the file will be generated to the current working directory. """, ) diff --git a/hermetic_build/release_note_generation/commit_message_formatter.py b/hermetic_build/release_note_generation/commit_message_formatter.py index 48c060c571..c86c1df192 100644 --- a/hermetic_build/release_note_generation/commit_message_formatter.py +++ b/hermetic_build/release_note_generation/commit_message_formatter.py @@ -14,7 +14,7 @@ import re from git import Commit -from library_generation.model.config_change import ConfigChange, ChangeType +from common.model.config_change import ConfigChange, ChangeType from common.model.generation_config import ( GAPIC_GENERATOR_VERSION, LIBRARIES_BOM_VERSION, diff --git a/hermetic_build/release_note_generation/generate_pr_description.py b/hermetic_build/release_note_generation/generate_pr_description.py index b71facd7a4..d544268092 100755 --- a/hermetic_build/release_note_generation/generate_pr_description.py +++ b/hermetic_build/release_note_generation/generate_pr_description.py @@ -18,8 +18,8 @@ from typing import Dict from git import Commit, Repo -from library_generation.model.config_change import ConfigChange -from library_generation.utils.proto_path_utils import find_versioned_proto_path +from common.model.config_change import ConfigChange +from common.utils.proto_path_utils import find_versioned_proto_path from release_note_generation.commit_message_formatter import ( format_commit_message, format_repo_level_change, diff --git a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py index ac888cea7b..b7891495ee 100644 --- a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest from unittest.mock import patch -from library_generation.model.config_change import ( +from common.model.config_change import ( ConfigChange, ChangeType, LibraryChange, diff --git a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py index 7c6f0db9ac..9659af5b2d 100644 --- a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py @@ -18,7 +18,7 @@ get_repo_level_commit_messages, generate_pr_descriptions, ) -from library_generation.model.config_change import ( +from common.model.config_change import ( ConfigChange, ChangeType, LibraryChange, diff --git a/java-common-protos/grpc-google-common-protos/pom.xml b/java-common-protos/grpc-google-common-protos/pom.xml index 7abb58d191..37479f71b4 100644 --- a/java-common-protos/grpc-google-common-protos/pom.xml +++ b/java-common-protos/grpc-google-common-protos/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.49.0 grpc-google-common-protos GRPC library for grpc-google-common-protos com.google.api.grpc google-common-protos-parent - 2.48.0 + 2.49.0 diff --git a/java-common-protos/pom.xml b/java-common-protos/pom.xml index d381fc9e6f..a9153be9e2 100644 --- a/java-common-protos/pom.xml +++ b/java-common-protos/pom.xml @@ -4,7 +4,7 @@ com.google.api.grpc google-common-protos-parent pom - 2.48.0 + 2.49.0 Google Common Protos Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.50.0 ../gapic-generator-java-pom-parent @@ -61,7 +61,7 @@ com.google.cloud third-party-dependencies - 3.39.0 + 3.40.0 pom import @@ -75,7 +75,7 @@ com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.49.0 io.grpc @@ -87,7 +87,7 @@ com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.49.0 com.google.guava diff --git a/java-common-protos/proto-google-common-protos/pom.xml b/java-common-protos/proto-google-common-protos/pom.xml index 3ce2807f44..998b861576 100644 --- a/java-common-protos/proto-google-common-protos/pom.xml +++ b/java-common-protos/proto-google-common-protos/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.49.0 proto-google-common-protos PROTO library for proto-google-common-protos com.google.api.grpc google-common-protos-parent - 2.48.0 + 2.49.0 diff --git a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/ClientProto.java b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/ClientProto.java index 3f33631b0f..86a8da410d 100644 --- a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/ClientProto.java +++ b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/ClientProto.java @@ -292,53 +292,54 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + "alue\030\002 \001(\t:\0028\001\"A\n\013CppSettings\0222\n\006common\030" + "\001 \001(\0132\".google.api.CommonLanguageSetting" + "s\"A\n\013PhpSettings\0222\n\006common\030\001 \001(\0132\".googl" - + "e.api.CommonLanguageSettings\"\313\001\n\016PythonS" + + "e.api.CommonLanguageSettings\"\364\001\n\016PythonS" + "ettings\0222\n\006common\030\001 \001(\0132\".google.api.Com" + "monLanguageSettings\022N\n\025experimental_feat" + "ures\030\002 \001(\0132/.google.api.PythonSettings.E" - + "xperimentalFeatures\0325\n\024ExperimentalFeatu" - + "res\022\035\n\025rest_async_io_enabled\030\001 \001(\010\"B\n\014No" - + "deSettings\0222\n\006common\030\001 \001(\0132\".google.api." - + "CommonLanguageSettings\"\252\003\n\016DotnetSetting" - + "s\0222\n\006common\030\001 \001(\0132\".google.api.CommonLan" - + "guageSettings\022I\n\020renamed_services\030\002 \003(\0132" - + "/.google.api.DotnetSettings.RenamedServi" - + "cesEntry\022K\n\021renamed_resources\030\003 \003(\01320.go" - + "ogle.api.DotnetSettings.RenamedResources" - + "Entry\022\031\n\021ignored_resources\030\004 \003(\t\022 \n\030forc" - + "ed_namespace_aliases\030\005 \003(\t\022\036\n\026handwritte" - + "n_signatures\030\006 \003(\t\0326\n\024RenamedServicesEnt" - + "ry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\0327\n\025Re" - + "namedResourcesEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005valu" - + "e\030\002 \001(\t:\0028\001\"B\n\014RubySettings\0222\n\006common\030\001 " - + "\001(\0132\".google.api.CommonLanguageSettings\"" - + "@\n\nGoSettings\0222\n\006common\030\001 \001(\0132\".google.a" - + "pi.CommonLanguageSettings\"\317\002\n\016MethodSett" - + "ings\022\020\n\010selector\030\001 \001(\t\022<\n\014long_running\030\002" - + " \001(\0132&.google.api.MethodSettings.LongRun" - + "ning\022\035\n\025auto_populated_fields\030\003 \003(\t\032\315\001\n\013" - + "LongRunning\0225\n\022initial_poll_delay\030\001 \001(\0132" - + "\031.google.protobuf.Duration\022\035\n\025poll_delay" - + "_multiplier\030\002 \001(\002\0221\n\016max_poll_delay\030\003 \001(" - + "\0132\031.google.protobuf.Duration\0225\n\022total_po" - + "ll_timeout\030\004 \001(\0132\031.google.protobuf.Durat" - + "ion\"+\n\030SelectiveGapicGeneration\022\017\n\007metho" - + "ds\030\001 \003(\t*\243\001\n\031ClientLibraryOrganization\022+" - + "\n\'CLIENT_LIBRARY_ORGANIZATION_UNSPECIFIE" - + "D\020\000\022\t\n\005CLOUD\020\001\022\007\n\003ADS\020\002\022\n\n\006PHOTOS\020\003\022\017\n\013S" - + "TREET_VIEW\020\004\022\014\n\010SHOPPING\020\005\022\007\n\003GEO\020\006\022\021\n\rG" - + "ENERATIVE_AI\020\007*g\n\030ClientLibraryDestinati" - + "on\022*\n&CLIENT_LIBRARY_DESTINATION_UNSPECI" - + "FIED\020\000\022\n\n\006GITHUB\020\n\022\023\n\017PACKAGE_MANAGER\020\024:" - + "9\n\020method_signature\022\036.google.protobuf.Me" - + "thodOptions\030\233\010 \003(\t:6\n\014default_host\022\037.goo" - + "gle.protobuf.ServiceOptions\030\231\010 \001(\t:6\n\014oa" - + "uth_scopes\022\037.google.protobuf.ServiceOpti" - + "ons\030\232\010 \001(\t:8\n\013api_version\022\037.google.proto" - + "buf.ServiceOptions\030\301\272\253\372\001 \001(\tBi\n\016com.goog" - + "le.apiB\013ClientProtoP\001ZAgoogle.golang.org" - + "/genproto/googleapis/api/annotations;ann" - + "otations\242\002\004GAPIb\006proto3" + + "xperimentalFeatures\032^\n\024ExperimentalFeatu" + + "res\022\035\n\025rest_async_io_enabled\030\001 \001(\010\022\'\n\037pr" + + "otobuf_pythonic_types_enabled\030\002 \001(\010\"B\n\014N" + + "odeSettings\0222\n\006common\030\001 \001(\0132\".google.api" + + ".CommonLanguageSettings\"\252\003\n\016DotnetSettin" + + "gs\0222\n\006common\030\001 \001(\0132\".google.api.CommonLa" + + "nguageSettings\022I\n\020renamed_services\030\002 \003(\013" + + "2/.google.api.DotnetSettings.RenamedServ" + + "icesEntry\022K\n\021renamed_resources\030\003 \003(\01320.g" + + "oogle.api.DotnetSettings.RenamedResource" + + "sEntry\022\031\n\021ignored_resources\030\004 \003(\t\022 \n\030for" + + "ced_namespace_aliases\030\005 \003(\t\022\036\n\026handwritt" + + "en_signatures\030\006 \003(\t\0326\n\024RenamedServicesEn" + + "try\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\0327\n\025R" + + "enamedResourcesEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005val" + + "ue\030\002 \001(\t:\0028\001\"B\n\014RubySettings\0222\n\006common\030\001" + + " \001(\0132\".google.api.CommonLanguageSettings" + + "\"@\n\nGoSettings\0222\n\006common\030\001 \001(\0132\".google." + + "api.CommonLanguageSettings\"\317\002\n\016MethodSet" + + "tings\022\020\n\010selector\030\001 \001(\t\022<\n\014long_running\030" + + "\002 \001(\0132&.google.api.MethodSettings.LongRu" + + "nning\022\035\n\025auto_populated_fields\030\003 \003(\t\032\315\001\n" + + "\013LongRunning\0225\n\022initial_poll_delay\030\001 \001(\013" + + "2\031.google.protobuf.Duration\022\035\n\025poll_dela" + + "y_multiplier\030\002 \001(\002\0221\n\016max_poll_delay\030\003 \001" + + "(\0132\031.google.protobuf.Duration\0225\n\022total_p" + + "oll_timeout\030\004 \001(\0132\031.google.protobuf.Dura" + + "tion\"+\n\030SelectiveGapicGeneration\022\017\n\007meth" + + "ods\030\001 \003(\t*\243\001\n\031ClientLibraryOrganization\022" + + "+\n\'CLIENT_LIBRARY_ORGANIZATION_UNSPECIFI" + + "ED\020\000\022\t\n\005CLOUD\020\001\022\007\n\003ADS\020\002\022\n\n\006PHOTOS\020\003\022\017\n\013" + + "STREET_VIEW\020\004\022\014\n\010SHOPPING\020\005\022\007\n\003GEO\020\006\022\021\n\r" + + "GENERATIVE_AI\020\007*g\n\030ClientLibraryDestinat" + + "ion\022*\n&CLIENT_LIBRARY_DESTINATION_UNSPEC" + + "IFIED\020\000\022\n\n\006GITHUB\020\n\022\023\n\017PACKAGE_MANAGER\020\024" + + ":9\n\020method_signature\022\036.google.protobuf.M" + + "ethodOptions\030\233\010 \003(\t:6\n\014default_host\022\037.go" + + "ogle.protobuf.ServiceOptions\030\231\010 \001(\t:6\n\014o" + + "auth_scopes\022\037.google.protobuf.ServiceOpt" + + "ions\030\232\010 \001(\t:8\n\013api_version\022\037.google.prot" + + "obuf.ServiceOptions\030\301\272\253\372\001 \001(\tBi\n\016com.goo" + + "gle.apiB\013ClientProtoP\001ZAgoogle.golang.or" + + "g/genproto/googleapis/api/annotations;an" + + "notations\242\002\004GAPIb\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( @@ -433,7 +434,7 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_google_api_PythonSettings_ExperimentalFeatures_descriptor, new java.lang.String[] { - "RestAsyncIoEnabled", + "RestAsyncIoEnabled", "ProtobufPythonicTypesEnabled", }); internal_static_google_api_NodeSettings_descriptor = getDescriptor().getMessageTypes().get(7); internal_static_google_api_NodeSettings_fieldAccessorTable = diff --git a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/HttpBodyProto.java b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/HttpBodyProto.java index 97e6daa43f..c43d36461d 100644 --- a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/HttpBodyProto.java +++ b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/HttpBodyProto.java @@ -44,10 +44,10 @@ public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { "\n\031google/api/httpbody.proto\022\ngoogle.api\032" + "\031google/protobuf/any.proto\"X\n\010HttpBody\022\024" + "\n\014content_type\030\001 \001(\t\022\014\n\004data\030\002 \001(\014\022(\n\nex" - + "tensions\030\003 \003(\0132\024.google.protobuf.AnyBh\n\016" + + "tensions\030\003 \003(\0132\024.google.protobuf.AnyBe\n\016" + "com.google.apiB\rHttpBodyProtoP\001Z;google." + "golang.org/genproto/googleapis/api/httpb" - + "ody;httpbody\370\001\001\242\002\004GAPIb\006proto3" + + "ody;httpbody\242\002\004GAPIb\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( diff --git a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/PythonSettings.java b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/PythonSettings.java index bda54c7919..2f84ad752c 100644 --- a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/PythonSettings.java +++ b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/PythonSettings.java @@ -78,6 +78,22 @@ public interface ExperimentalFeaturesOrBuilder * @return The restAsyncIoEnabled. */ boolean getRestAsyncIoEnabled(); + + /** + * + * + *
+     * Enables generation of protobuf code using new types that are more
+     * Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+     * enabled by default 1 month after launching the feature in preview
+     * packages.
+     * 
+ * + * bool protobuf_pythonic_types_enabled = 2; + * + * @return The protobufPythonicTypesEnabled. + */ + boolean getProtobufPythonicTypesEnabled(); } /** * @@ -144,6 +160,27 @@ public boolean getRestAsyncIoEnabled() { return restAsyncIoEnabled_; } + public static final int PROTOBUF_PYTHONIC_TYPES_ENABLED_FIELD_NUMBER = 2; + private boolean protobufPythonicTypesEnabled_ = false; + /** + * + * + *
+     * Enables generation of protobuf code using new types that are more
+     * Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+     * enabled by default 1 month after launching the feature in preview
+     * packages.
+     * 
+ * + * bool protobuf_pythonic_types_enabled = 2; + * + * @return The protobufPythonicTypesEnabled. + */ + @java.lang.Override + public boolean getProtobufPythonicTypesEnabled() { + return protobufPythonicTypesEnabled_; + } + private byte memoizedIsInitialized = -1; @java.lang.Override @@ -161,6 +198,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io if (restAsyncIoEnabled_ != false) { output.writeBool(1, restAsyncIoEnabled_); } + if (protobufPythonicTypesEnabled_ != false) { + output.writeBool(2, protobufPythonicTypesEnabled_); + } getUnknownFields().writeTo(output); } @@ -173,6 +213,10 @@ public int getSerializedSize() { if (restAsyncIoEnabled_ != false) { size += com.google.protobuf.CodedOutputStream.computeBoolSize(1, restAsyncIoEnabled_); } + if (protobufPythonicTypesEnabled_ != false) { + size += + com.google.protobuf.CodedOutputStream.computeBoolSize(2, protobufPythonicTypesEnabled_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -190,6 +234,8 @@ public boolean equals(final java.lang.Object obj) { (com.google.api.PythonSettings.ExperimentalFeatures) obj; if (getRestAsyncIoEnabled() != other.getRestAsyncIoEnabled()) return false; + if (getProtobufPythonicTypesEnabled() != other.getProtobufPythonicTypesEnabled()) + return false; if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -203,6 +249,9 @@ public int hashCode() { hash = (19 * hash) + getDescriptor().hashCode(); hash = (37 * hash) + REST_ASYNC_IO_ENABLED_FIELD_NUMBER; hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getRestAsyncIoEnabled()); + hash = (37 * hash) + PROTOBUF_PYTHONIC_TYPES_ENABLED_FIELD_NUMBER; + hash = + (53 * hash) + com.google.protobuf.Internal.hashBoolean(getProtobufPythonicTypesEnabled()); hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -347,6 +396,7 @@ public Builder clear() { super.clear(); bitField0_ = 0; restAsyncIoEnabled_ = false; + protobufPythonicTypesEnabled_ = false; return this; } @@ -386,6 +436,9 @@ private void buildPartial0(com.google.api.PythonSettings.ExperimentalFeatures re if (((from_bitField0_ & 0x00000001) != 0)) { result.restAsyncIoEnabled_ = restAsyncIoEnabled_; } + if (((from_bitField0_ & 0x00000002) != 0)) { + result.protobufPythonicTypesEnabled_ = protobufPythonicTypesEnabled_; + } } @java.lang.Override @@ -439,6 +492,9 @@ public Builder mergeFrom(com.google.api.PythonSettings.ExperimentalFeatures othe if (other.getRestAsyncIoEnabled() != false) { setRestAsyncIoEnabled(other.getRestAsyncIoEnabled()); } + if (other.getProtobufPythonicTypesEnabled() != false) { + setProtobufPythonicTypesEnabled(other.getProtobufPythonicTypesEnabled()); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -471,6 +527,12 @@ public Builder mergeFrom( bitField0_ |= 0x00000001; break; } // case 8 + case 16: + { + protobufPythonicTypesEnabled_ = input.readBool(); + bitField0_ |= 0x00000002; + break; + } // case 16 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { @@ -552,6 +614,68 @@ public Builder clearRestAsyncIoEnabled() { return this; } + private boolean protobufPythonicTypesEnabled_; + /** + * + * + *
+       * Enables generation of protobuf code using new types that are more
+       * Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+       * enabled by default 1 month after launching the feature in preview
+       * packages.
+       * 
+ * + * bool protobuf_pythonic_types_enabled = 2; + * + * @return The protobufPythonicTypesEnabled. + */ + @java.lang.Override + public boolean getProtobufPythonicTypesEnabled() { + return protobufPythonicTypesEnabled_; + } + /** + * + * + *
+       * Enables generation of protobuf code using new types that are more
+       * Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+       * enabled by default 1 month after launching the feature in preview
+       * packages.
+       * 
+ * + * bool protobuf_pythonic_types_enabled = 2; + * + * @param value The protobufPythonicTypesEnabled to set. + * @return This builder for chaining. + */ + public Builder setProtobufPythonicTypesEnabled(boolean value) { + + protobufPythonicTypesEnabled_ = value; + bitField0_ |= 0x00000002; + onChanged(); + return this; + } + /** + * + * + *
+       * Enables generation of protobuf code using new types that are more
+       * Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+       * enabled by default 1 month after launching the feature in preview
+       * packages.
+       * 
+ * + * bool protobuf_pythonic_types_enabled = 2; + * + * @return This builder for chaining. + */ + public Builder clearProtobufPythonicTypesEnabled() { + bitField0_ = (bitField0_ & ~0x00000002); + protobufPythonicTypesEnabled_ = false; + onChanged(); + return this; + } + @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { diff --git a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimit.java b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimit.java index 6bbe88f418..3a7b46b386 100644 --- a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimit.java +++ b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimit.java @@ -392,8 +392,8 @@ public com.google.protobuf.ByteString getMetricBytes() { * *
    * Specify the unit of the quota limit. It uses the same syntax as
-   * [Metric.unit][]. The supported unit kinds are determined by the quota
-   * backend system.
+   * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+   * unit kinds are determined by the quota backend system.
    *
    * Here are some examples:
    * * "1/min/{project}" for quota per minute per project.
@@ -423,8 +423,8 @@ public java.lang.String getUnit() {
    *
    * 
    * Specify the unit of the quota limit. It uses the same syntax as
-   * [Metric.unit][]. The supported unit kinds are determined by the quota
-   * backend system.
+   * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+   * unit kinds are determined by the quota backend system.
    *
    * Here are some examples:
    * * "1/min/{project}" for quota per minute per project.
@@ -1909,8 +1909,8 @@ public Builder setMetricBytes(com.google.protobuf.ByteString value) {
      *
      * 
      * Specify the unit of the quota limit. It uses the same syntax as
-     * [Metric.unit][]. The supported unit kinds are determined by the quota
-     * backend system.
+     * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+     * unit kinds are determined by the quota backend system.
      *
      * Here are some examples:
      * * "1/min/{project}" for quota per minute per project.
@@ -1939,8 +1939,8 @@ public java.lang.String getUnit() {
      *
      * 
      * Specify the unit of the quota limit. It uses the same syntax as
-     * [Metric.unit][]. The supported unit kinds are determined by the quota
-     * backend system.
+     * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+     * unit kinds are determined by the quota backend system.
      *
      * Here are some examples:
      * * "1/min/{project}" for quota per minute per project.
@@ -1969,8 +1969,8 @@ public com.google.protobuf.ByteString getUnitBytes() {
      *
      * 
      * Specify the unit of the quota limit. It uses the same syntax as
-     * [Metric.unit][]. The supported unit kinds are determined by the quota
-     * backend system.
+     * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+     * unit kinds are determined by the quota backend system.
      *
      * Here are some examples:
      * * "1/min/{project}" for quota per minute per project.
@@ -1998,8 +1998,8 @@ public Builder setUnit(java.lang.String value) {
      *
      * 
      * Specify the unit of the quota limit. It uses the same syntax as
-     * [Metric.unit][]. The supported unit kinds are determined by the quota
-     * backend system.
+     * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+     * unit kinds are determined by the quota backend system.
      *
      * Here are some examples:
      * * "1/min/{project}" for quota per minute per project.
@@ -2023,8 +2023,8 @@ public Builder clearUnit() {
      *
      * 
      * Specify the unit of the quota limit. It uses the same syntax as
-     * [Metric.unit][]. The supported unit kinds are determined by the quota
-     * backend system.
+     * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+     * unit kinds are determined by the quota backend system.
      *
      * Here are some examples:
      * * "1/min/{project}" for quota per minute per project.
diff --git a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimitOrBuilder.java b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimitOrBuilder.java
index c74a69f111..f420d609db 100644
--- a/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimitOrBuilder.java
+++ b/java-common-protos/proto-google-common-protos/src/main/java/com/google/api/QuotaLimitOrBuilder.java
@@ -214,8 +214,8 @@ public interface QuotaLimitOrBuilder
    *
    * 
    * Specify the unit of the quota limit. It uses the same syntax as
-   * [Metric.unit][]. The supported unit kinds are determined by the quota
-   * backend system.
+   * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+   * unit kinds are determined by the quota backend system.
    *
    * Here are some examples:
    * * "1/min/{project}" for quota per minute per project.
@@ -234,8 +234,8 @@ public interface QuotaLimitOrBuilder
    *
    * 
    * Specify the unit of the quota limit. It uses the same syntax as
-   * [Metric.unit][]. The supported unit kinds are determined by the quota
-   * backend system.
+   * [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+   * unit kinds are determined by the quota backend system.
    *
    * Here are some examples:
    * * "1/min/{project}" for quota per minute per project.
diff --git a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/client.proto b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/client.proto
index 7ba1db3da0..6003be5307 100644
--- a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/client.proto
+++ b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/client.proto
@@ -275,6 +275,12 @@ message PythonSettings {
     // This feature will be enabled by default 1 month after launching the
     // feature in preview packages.
     bool rest_async_io_enabled = 1;
+
+    // Enables generation of protobuf code using new types that are more
+    // Pythonic which are included in `protobuf>=5.29.x`. This feature will be
+    // enabled by default 1 month after launching the feature in preview
+    // packages.
+    bool protobuf_pythonic_types_enabled = 2;
   }
 
   // Some settings.
diff --git a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/httpbody.proto b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/httpbody.proto
index 920612dc72..32952715de 100644
--- a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/httpbody.proto
+++ b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/httpbody.proto
@@ -18,7 +18,6 @@ package google.api;
 
 import "google/protobuf/any.proto";
 
-option cc_enable_arenas = true;
 option go_package = "google.golang.org/genproto/googleapis/api/httpbody;httpbody";
 option java_multiple_files = true;
 option java_outer_classname = "HttpBodyProto";
diff --git a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/quota.proto b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/quota.proto
index eb80cb6f02..25adb344cd 100644
--- a/java-common-protos/proto-google-common-protos/src/main/proto/google/api/quota.proto
+++ b/java-common-protos/proto-google-common-protos/src/main/proto/google/api/quota.proto
@@ -161,8 +161,8 @@ message QuotaLimit {
   string metric = 8;
 
   // Specify the unit of the quota limit. It uses the same syntax as
-  // [Metric.unit][]. The supported unit kinds are determined by the quota
-  // backend system.
+  // [MetricDescriptor.unit][google.api.MetricDescriptor.unit]. The supported
+  // unit kinds are determined by the quota backend system.
   //
   // Here are some examples:
   // * "1/min/{project}" for quota per minute per project.
diff --git a/java-core/google-cloud-core-bom/pom.xml b/java-core/google-cloud-core-bom/pom.xml
index a29076962e..708fb0136c 100644
--- a/java-core/google-cloud-core-bom/pom.xml
+++ b/java-core/google-cloud-core-bom/pom.xml
@@ -3,13 +3,13 @@
   4.0.0
   com.google.cloud
   google-cloud-core-bom
-  2.47.0
+  2.48.0
   pom
 
   
     com.google.api
     gapic-generator-java-pom-parent
-    2.49.0
+    2.50.0
     ../../gapic-generator-java-pom-parent
   
 
@@ -23,17 +23,17 @@
       
         com.google.cloud
         google-cloud-core
-        2.47.0
+        2.48.0
       
       
         com.google.cloud
         google-cloud-core-grpc
-        2.47.0
+        2.48.0
       
       
         com.google.cloud
         google-cloud-core-http
-        2.47.0
+        2.48.0
       
     
   
diff --git a/java-core/google-cloud-core-grpc/pom.xml b/java-core/google-cloud-core-grpc/pom.xml
index 784f52e334..9d2477b1a4 100644
--- a/java-core/google-cloud-core-grpc/pom.xml
+++ b/java-core/google-cloud-core-grpc/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   com.google.cloud
   google-cloud-core-grpc
-  2.47.0
+  2.48.0
   jar
   Google Cloud Core gRPC
   
@@ -12,7 +12,7 @@
   
     com.google.cloud
     google-cloud-core-parent
-    2.47.0
+    2.48.0
   
   
     google-cloud-core-grpc
diff --git a/java-core/google-cloud-core-http/pom.xml b/java-core/google-cloud-core-http/pom.xml
index 753801f14a..91ff87efcf 100644
--- a/java-core/google-cloud-core-http/pom.xml
+++ b/java-core/google-cloud-core-http/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   com.google.cloud
   google-cloud-core-http
-  2.47.0
+  2.48.0
   jar
   Google Cloud Core HTTP
   
@@ -12,7 +12,7 @@
   
     com.google.cloud
     google-cloud-core-parent
-    2.47.0
+    2.48.0
   
   
     google-cloud-core-http
diff --git a/java-core/google-cloud-core/clirr-ignored-differences.xml b/java-core/google-cloud-core/clirr-ignored-differences.xml
index 0f7f80a7b4..6792bcb968 100644
--- a/java-core/google-cloud-core/clirr-ignored-differences.xml
+++ b/java-core/google-cloud-core/clirr-ignored-differences.xml
@@ -12,4 +12,10 @@
         com/google/cloud/ReadChannel
         long limit()
     
+    
+        
+        7012
+        com/google/cloud/testing/BaseEmulatorHelper$EmulatorRunner
+        int waitForDuration(java.time.Duration)
+    
 
diff --git a/java-core/google-cloud-core/pom.xml b/java-core/google-cloud-core/pom.xml
index fadc83f63c..bcdcb6bd45 100644
--- a/java-core/google-cloud-core/pom.xml
+++ b/java-core/google-cloud-core/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   com.google.cloud
   google-cloud-core
-  2.47.0
+  2.48.0
   jar
   Google Cloud Core
   
@@ -12,7 +12,7 @@
   
     com.google.cloud
     google-cloud-core-parent
-    2.47.0
+    2.48.0
   
   
     google-cloud-core
diff --git a/java-core/google-cloud-core/src/main/java/com/google/cloud/RetryOption.java b/java-core/google-cloud-core/src/main/java/com/google/cloud/RetryOption.java
index a1069b48a2..f69b0d1c76 100644
--- a/java-core/google-cloud-core/src/main/java/com/google/cloud/RetryOption.java
+++ b/java-core/google-cloud-core/src/main/java/com/google/cloud/RetryOption.java
@@ -16,12 +16,13 @@
 
 package com.google.cloud;
 
+import static com.google.api.gax.util.TimeConversionUtils.toJavaTimeDuration;
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import com.google.api.core.BetaApi;
+import com.google.api.core.ObsoleteApi;
 import com.google.api.gax.retrying.RetrySettings;
 import java.io.Serializable;
-import org.threeten.bp.Duration;
 
 /**
  * This class represents an options wrapper around the {@link RetrySettings} class and is an
@@ -51,13 +52,25 @@ private RetryOption(OptionType type, Object value) {
     this.value = checkNotNull(value);
   }
 
-  /** See {@link RetrySettings#getTotalTimeout()}. */
-  public static RetryOption totalTimeout(Duration totalTimeout) {
+  /** This method is obsolete. Use {@link #totalTimeoutDuration(java.time.Duration)} instead */
+  @ObsoleteApi("Use totalTimeouDuration() instead")
+  public static RetryOption totalTimeout(org.threeten.bp.Duration totalTimeout) {
+    return totalTimeoutDuration(toJavaTimeDuration(totalTimeout));
+  }
+
+  /** See {@link RetrySettings#getTotalTimeoutDuration()}. */
+  public static RetryOption totalTimeoutDuration(java.time.Duration totalTimeout) {
     return new RetryOption(OptionType.TOTAL_TIMEOUT, totalTimeout);
   }
 
-  /** See {@link RetrySettings#getInitialRetryDelay()}. */
-  public static RetryOption initialRetryDelay(Duration initialRetryDelay) {
+  /** This method is obsolete. Use {@link #initialRetryDelayDuration(java.time.Duration)} instead */
+  @ObsoleteApi("Use initialRetryDelayDuration() instead")
+  public static RetryOption initialRetryDelay(org.threeten.bp.Duration initialRetryDelay) {
+    return initialRetryDelayDuration(toJavaTimeDuration(initialRetryDelay));
+  }
+
+  /** See {@link RetrySettings#getInitialRetryDelayDuration()}. */
+  public static RetryOption initialRetryDelayDuration(java.time.Duration initialRetryDelay) {
     return new RetryOption(OptionType.INITIAL_RETRY_DELAY, initialRetryDelay);
   }
 
@@ -66,8 +79,14 @@ public static RetryOption retryDelayMultiplier(double retryDelayMultiplier) {
     return new RetryOption(OptionType.RETRY_DELAY_MULTIPLIER, retryDelayMultiplier);
   }
 
-  /** See {@link RetrySettings#getMaxRetryDelay()}. */
-  public static RetryOption maxRetryDelay(Duration maxRetryDelay) {
+  /** This method is obsolete. Use {@link #maxRetryDelayDuration(java.time.Duration)} instead */
+  @ObsoleteApi("Use maxRetryDelayDuration() instead")
+  public static RetryOption maxRetryDelay(org.threeten.bp.Duration maxRetryDelay) {
+    return maxRetryDelayDuration(toJavaTimeDuration(maxRetryDelay));
+  }
+
+  /** See {@link RetrySettings#getMaxRetryDelayDuration()}. */
+  public static RetryOption maxRetryDelayDuration(java.time.Duration maxRetryDelay) {
     return new RetryOption(OptionType.MAX_RETRY_DELAY, maxRetryDelay);
   }
 
@@ -124,16 +143,16 @@ public static RetrySettings mergeToSettings(RetrySettings settings, RetryOption.
     for (RetryOption option : options) {
       switch (option.type) {
         case TOTAL_TIMEOUT:
-          builder.setTotalTimeout((Duration) option.value);
+          builder.setTotalTimeoutDuration((java.time.Duration) option.value);
           break;
         case INITIAL_RETRY_DELAY:
-          builder.setInitialRetryDelay((Duration) option.value);
+          builder.setInitialRetryDelayDuration((java.time.Duration) option.value);
           break;
         case RETRY_DELAY_MULTIPLIER:
           builder.setRetryDelayMultiplier((Double) option.value);
           break;
         case MAX_RETRY_DELAY:
-          builder.setMaxRetryDelay((Duration) option.value);
+          builder.setMaxRetryDelayDuration((java.time.Duration) option.value);
           break;
         case MAX_ATTEMPTS:
           builder.setMaxAttempts((Integer) option.value);
diff --git a/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java b/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java
index 985fac4804..92aaa9d6a9 100644
--- a/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java
+++ b/java-core/google-cloud-core/src/main/java/com/google/cloud/ServiceOptions.java
@@ -63,6 +63,7 @@
 import java.io.Serializable;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -70,7 +71,6 @@
 import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
-import org.threeten.bp.Duration;
 
 /**
  * Abstract class representing service options.
@@ -787,13 +787,13 @@ public static RetrySettings getNoRetrySettings() {
   private static RetrySettings.Builder getDefaultRetrySettingsBuilder() {
     return RetrySettings.newBuilder()
         .setMaxAttempts(6)
-        .setInitialRetryDelay(Duration.ofMillis(1000L))
-        .setMaxRetryDelay(Duration.ofMillis(32_000L))
+        .setInitialRetryDelayDuration(Duration.ofMillis(1000L))
+        .setMaxRetryDelayDuration(Duration.ofMillis(32_000L))
         .setRetryDelayMultiplier(2.0)
-        .setTotalTimeout(Duration.ofMillis(50_000L))
-        .setInitialRpcTimeout(Duration.ofMillis(50_000L))
+        .setTotalTimeoutDuration(Duration.ofMillis(50_000L))
+        .setInitialRpcTimeoutDuration(Duration.ofMillis(50_000L))
         .setRpcTimeoutMultiplier(1.0)
-        .setMaxRpcTimeout(Duration.ofMillis(50_000L));
+        .setMaxRpcTimeoutDuration(Duration.ofMillis(50_000L));
   }
 
   protected abstract Set getScopes();
diff --git a/java-core/google-cloud-core/src/main/java/com/google/cloud/Timestamp.java b/java-core/google-cloud-core/src/main/java/com/google/cloud/Timestamp.java
index e0308c3836..d24cb2a37e 100644
--- a/java-core/google-cloud-core/src/main/java/com/google/cloud/Timestamp.java
+++ b/java-core/google-cloud-core/src/main/java/com/google/cloud/Timestamp.java
@@ -18,18 +18,19 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 
+import com.google.api.core.ObsoleteApi;
 import com.google.protobuf.util.Timestamps;
 import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
 import java.util.Date;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import org.threeten.bp.Instant;
-import org.threeten.bp.LocalDateTime;
-import org.threeten.bp.ZoneOffset;
-import org.threeten.bp.format.DateTimeFormatter;
-import org.threeten.bp.format.DateTimeFormatterBuilder;
-import org.threeten.bp.format.DateTimeParseException;
-import org.threeten.bp.temporal.TemporalAccessor;
 
 /**
  * Represents a timestamp with nanosecond precision. Timestamps cover the range [0001-01-01,
@@ -54,7 +55,7 @@ public final class Timestamp implements Comparable, Serializable {
       new DateTimeFormatterBuilder()
           .appendOptional(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
           .optionalStart()
-          .appendOffsetId()
+          .appendZoneOrOffsetId()
           .optionalEnd()
           .toFormatter()
           .withZone(ZoneOffset.UTC);
@@ -189,6 +190,17 @@ public com.google.protobuf.Timestamp toProto() {
     return com.google.protobuf.Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build();
   }
 
+  /** This method is obsolete. Use {@link #parseTimestampDuration(String)} instead */
+  @ObsoleteApi("Use parseTimestampDuration(String) instead")
+  public static Timestamp parseTimestamp(String timestamp) {
+    try {
+      return parseTimestampDuration(timestamp);
+    } catch (DateTimeParseException ex) {
+      throw new org.threeten.bp.format.DateTimeParseException(
+          ex.getMessage(), ex.getParsedString(), ex.getErrorIndex());
+    }
+  }
+
   /**
    * Creates a Timestamp instance from the given string. Input string should be in the RFC 3339
    * format, like '2020-12-01T10:15:30.000Z' or with the timezone offset, such as
@@ -198,7 +210,7 @@ public com.google.protobuf.Timestamp toProto() {
    * @return created Timestamp
    * @throws DateTimeParseException if unable to parse
    */
-  public static Timestamp parseTimestamp(String timestamp) {
+  public static Timestamp parseTimestampDuration(String timestamp) {
     TemporalAccessor temporalAccessor = timestampParser.parse(timestamp);
     Instant instant = Instant.from(temporalAccessor);
     return ofTimeSecondsAndNanos(instant.getEpochSecond(), instant.getNano());
diff --git a/java-core/google-cloud-core/src/main/java/com/google/cloud/testing/BaseEmulatorHelper.java b/java-core/google-cloud-core/src/main/java/com/google/cloud/testing/BaseEmulatorHelper.java
index 9679c6299c..93f7ea0f59 100644
--- a/java-core/google-cloud-core/src/main/java/com/google/cloud/testing/BaseEmulatorHelper.java
+++ b/java-core/google-cloud-core/src/main/java/com/google/cloud/testing/BaseEmulatorHelper.java
@@ -16,8 +16,12 @@
 
 package com.google.cloud.testing;
 
+import static com.google.api.gax.util.TimeConversionUtils.toJavaTimeDuration;
+import static com.google.api.gax.util.TimeConversionUtils.toThreetenDuration;
+
 import com.google.api.core.CurrentMillisClock;
 import com.google.api.core.InternalApi;
+import com.google.api.core.ObsoleteApi;
 import com.google.cloud.ExceptionHandler;
 import com.google.cloud.RetryHelper;
 import com.google.cloud.ServiceOptions;
@@ -56,7 +60,6 @@
 import java.util.logging.Logger;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
-import org.threeten.bp.Duration;
 
 /** Utility class to start and stop a local service which is used by unit testing. */
 @InternalApi
@@ -112,14 +115,21 @@ protected final void startProcess(String blockUntilOutput)
     }
   }
 
+  /** This method is obsolete. Use {@link #waitForProcessDuration(java.time.Duration)} instead */
+  @ObsoleteApi("Use waitForProcessDuration(java.time.Duration) instead")
+  protected final int waitForProcess(org.threeten.bp.Duration timeout)
+      throws IOException, InterruptedException, TimeoutException {
+    return waitForProcessDuration(toJavaTimeDuration(timeout));
+  }
+
   /**
    * Waits for the local service's subprocess to terminate, and stop any possible thread listening
    * for its output.
    */
-  protected final int waitForProcess(Duration timeout)
+  protected final int waitForProcessDuration(java.time.Duration timeout)
       throws IOException, InterruptedException, TimeoutException {
     if (activeRunner != null) {
-      int exitCode = activeRunner.waitFor(timeout);
+      int exitCode = activeRunner.waitForDuration(timeout);
       activeRunner = null;
       return exitCode;
     }
@@ -130,7 +140,7 @@ protected final int waitForProcess(Duration timeout)
     return 0;
   }
 
-  private static int waitForProcess(final Process process, Duration timeout)
+  private static int waitForProcess(final Process process, java.time.Duration timeout)
       throws InterruptedException, TimeoutException {
     if (process == null) {
       return 0;
@@ -180,10 +190,17 @@ public String getProjectId() {
   /** Starts the local emulator. */
   public abstract void start() throws IOException, InterruptedException;
 
-  /** Stops the local emulator. */
-  public abstract void stop(Duration timeout)
+  /** This method is obsolete. Use {@link #stopDuration(java.time.Duration)} instead */
+  @ObsoleteApi("Use stopDuration() instead")
+  public abstract void stop(org.threeten.bp.Duration timeout)
       throws IOException, InterruptedException, TimeoutException;
 
+  /** Stops the local emulator. */
+  public void stopDuration(java.time.Duration timeout)
+      throws IOException, InterruptedException, TimeoutException {
+    stop(toThreetenDuration(timeout));
+  }
+
   /** Resets the internal state of the emulator. */
   public abstract void reset() throws IOException;
 
@@ -226,8 +243,15 @@ protected interface EmulatorRunner {
     /** Starts the emulator associated to this runner. */
     void start() throws IOException;
 
+    /** This method is obsolete. Use {@link #waitForDuration(java.time.Duration)} instead */
+    @ObsoleteApi("Use waitForDuration() instead")
+    int waitFor(org.threeten.bp.Duration timeout) throws InterruptedException, TimeoutException;
+
     /** Wait for the emulator associated to this runner to terminate, returning the exit status. */
-    int waitFor(Duration timeout) throws InterruptedException, TimeoutException;
+    default int waitForDuration(java.time.Duration timeout)
+        throws InterruptedException, TimeoutException {
+      return waitFor(toThreetenDuration(timeout));
+    }
 
     /** Returns the process associated to the emulator, if any. */
     Process getProcess();
@@ -263,9 +287,17 @@ public void start() throws IOException {
       log.fine("Starting emulator via Google Cloud SDK");
       process = CommandWrapper.create().setCommand(commandText).setRedirectErrorStream().start();
     }
+    /** This method is obsolete. Use {@link #waitForDuration(java.time.Duration)} instead */
+    @ObsoleteApi("Use waitForDuration() instead")
+    @Override
+    public int waitFor(org.threeten.bp.Duration timeout)
+        throws InterruptedException, TimeoutException {
+      return waitForDuration(toJavaTimeDuration(timeout));
+    }
 
     @Override
-    public int waitFor(Duration timeout) throws InterruptedException, TimeoutException {
+    public int waitForDuration(java.time.Duration timeout)
+        throws InterruptedException, TimeoutException {
       return waitForProcess(process, timeout);
     }
 
@@ -374,8 +406,16 @@ public Path call() throws IOException {
               .start();
     }
 
+    /** This method is obsolete. Use {@link #waitForDuration(java.time.Duration)} instead */
+    @ObsoleteApi("Use waitForDuration() instead")
     @Override
-    public int waitFor(Duration timeout) throws InterruptedException, TimeoutException {
+    public int waitFor(org.threeten.bp.Duration timeout)
+        throws InterruptedException, TimeoutException {
+      return waitForDuration(toJavaTimeDuration(timeout));
+    }
+
+    public int waitForDuration(java.time.Duration timeout)
+        throws InterruptedException, TimeoutException {
       return waitForProcess(process, timeout);
     }
 
diff --git a/java-core/google-cloud-core/src/test/java/com/google/cloud/RetryOptionTest.java b/java-core/google-cloud-core/src/test/java/com/google/cloud/RetryOptionTest.java
index a458d31f67..192cc21f5f 100644
--- a/java-core/google-cloud-core/src/test/java/com/google/cloud/RetryOptionTest.java
+++ b/java-core/google-cloud-core/src/test/java/com/google/cloud/RetryOptionTest.java
@@ -20,27 +20,30 @@
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 
 import com.google.api.gax.retrying.RetrySettings;
+import java.time.Duration;
 import org.junit.jupiter.api.Test;
-import org.threeten.bp.Duration;
 
 class RetryOptionTest {
+  private static final long TOTAL_TIMEOUT_MILLIS = 420l;
+  private static final long INITIAL_RETRY_DELAY_MILLIS = 42l;
+  private static final long MAX_RETRY_DELAY_MILLIS = 100l;
 
   private static final RetryOption TOTAL_TIMEOUT =
-      RetryOption.totalTimeout(Duration.ofMillis(420L));
+      RetryOption.totalTimeoutDuration(Duration.ofMillis(TOTAL_TIMEOUT_MILLIS));
   private static final RetryOption INITIAL_RETRY_DELAY =
-      RetryOption.initialRetryDelay(Duration.ofMillis(42L));
+      RetryOption.initialRetryDelayDuration(Duration.ofMillis(INITIAL_RETRY_DELAY_MILLIS));
   private static final RetryOption RETRY_DELAY_MULTIPLIER = RetryOption.retryDelayMultiplier(1.5);
   private static final RetryOption MAX_RETRY_DELAY =
-      RetryOption.maxRetryDelay(Duration.ofMillis(100));
+      RetryOption.maxRetryDelayDuration(Duration.ofMillis(MAX_RETRY_DELAY_MILLIS));
   private static final RetryOption MAX_ATTEMPTS = RetryOption.maxAttempts(100);
   private static final RetryOption JITTERED = RetryOption.jittered(false);
 
   private static final RetrySettings retrySettings =
       RetrySettings.newBuilder()
-          .setTotalTimeout(Duration.ofMillis(420L))
-          .setInitialRetryDelay(Duration.ofMillis(42L))
+          .setTotalTimeoutDuration(Duration.ofMillis(420L))
+          .setInitialRetryDelayDuration(Duration.ofMillis(42L))
           .setRetryDelayMultiplier(1.5)
-          .setMaxRetryDelay(Duration.ofMillis(100))
+          .setMaxRetryDelayDuration(Duration.ofMillis(100))
           .setMaxAttempts(100)
           .setJittered(false)
           .build();
@@ -61,10 +64,10 @@ void testEqualsAndHashCode() {
     assertNotEquals(MAX_ATTEMPTS, MAX_RETRY_DELAY);
     assertNotEquals(JITTERED, MAX_ATTEMPTS);
 
-    RetryOption totalTimeout = RetryOption.totalTimeout(Duration.ofMillis(420L));
-    RetryOption initialRetryDelay = RetryOption.initialRetryDelay(Duration.ofMillis(42L));
+    RetryOption totalTimeout = RetryOption.totalTimeoutDuration(Duration.ofMillis(420L));
+    RetryOption initialRetryDelay = RetryOption.initialRetryDelayDuration(Duration.ofMillis(42L));
     RetryOption retryDelayMultiplier = RetryOption.retryDelayMultiplier(1.5);
-    RetryOption maxRetryDelay = RetryOption.maxRetryDelay(Duration.ofMillis(100));
+    RetryOption maxRetryDelay = RetryOption.maxRetryDelayDuration(Duration.ofMillis(100));
     RetryOption maxAttempts = RetryOption.maxAttempts(100);
     RetryOption jittered = RetryOption.jittered(false);
 
@@ -101,17 +104,17 @@ void testMergeToSettings() {
     assertEquals(retrySettings, mergedRetrySettings);
 
     defRetrySettings =
-        defRetrySettings.toBuilder().setTotalTimeout(Duration.ofMillis(420L)).build();
+        defRetrySettings.toBuilder().setTotalTimeoutDuration(Duration.ofMillis(420L)).build();
     mergedRetrySettings = RetryOption.mergeToSettings(defRetrySettings, TOTAL_TIMEOUT);
     assertEquals(defRetrySettings, mergedRetrySettings);
 
     defRetrySettings =
-        defRetrySettings.toBuilder().setMaxRetryDelay(Duration.ofMillis(100)).build();
+        defRetrySettings.toBuilder().setMaxRetryDelayDuration(Duration.ofMillis(100)).build();
     mergedRetrySettings = RetryOption.mergeToSettings(defRetrySettings, MAX_RETRY_DELAY);
     assertEquals(defRetrySettings, mergedRetrySettings);
 
     defRetrySettings =
-        defRetrySettings.toBuilder().setInitialRetryDelay(Duration.ofMillis(42L)).build();
+        defRetrySettings.toBuilder().setInitialRetryDelayDuration(Duration.ofMillis(42L)).build();
     mergedRetrySettings = RetryOption.mergeToSettings(defRetrySettings, INITIAL_RETRY_DELAY);
     assertEquals(defRetrySettings, mergedRetrySettings);
 
@@ -127,4 +130,20 @@ void testMergeToSettings() {
     mergedRetrySettings = RetryOption.mergeToSettings(defRetrySettings, JITTERED);
     assertEquals(defRetrySettings, mergedRetrySettings);
   }
+
+  @Test
+  public void threetenMethods_producesEquivalentJavaTimeRetryOptions() {
+
+    final RetryOption totalTimeoutThreeten =
+        RetryOption.totalTimeout(org.threeten.bp.Duration.ofMillis(TOTAL_TIMEOUT_MILLIS));
+    final RetryOption initialRetryDelayThreeten =
+        RetryOption.initialRetryDelay(
+            org.threeten.bp.Duration.ofMillis(INITIAL_RETRY_DELAY_MILLIS));
+    final RetryOption maxRetryDelayThreeten =
+        RetryOption.maxRetryDelay(org.threeten.bp.Duration.ofMillis(MAX_RETRY_DELAY_MILLIS));
+
+    assertEquals(TOTAL_TIMEOUT, totalTimeoutThreeten);
+    assertEquals(INITIAL_RETRY_DELAY, initialRetryDelayThreeten);
+    assertEquals(MAX_RETRY_DELAY, maxRetryDelayThreeten);
+  }
 }
diff --git a/java-core/google-cloud-core/src/test/java/com/google/cloud/SerializationTest.java b/java-core/google-cloud-core/src/test/java/com/google/cloud/SerializationTest.java
index 6c35c665b5..f591578f11 100644
--- a/java-core/google-cloud-core/src/test/java/com/google/cloud/SerializationTest.java
+++ b/java-core/google-cloud-core/src/test/java/com/google/cloud/SerializationTest.java
@@ -23,7 +23,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import java.io.Serializable;
-import org.threeten.bp.Duration;
+import java.time.Duration;
 
 public class SerializationTest extends BaseSerializationTest {
 
@@ -37,7 +37,7 @@ public class SerializationTest extends BaseSerializationTest {
   private static final Role SOME_ROLE = Role.viewer();
   private static final Policy SOME_IAM_POLICY = Policy.newBuilder().build();
   private static final RetryOption CHECKING_PERIOD =
-      RetryOption.initialRetryDelay(Duration.ofSeconds(42));
+      RetryOption.initialRetryDelayDuration(Duration.ofSeconds(42));
   private static final LabelDescriptor LABEL_DESCRIPTOR =
       new LabelDescriptor("project_id", ValueType.STRING, "The project id");
   private static final MonitoredResourceDescriptor MONITORED_RESOURCE_DESCRIPTOR =
diff --git a/java-core/google-cloud-core/src/test/java/com/google/cloud/TimestampTest.java b/java-core/google-cloud-core/src/test/java/com/google/cloud/TimestampTest.java
index ba2ad5b701..6d6340b417 100644
--- a/java-core/google-cloud-core/src/test/java/com/google/cloud/TimestampTest.java
+++ b/java-core/google-cloud-core/src/test/java/com/google/cloud/TimestampTest.java
@@ -22,6 +22,7 @@
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import com.google.common.testing.EqualsTester;
+import java.time.format.DateTimeParseException;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.GregorianCalendar;
@@ -237,14 +238,67 @@ void parseTimestampWithoutTimeZoneOffset() {
 
   @Test
   void parseTimestampWithTimeZoneOffset() {
-    assertThat(Timestamp.parseTimestamp("0001-01-01T00:00:00-00:00"))
+    // Max values
+    assertThat(Timestamp.parseTimestampDuration("0001-01-01T00:00:00-00:00"))
         .isEqualTo(Timestamp.MIN_VALUE);
-    assertThat(Timestamp.parseTimestamp("9999-12-31T23:59:59.999999999-00:00"))
+    assertThat(Timestamp.parseTimestampDuration("9999-12-31T23:59:59.999999999-00:00"))
         .isEqualTo(Timestamp.MAX_VALUE);
-    assertThat(Timestamp.parseTimestamp("2020-12-06T19:21:12.123+05:30"))
+    // Extreme values (close to min/max)
+    assertThat(Timestamp.parseTimestampDuration("0001-01-01T00:00:00.000000001Z"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(Timestamp.MIN_VALUE.getSeconds(), 1));
+    assertThat(Timestamp.parseTimestampDuration("9999-12-31T23:59:59.999999998Z"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(Timestamp.MAX_VALUE.getSeconds(), 999999998));
+    // Common use cases
+    assertThat(Timestamp.parseTimestampDuration("2020-07-10T14:03:00.123-07:00"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1594414980, 123000000));
+    assertThat(Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123+05:30"))
         .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1607262672, 123000000));
-    assertThat(Timestamp.parseTimestamp("2020-07-10T14:03:00-07:00"))
+    // We also confirm that parsing a timestamp with nano precision will behave the same as the
+    // threeten counterpart
+    assertThat(Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123+05:30"))
+        .isEqualTo(Timestamp.parseTimestamp("2020-12-06T19:21:12.123+05:30"));
+    // Timestamps with fractional seconds at nanosecond level
+    assertThat(Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123456789+05:30"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1607262672, 123456789));
+    // Fractional seconds beyond nanos should throw an exception
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123456789321+05:30"));
+    // Missing components (should throw exceptions)
+    assertThrows(
+        DateTimeParseException.class, () -> Timestamp.parseTimestampDuration("2020-12-06"));
+    // Whitespace should not be supported
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("  2020-12-06T19:21:12.123+05:30  "));
+    // It should be case-insensitive
+    assertThat(Timestamp.parseTimestampDuration("2020-07-10t14:03:00-07:00"))
         .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1594414980, 0));
+    // Invalid time zone offsets
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123+25:00"));
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123-25:00"));
+    // Int values for SecondOfMinute should be between 0 and 59
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("2016-12-31T23:59:60Z"));
+  }
+
+  @Test
+  void parseTimestampWithZoneString() {
+    // Valid RFC 3339 timestamps with time zone names
+    assertThat(Timestamp.parseTimestampDuration("2020-12-06T08:51:12.123America/Toronto"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1607262672, 123000000));
+    assertThat(Timestamp.parseTimestampDuration("2023-04-10T22:42:10.123456789Europe/London"))
+        .isEqualTo(Timestamp.ofTimeSecondsAndNanos(1681162930, 123456789));
+
+    // Invalid time zone names
+    assertThrows(
+        DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("2020-12-06T19:21:12.123Invalid/TimeZone"));
   }
 
   @Test
@@ -281,4 +335,14 @@ void comparable() {
   void serialization() {
     reserializeAndAssert(Timestamp.parseTimestamp("9999-12-31T23:59:59.999999999Z"));
   }
+
+  @Test
+  void parseInvalidTimestampThreetenThrowsThreetenException() {
+    assertThrows(
+        org.threeten.bp.format.DateTimeParseException.class,
+        () -> Timestamp.parseTimestamp("00x1-01-01T00:00:00"));
+    assertThrows(
+        java.time.format.DateTimeParseException.class,
+        () -> Timestamp.parseTimestampDuration("00x1-01-01T00:00:00"));
+  }
 }
diff --git a/java-core/google-cloud-core/src/test/java/com/google/cloud/testing/BaseEmulatorHelperTest.java b/java-core/google-cloud-core/src/test/java/com/google/cloud/testing/BaseEmulatorHelperTest.java
index 79b58a83c9..bfc2666216 100644
--- a/java-core/google-cloud-core/src/test/java/com/google/cloud/testing/BaseEmulatorHelperTest.java
+++ b/java-core/google-cloud-core/src/test/java/com/google/cloud/testing/BaseEmulatorHelperTest.java
@@ -28,12 +28,12 @@
 import java.net.URL;
 import java.net.URLConnection;
 import java.net.URLStreamHandler;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Logger;
 import org.easymock.EasyMock;
 import org.junit.jupiter.api.Test;
-import org.threeten.bp.Duration;
 
 class BaseEmulatorHelperTest {
 
@@ -71,10 +71,18 @@ public void start() throws IOException, InterruptedException {
     }
 
     @Override
-    public void stop(Duration timeout) throws IOException, InterruptedException, TimeoutException {
+    public void stop(org.threeten.bp.Duration timeout)
+        throws IOException, InterruptedException, TimeoutException {
+      // we call the threeten method directly to confirm behavior
       waitForProcess(timeout);
     }
 
+    @Override
+    public void stopDuration(Duration timeout)
+        throws IOException, InterruptedException, TimeoutException {
+      super.stopDuration(timeout);
+    }
+
     @Override
     public void reset() throws IOException {
       // do nothing
@@ -91,13 +99,33 @@ void testEmulatorHelper() throws IOException, InterruptedException, TimeoutExcep
     emulatorRunner.start();
     EasyMock.expectLastCall();
     EasyMock.expect(emulatorRunner.getProcess()).andReturn(process);
-    emulatorRunner.waitFor(Duration.ofMinutes(1));
+    emulatorRunner.waitForDuration(Duration.ofMinutes(1));
+    EasyMock.expectLastCall().andReturn(0);
+    EasyMock.replay(process, emulatorRunner);
+    TestEmulatorHelper helper =
+        new TestEmulatorHelper(ImmutableList.of(emulatorRunner), BLOCK_UNTIL);
+    helper.start();
+    helper.stopDuration(Duration.ofMinutes(1));
+    EasyMock.verify();
+  }
+
+  @Test
+  void testEmulatorHelperThreeten() throws IOException, InterruptedException, TimeoutException {
+    Process process = EasyMock.createStrictMock(Process.class);
+    InputStream stream = new ByteArrayInputStream(BLOCK_UNTIL.getBytes(Charsets.UTF_8));
+    EmulatorRunner emulatorRunner = EasyMock.createStrictMock(EmulatorRunner.class);
+    EasyMock.expect(process.getInputStream()).andReturn(stream);
+    EasyMock.expect(emulatorRunner.isAvailable()).andReturn(true);
+    emulatorRunner.start();
+    EasyMock.expectLastCall();
+    EasyMock.expect(emulatorRunner.getProcess()).andReturn(process);
+    emulatorRunner.waitForDuration(java.time.Duration.ofMinutes(1));
     EasyMock.expectLastCall().andReturn(0);
     EasyMock.replay(process, emulatorRunner);
     TestEmulatorHelper helper =
         new TestEmulatorHelper(ImmutableList.of(emulatorRunner), BLOCK_UNTIL);
     helper.start();
-    helper.stop(Duration.ofMinutes(1));
+    helper.stop(org.threeten.bp.Duration.ofMinutes(1));
     EasyMock.verify();
   }
 
@@ -157,13 +185,13 @@ void testEmulatorHelperMultipleRunners()
     secondRunner.start();
     EasyMock.expectLastCall();
     EasyMock.expect(secondRunner.getProcess()).andReturn(process);
-    secondRunner.waitFor(Duration.ofMinutes(1));
+    secondRunner.waitForDuration(Duration.ofMinutes(1));
     EasyMock.expectLastCall().andReturn(0);
     EasyMock.replay(process, secondRunner);
     TestEmulatorHelper helper =
         new TestEmulatorHelper(ImmutableList.of(firstRunner, secondRunner), BLOCK_UNTIL);
     helper.start();
-    helper.stop(Duration.ofMinutes(1));
+    helper.stopDuration(Duration.ofMinutes(1));
     EasyMock.verify();
   }
 
diff --git a/java-core/pom.xml b/java-core/pom.xml
index d1b558fe97..88c5e0d6a9 100644
--- a/java-core/pom.xml
+++ b/java-core/pom.xml
@@ -4,7 +4,7 @@
   com.google.cloud
   google-cloud-core-parent
   pom
-  2.47.0
+  2.48.0
   Google Cloud Core Parent
   
     Java idiomatic client for Google Cloud Platform services.
@@ -13,7 +13,7 @@
   
     com.google.api
     gapic-generator-java-pom-parent
-    2.49.0
+    2.50.0
     ../gapic-generator-java-pom-parent
   
 
@@ -33,7 +33,7 @@
       
         com.google.cloud
         google-cloud-shared-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
diff --git a/java-iam/grpc-google-iam-v1/pom.xml b/java-iam/grpc-google-iam-v1/pom.xml
index 6600488873..780e4a4189 100644
--- a/java-iam/grpc-google-iam-v1/pom.xml
+++ b/java-iam/grpc-google-iam-v1/pom.xml
@@ -4,13 +4,13 @@
   4.0.0
   com.google.api.grpc
   grpc-google-iam-v1
-  1.43.0
+  1.44.0
   grpc-google-iam-v1
   GRPC library for grpc-google-iam-v1
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-iam/grpc-google-iam-v2/pom.xml b/java-iam/grpc-google-iam-v2/pom.xml
index 397daefb9f..212586e1da 100644
--- a/java-iam/grpc-google-iam-v2/pom.xml
+++ b/java-iam/grpc-google-iam-v2/pom.xml
@@ -4,13 +4,13 @@
   4.0.0
   com.google.api.grpc
   grpc-google-iam-v2
-  1.43.0
+  1.44.0
   grpc-google-iam-v2
   GRPC library for proto-google-iam-v2
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-iam/grpc-google-iam-v2beta/pom.xml b/java-iam/grpc-google-iam-v2beta/pom.xml
index 5a3a881579..e5cb529a72 100644
--- a/java-iam/grpc-google-iam-v2beta/pom.xml
+++ b/java-iam/grpc-google-iam-v2beta/pom.xml
@@ -4,13 +4,13 @@
   4.0.0
   com.google.api.grpc
   grpc-google-iam-v2beta
-  1.43.0
+  1.44.0
   grpc-google-iam-v2beta
   GRPC library for proto-google-iam-v1
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-iam/pom.xml b/java-iam/pom.xml
index 9dca620fac..ceed036363 100644
--- a/java-iam/pom.xml
+++ b/java-iam/pom.xml
@@ -4,7 +4,7 @@
   com.google.cloud
   google-iam-parent
   pom
-  1.43.0
+  1.44.0
   Google IAM Parent
   
     Java idiomatic client for Google Cloud Platform services.
@@ -13,7 +13,7 @@
   
     com.google.api
     gapic-generator-java-pom-parent
-    2.49.0
+    2.50.0
     ../gapic-generator-java-pom-parent
   
 
@@ -60,7 +60,7 @@
       
         com.google.cloud
         third-party-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
@@ -88,44 +88,44 @@
       
         com.google.api
         gax-bom
-        2.57.0
+        2.58.0
         pom
         import
       
       
         com.google.api.grpc
         proto-google-iam-v2
-        1.43.0
+        1.44.0
       
       
         com.google.api.grpc
         grpc-google-iam-v2
-        1.43.0
+        1.44.0
       
       
         com.google.api.grpc
         proto-google-common-protos
-        2.48.0
+        2.49.0
       
       
         com.google.api.grpc
         proto-google-iam-v2beta
-        1.43.0
+        1.44.0
       
       
         com.google.api.grpc
         grpc-google-iam-v1
-        1.43.0
+        1.44.0
       
       
         com.google.api.grpc
         grpc-google-iam-v2beta
-        1.43.0
+        1.44.0
       
       
         com.google.api.grpc
         proto-google-iam-v1
-        1.43.0
+        1.44.0
       
       
         javax.annotation
diff --git a/java-iam/proto-google-iam-v1/pom.xml b/java-iam/proto-google-iam-v1/pom.xml
index 56e0b12453..8bc53119b9 100644
--- a/java-iam/proto-google-iam-v1/pom.xml
+++ b/java-iam/proto-google-iam-v1/pom.xml
@@ -3,13 +3,13 @@
   4.0.0
   com.google.api.grpc
   proto-google-iam-v1
-  1.43.0
+  1.44.0
   proto-google-iam-v1
   PROTO library for proto-google-iam-v1
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-iam/proto-google-iam-v2/pom.xml b/java-iam/proto-google-iam-v2/pom.xml
index caadea7aa1..a3a38e386a 100644
--- a/java-iam/proto-google-iam-v2/pom.xml
+++ b/java-iam/proto-google-iam-v2/pom.xml
@@ -4,13 +4,13 @@
   4.0.0
   com.google.api.grpc
   proto-google-iam-v2
-  1.43.0
+  1.44.0
   proto-google-iam-v2
   Proto library for proto-google-iam-v1
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-iam/proto-google-iam-v2beta/pom.xml b/java-iam/proto-google-iam-v2beta/pom.xml
index fc12546e97..b6318afbeb 100644
--- a/java-iam/proto-google-iam-v2beta/pom.xml
+++ b/java-iam/proto-google-iam-v2beta/pom.xml
@@ -4,13 +4,13 @@
   4.0.0
   com.google.api.grpc
   proto-google-iam-v2beta
-  1.43.0
+  1.44.0
   proto-google-iam-v2beta
   Proto library for proto-google-iam-v1
   
     com.google.cloud
     google-iam-parent
-    1.43.0
+    1.44.0
   
   
     
diff --git a/java-shared-dependencies/README.md b/java-shared-dependencies/README.md
index 517df7e8e5..42bdb3913b 100644
--- a/java-shared-dependencies/README.md
+++ b/java-shared-dependencies/README.md
@@ -14,7 +14,7 @@ If you are using Maven, add this to the `dependencyManagement` section.
     
       com.google.cloud
       google-cloud-shared-dependencies
-      3.39.0
+      3.40.0
       pom
       import
     
diff --git a/java-shared-dependencies/dependency-convergence-check/pom.xml b/java-shared-dependencies/dependency-convergence-check/pom.xml
index 44260c53ed..38c9aec3b8 100644
--- a/java-shared-dependencies/dependency-convergence-check/pom.xml
+++ b/java-shared-dependencies/dependency-convergence-check/pom.xml
@@ -3,7 +3,7 @@
   4.0.0
   com.google.cloud
   shared-dependencies-dependency-convergence-test
-  3.39.0
+  3.40.0
   Dependency convergence test for certain artifacts in Google Cloud Shared Dependencies
   
     An dependency convergence test case for the shared dependencies BOM. A failure of this test case means
diff --git a/java-shared-dependencies/first-party-dependencies/pom.xml b/java-shared-dependencies/first-party-dependencies/pom.xml
index 5bfcad13cd..c5154208ad 100644
--- a/java-shared-dependencies/first-party-dependencies/pom.xml
+++ b/java-shared-dependencies/first-party-dependencies/pom.xml
@@ -6,7 +6,7 @@
   com.google.cloud
   first-party-dependencies
   pom
-  3.39.0
+  3.40.0
   Google Cloud First-party Shared Dependencies
   
     Shared first-party dependencies for Google Cloud Java libraries.
@@ -15,7 +15,7 @@
   
     com.google.cloud
     google-cloud-shared-config
-    1.11.3
+    1.12.0
     
   
 
@@ -33,7 +33,7 @@
         
         com.google.api
         gapic-generator-java-bom
-        2.49.0
+        2.50.0
         pom
         import
       
@@ -45,7 +45,7 @@
       
         com.google.cloud
         google-cloud-core-bom
-        2.47.0
+        2.48.0
         pom
         import
       
@@ -69,13 +69,13 @@
       
         com.google.cloud
         google-cloud-core
-        2.47.0
+        2.48.0
         test-jar
       
       
         com.google.cloud
         google-cloud-core
-        2.47.0
+        2.48.0
         tests
       
     
diff --git a/java-shared-dependencies/pom.xml b/java-shared-dependencies/pom.xml
index e1d8b0ac8d..28bfe46b38 100644
--- a/java-shared-dependencies/pom.xml
+++ b/java-shared-dependencies/pom.xml
@@ -4,7 +4,7 @@
   com.google.cloud
   google-cloud-shared-dependencies
   pom
-  3.39.0
+  3.40.0
   
     first-party-dependencies
     third-party-dependencies
@@ -17,7 +17,7 @@
   
     com.google.api
     gapic-generator-java-pom-parent
-    2.49.0
+    2.50.0
     ../gapic-generator-java-pom-parent
   
 
@@ -31,14 +31,14 @@
       
         com.google.cloud
         first-party-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
       
         com.google.cloud
         third-party-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
diff --git a/java-shared-dependencies/third-party-dependencies/pom.xml b/java-shared-dependencies/third-party-dependencies/pom.xml
index f2b2489c2c..92ebd9eb19 100644
--- a/java-shared-dependencies/third-party-dependencies/pom.xml
+++ b/java-shared-dependencies/third-party-dependencies/pom.xml
@@ -6,7 +6,7 @@
   com.google.cloud
   third-party-dependencies
   pom
-  3.39.0
+  3.40.0
   Google Cloud Third-party Shared Dependencies
   
     Shared third-party dependencies for Google Cloud Java libraries.
@@ -15,7 +15,7 @@
   
     com.google.api
     gapic-generator-java-pom-parent
-    2.49.0
+    2.50.0
     ../../gapic-generator-java-pom-parent
   
 
diff --git a/java-shared-dependencies/upper-bound-check/pom.xml b/java-shared-dependencies/upper-bound-check/pom.xml
index 60ffedc992..cc7edda551 100644
--- a/java-shared-dependencies/upper-bound-check/pom.xml
+++ b/java-shared-dependencies/upper-bound-check/pom.xml
@@ -4,7 +4,7 @@
   com.google.cloud
   shared-dependencies-upper-bound-test
   pom
-  3.39.0
+  3.40.0
   Upper bound test for Google Cloud Shared Dependencies
   
     An upper bound test case for the shared dependencies BOM. A failure of this test case means
@@ -16,7 +16,7 @@
     com.google.cloud
     
     google-cloud-shared-config
-    1.11.3
+    1.12.0
     
   
 
@@ -30,7 +30,7 @@
       
         com.google.cloud
         google-cloud-shared-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
diff --git a/release-please-config.json b/release-please-config.json
index df5cc02468..c264303ad1 100644
--- a/release-please-config.json
+++ b/release-please-config.json
@@ -11,6 +11,7 @@
         ".cloudbuild/graalvm/cloudbuild-test-a.yaml",
         ".cloudbuild/graalvm/cloudbuild-test-b.yaml",
         ".cloudbuild/library_generation/cloudbuild-library-generation-push.yaml",
+        ".cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml",
         ".cloudbuild/library_generation/library_generation.Dockerfile"
       ]
     }
diff --git a/renovate.json b/renovate.json
index e0ef6f291f..f9db6a2237 100644
--- a/renovate.json
+++ b/renovate.json
@@ -63,6 +63,13 @@
     }
   ],
   "packageRules": [
+    {
+      "matchPackageNames": ["com.google.cloud:google-cloud-shared-config"],
+      "registryUrls": [
+        "https://repo.maven.apache.org/maven2/",
+        "https://repo1.maven.org/maven2"
+      ]
+    },
     {
       "matchUpdateTypes": [
         "major"
diff --git a/sdk-platform-java-config/pom.xml b/sdk-platform-java-config/pom.xml
index 2f46519bce..ee692e1aaf 100644
--- a/sdk-platform-java-config/pom.xml
+++ b/sdk-platform-java-config/pom.xml
@@ -4,7 +4,7 @@
     com.google.cloud
     sdk-platform-java-config
     pom
-    3.39.0 
+    3.40.0 
     SDK Platform For Java Configurations
     
         Shared build configuration for Google Cloud Java libraries.
@@ -13,10 +13,10 @@
     
         com.google.cloud
         google-cloud-shared-config
-        1.11.3
+        1.12.0
     
 
     
-        3.39.0 
+        3.40.0 
     
 
\ No newline at end of file
diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java
index a04432db8b..7a04ee146d 100644
--- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java
+++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java
@@ -116,7 +116,7 @@
  *         RetrySettings.newBuilder()
  *             .setInitialRetryDelayDuration(Duration.ofMillis(500))
  *             .setRetryDelayMultiplier(1.5)
- *             .setMaxRetryDelay(Duration.ofMillis(5000))
+ *             .setMaxRetryDelayDuration(Duration.ofMillis(5000))
  *             .setTotalTimeoutDuration(Duration.ofHours(24))
  *             .build());
  * echoSettingsBuilder
diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java
index e253bad663..cfa46cdc9c 100644
--- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java
+++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java
@@ -117,7 +117,7 @@
  *         RetrySettings.newBuilder()
  *             .setInitialRetryDelayDuration(Duration.ofMillis(500))
  *             .setRetryDelayMultiplier(1.5)
- *             .setMaxRetryDelay(Duration.ofMillis(5000))
+ *             .setMaxRetryDelayDuration(Duration.ofMillis(5000))
  *             .setTotalTimeoutDuration(Duration.ofHours(24))
  *             .build());
  * messagingSettingsBuilder
diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java
index c339fa1e7e..93765029b6 100644
--- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java
+++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java
@@ -82,10 +82,10 @@
 import com.google.showcase.v1beta1.WaitRequest;
 import com.google.showcase.v1beta1.WaitResponse;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Generated;
-import org.threeten.bp.Duration;
 
 // AUTO-GENERATED DOCUMENTATION AND CLASS.
 /**
@@ -152,7 +152,7 @@
  *         RetrySettings.newBuilder()
  *             .setInitialRetryDelayDuration(Duration.ofMillis(500))
  *             .setRetryDelayMultiplier(1.5)
- *             .setMaxRetryDelay(Duration.ofMillis(5000))
+ *             .setMaxRetryDelayDuration(Duration.ofMillis(5000))
  *             .setTotalTimeoutDuration(Duration.ofHours(24))
  *             .build());
  * echoSettingsBuilder
@@ -624,21 +624,21 @@ public static class Builder extends StubSettings.Builder actualMetricDataList = getMetricDataList();
@@ -373,7 +370,6 @@ void testGrpc_operationCancelled_recordsMetrics() throws Exception {
     verifyStatusAttribute(actualMetricDataList, statusCountList);
   }
 
-  @Disabled("https://github.com/googleapis/sdk-platform-java/issues/2503")
   @Test
   void testHttpJson_operationCancelled_recordsMetrics() throws Exception {
     int attemptCount = 1;
@@ -430,7 +426,6 @@ void testGrpc_operationFailed_recordsMetrics() throws InterruptedException {
     verifyStatusAttribute(actualMetricDataList, statusCountList);
   }
 
-  @Disabled("https://github.com/googleapis/sdk-platform-java/issues/2503")
   @Test
   void testHttpJson_operationFailed_recordsMetrics() throws InterruptedException {
     int attemptCount = 1;
@@ -526,7 +521,6 @@ void testGrpc_attemptFailedRetriesExhausted_recordsMetrics() throws Exception {
     grpcClient.awaitTermination(TestClientInitializer.AWAIT_TERMINATION_SECONDS, TimeUnit.SECONDS);
   }
 
-  @Disabled("https://github.com/googleapis/sdk-platform-java/issues/2503")
   @Test
   void testHttpJson_attemptFailedRetriesExhausted_recordsMetrics() throws Exception {
     int attemptCount = 3;
@@ -621,7 +615,6 @@ void testGrpc_attemptPermanentFailure_recordsMetrics() throws InterruptedExcepti
     verifyStatusAttribute(actualMetricDataList, statusCountList);
   }
 
-  @Disabled("https://github.com/googleapis/sdk-platform-java/issues/2503")
   @Test
   void testHttpJson_attemptPermanentFailure_recordsMetrics() throws InterruptedException {
     int attemptCount = 1;
@@ -722,7 +715,6 @@ void testGrpc_multipleFailedAttempts_successfulOperation() throws Exception {
     grpcClient.awaitTermination(TestClientInitializer.AWAIT_TERMINATION_SECONDS, TimeUnit.SECONDS);
   }
 
-  @Disabled("https://github.com/googleapis/sdk-platform-java/issues/2503")
   @Test
   void testHttpJson_multipleFailedAttempts_successfulOperation() throws Exception {
     int attemptCount = 3;
@@ -737,7 +729,7 @@ void testHttpJson_multipleFailedAttempts_successfulOperation() throws Exception
 
     EchoStubSettings.Builder httpJsonEchoSettingsBuilder = EchoStubSettings.newHttpJsonBuilder();
     httpJsonEchoSettingsBuilder
-        .echoSettings()
+        .blockSettings()
         .setRetrySettings(retrySettings)
         .setRetryableCodes(ImmutableSet.of(Code.DEADLINE_EXCEEDED));
     EchoSettings httpJsonEchoSettings = EchoSettings.create(httpJsonEchoSettingsBuilder.build());
@@ -771,7 +763,7 @@ void testHttpJson_multipleFailedAttempts_successfulOperation() throws Exception
             .setSuccess(BlockResponse.newBuilder().setContent("httpjsonBlockResponse"))
             .build();
 
-    grpcClient.block(blockRequest);
+    httpClient.block(blockRequest);
 
     List actualMetricDataList = getMetricDataList();
     verifyPointDataSum(actualMetricDataList, attemptCount);
diff --git a/showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITVersionHeaders.java b/showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITVersionHeaders.java
index 09255fe278..303cab98ec 100644
--- a/showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITVersionHeaders.java
+++ b/showcase/gapic-showcase/src/test/java/com/google/showcase/v1beta1/it/ITVersionHeaders.java
@@ -241,7 +241,7 @@ void testHttpJsonCompliance_userApiVersionSetSuccess() throws IOException {
   @Test
   void testGrpcCall_sendsCorrectApiClientHeader() {
     Pattern defautlGrpcHeaderPattern =
-        Pattern.compile("gl-java/.* gapic/.*?--protobuf-.* gax/.* grpc/.* protobuf/.*");
+        Pattern.compile("gl-java/.* gapic/.*?--protobuf-\\d.* gax/.* grpc/.* protobuf/\\d.*");
     grpcClient.echo(EchoRequest.newBuilder().build());
     String headerValue = grpcInterceptor.metadata.get(API_CLIENT_HEADER_KEY);
     assertTrue(defautlGrpcHeaderPattern.matcher(headerValue).matches());
@@ -250,7 +250,7 @@ void testGrpcCall_sendsCorrectApiClientHeader() {
   @Test
   void testHttpJson_sendsCorrectApiClientHeader() {
     Pattern defautlHttpHeaderPattern =
-        Pattern.compile("gl-java/.* gapic/.*?--protobuf-.* gax/.* rest/ protobuf/.*");
+        Pattern.compile("gl-java/.* gapic/.*?--protobuf-\\d.* gax/.* rest/ protobuf/\\d.*");
     httpJsonClient.echo(EchoRequest.newBuilder().build());
     ArrayList headerValues =
         (ArrayList)
diff --git a/showcase/pom.xml b/showcase/pom.xml
index 85af0cd696..dc1676bb31 100644
--- a/showcase/pom.xml
+++ b/showcase/pom.xml
@@ -15,7 +15,7 @@
   
     com.google.cloud
     google-cloud-shared-config
-    1.11.3
+    1.12.0
     
   
 
@@ -34,7 +34,7 @@
       
         com.google.cloud
         google-cloud-shared-dependencies
-        3.39.0
+        3.40.0
         pom
         import
       
diff --git a/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java b/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java
index ee81471625..7d7f145b1a 100644
--- a/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java
+++ b/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java
@@ -52,9 +52,9 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import java.io.IOException;
+import java.time.Duration;
 import java.util.List;
 import javax.annotation.Generated;
-import org.threeten.bp.Duration;
 
 // AUTO-GENERATED DOCUMENTATION AND CLASS.
 /**
@@ -319,13 +319,13 @@ public static class Builder extends StubSettings.Builder