diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot-hermetic.yaml similarity index 86% rename from .github/.OwlBot.yaml rename to .github/.OwlBot-hermetic.yaml index a0bb6ff769..b2261f4827 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot-hermetic.yaml @@ -31,7 +31,7 @@ deep-copy-regex: dest: "/owl-bot-staging/v2/grpc-google-cloud-storage-v2/src" - source: "/google/storage/v2/.*-java/gapic-google-.*/src" dest: "/owl-bot-staging/v2/gapic-google-cloud-storage-v2/src" -- source: "/google/storage/v2/google-cloud-storage-v2-java/gapic-google-cloud-storage-v2-java/src/main/java/com/google/storage/v2/gapic_metadata.json" +- source: "/google/storage/v2/.*-java/gapic-google-.*/src/main/java/com/google/storage/v2/gapic_metadata.json" dest: "/owl-bot-staging/v2/gapic-google-cloud-storage-v2/src/main/resources/com/google/storage/v2/gapic_metadata.json" - source: "/google/storage/control/v2/.*-java/proto-google-.*/src" dest: "/owl-bot-staging/v2/proto-google-cloud-storage-control-v2/src" @@ -39,5 +39,5 @@ deep-copy-regex: dest: "/owl-bot-staging/v2/grpc-google-cloud-storage-control-v2/src" - source: "/google/storage/control/v2/.*-java/gapic-google-.*/src" dest: "/owl-bot-staging/v2/google-cloud-storage-control/src" -- source: "/google/storage/control/v2/.*-java/gapic-google-cloud-storage-control-v2-java/src/main/java/com/google/storage/control/v2/gapic_metadata.json" +- source: "/google/storage/control/v2/.*-java/gapic-google-.*/src/main/java/com/google/storage/control/v2/gapic_metadata.json" dest: "/owl-bot-staging/v2/google-cloud-storage-control/src/main/resources/com/google/storage/control/v2/gapic_metadata.json" diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml deleted file mode 100644 index 359fe71c19..0000000000 --- a/.github/.OwlBot.lock.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -docker: - image: gcr.io/cloud-devrel-public-resources/owlbot-java:latest - digest: sha256:72f0d373307d128b2cb720c5cb4d90b31f0e86529dd138c632710ae0c69efae3 -# created: 2024-06-05T18:32:21.724930324Z diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh new file mode 100644 index 0000000000..6c3f22d8f9 --- /dev/null +++ b/.github/scripts/hermetic_library_generation.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Using default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# parse image tag from the generation configuration. +image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" + + +# commit the change to the pull request. +if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt +else + # The image leaves intermediate folders and files it works with. Here we remove them + rm -rdf output googleapis "${baseline_generation_config}" + git add --all -- ':!pr_description.txt' +fi +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh new file mode 100644 index 0000000000..561a313040 --- /dev/null +++ b/.github/scripts/update_generation_config.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) + echo "${latest}" +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: Update generation configuration at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest generation config is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" +fi diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 28d2600374..d0f61d518e 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -15,11 +15,11 @@ branchProtectionRules: - units (11) - 'Kokoro - Test: Integration' - cla/google - - OwlBot Post Processor - 'Kokoro - Test: Java GraalVM Native Image' - 'Kokoro - Test: Java 17 GraalVM Native Image' - javadoc - - unmanaged_dependency_check +# TODO: re-enabled once https://github.com/googleapis/sdk-platform-java/pull/3078 is merged and released +# - unmanaged_dependency_check - pattern: 1.113.14-sp isAdminEnforced: true requiredApprovingReviewCount: 1 diff --git a/.github/trusted-contribution.yml b/.github/trusted-contribution.yml index 88d3ac9bf1..28d3db911d 100644 --- a/.github/trusted-contribution.yml +++ b/.github/trusted-contribution.yml @@ -1,6 +1,7 @@ trustedContributors: - renovate-bot - gcf-owl-bot[bot] +- release-please[bot] annotations: - type: comment diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 0000000000..cc49d69ff4 --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + paths: + - 'generation_config.yaml' + +jobs: + library_generation: + # skip pull requests coming from a forked repository + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index aa984df018..35eacb9ba3 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -17,6 +17,6 @@ jobs: # repository .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.32.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.33.0 with: bom-path: google-cloud-storage-bom/pom.xml diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml new file mode 100644 index 0000000000..3cf7739926 --- /dev/null +++ b/.github/workflows/update_generation_config.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update generation configuration +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-generation-config: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update params in generation config to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/update_generation_config.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index 00b7922115..1b1612f6d5 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.33.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index b16171fd67..8da4fbaf49 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.32.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.33.0" } env_vars: { diff --git a/.readme-partials.yaml b/.readme-partials.yaml index 27755efec7..0cb64e02a6 100644 --- a/.readme-partials.yaml +++ b/.readme-partials.yaml @@ -32,19 +32,19 @@ custom_content: | com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 ``` If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy - implementation platform('com.google.cloud:libraries-bom:2.40.1') + implementation platform('com.google.cloud:libraries-bom:2.41.0') implementation 'com.google.cloud:google-cloud-storage-control' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy - implementation 'com.google.cloud:google-cloud-storage-control:2.40.1' + implementation 'com.google.cloud:google-cloud-storage-control:2.41.0' ``` #### Creating an authorized service object diff --git a/.repo-metadata.json b/.repo-metadata.json index e37a0cdb0c..bc02423b9e 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,19 +2,20 @@ "api_shortname": "storage", "name_pretty": "Cloud Storage", "product_documentation": "https://cloud.google.com/storage", - "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-storage/latest/history", "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally.", - "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", + "client_documentation": "https://cloud.google.com/java/docs/reference/google-cloud-storage/latest/history", "release_level": "stable", + "transport": "grpc", "language": "java", "repo": "googleapis/java-storage", "repo_short": "java-storage", "distribution_name": "com.google.cloud:google-cloud-storage", - "codeowner_team": "@googleapis/cloud-storage-dpe", "api_id": "storage.googleapis.com", - "requires_billing": true, "library_type": "GAPIC_COMBO", - "extra_versioned_modules": "gapic-google-cloud-storage-v2", + "requires_billing": true, + "codeowner_team": "@googleapis/cloud-storage-dpe", "excluded_poms": "google-cloud-storage-bom,google-cloud-storage", + "issue_tracker": "https://issuetracker.google.com/savedsearches/559782", + "extra_versioned_modules": "gapic-google-cloud-storage-v2", "recommended_package": "com.google.cloud.storage" -} +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5821bc840d..a889954b2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [2.41.0](https://github.com/googleapis/java-storage/compare/v2.40.1...v2.41.0) (2024-07-31) + + +### Features + +* Enable gRPC client open telemetry metrics reporting ([#2590](https://github.com/googleapis/java-storage/issues/2590)) ([d153228](https://github.com/googleapis/java-storage/commit/d153228a301007b5952de9722f370dda0784473a)) + + +### Bug Fixes + +* Add UnknownHostException to set of retriable exception ([#2651](https://github.com/googleapis/java-storage/issues/2651)) ([18de9fc](https://github.com/googleapis/java-storage/commit/18de9fcdb831132336eca4112dfe0515174bba7b)) +* Update grpc resumable upload error categorization to be more tolerant ([#2644](https://github.com/googleapis/java-storage/issues/2644)) ([95697dd](https://github.com/googleapis/java-storage/commit/95697dd3d744351058c13793c6ae576820f6b638)) +* Update Storage#readAllBytes to respect shouldReturnRawInputStream option ([#2635](https://github.com/googleapis/java-storage/issues/2635)) ([dc883cc](https://github.com/googleapis/java-storage/commit/dc883cce5f547def7cfb34c4f8a2d409493e4cb9)) +* Update TransferManager downloads to reduce in memory buffering ([#2630](https://github.com/googleapis/java-storage/issues/2630)) ([fc2fd75](https://github.com/googleapis/java-storage/commit/fc2fd750ed60b840e6285a4b1f4ecce739df4c09)) +* Use fast calculation for totalRemaining number of bytes from multiple ByteBuffers ([#2633](https://github.com/googleapis/java-storage/issues/2633)) ([758b3dd](https://github.com/googleapis/java-storage/commit/758b3dd3cc4f6dfc2dfc12c3a77472d97c31c5d5)) + + +### Dependencies + +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240625-2.0.0 ([#2616](https://github.com/googleapis/java-storage/issues/2616)) ([b22babb](https://github.com/googleapis/java-storage/commit/b22babbe26572d8c4289a65a0b125b2a60e8ef79)) +* Update dependency com.google.apis:google-api-services-storage to v1-rev20240706-2.0.0 ([#2634](https://github.com/googleapis/java-storage/issues/2634)) ([1ccaa0c](https://github.com/googleapis/java-storage/commit/1ccaa0c64887a0661438957e9427237ee005ccf1)) +* Update dependency com.google.cloud:sdk-platform-java-config to v3.33.0 ([#2647](https://github.com/googleapis/java-storage/issues/2647)) ([8196259](https://github.com/googleapis/java-storage/commit/8196259927330ecfe3e604c24d248f7935e7fe0d)) +* Update dependency net.jqwik:jqwik to v1.9.0 ([#2608](https://github.com/googleapis/java-storage/issues/2608)) ([a20eb66](https://github.com/googleapis/java-storage/commit/a20eb660ddfa4b68d79ce04496064f3025676d5a)) +* Update dependency org.junit.vintage:junit-vintage-engine to v5.10.3 ([#2604](https://github.com/googleapis/java-storage/issues/2604)) ([8c79f39](https://github.com/googleapis/java-storage/commit/8c79f39ad78d100065c189bcf8e18644b29ff9ed)) +* Update junit-platform.version to v5.10.3 ([#2605](https://github.com/googleapis/java-storage/issues/2605)) ([a532ee4](https://github.com/googleapis/java-storage/commit/a532ee49e2ff5972ea8a2aabbab2dcf6fe0df774)) + ## [2.40.1](https://github.com/googleapis/java-storage/compare/v2.40.0...v2.40.1) (2024-06-26) diff --git a/README.md b/README.md index 33eacab64b..73fc298343 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file: com.google.cloud libraries-bom - 26.42.0 + 26.43.0 pom import @@ -46,12 +46,12 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-storage - 2.40.0 + 2.41.0 com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 ``` @@ -59,20 +59,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.42.0') +implementation platform('com.google.cloud:libraries-bom:2.41.0') implementation 'com.google.cloud:google-cloud-storage' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage:2.40.0' +implementation 'com.google.cloud:google-cloud-storage:2.41.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.40.0" +libraryDependencies += "com.google.cloud" % "google-cloud-storage" % "2.41.0" ``` @@ -141,19 +141,19 @@ If you are using Maven without the BOM, add this to your dependencies: com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 ``` If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:2.40.1') +implementation platform('com.google.cloud:libraries-bom:2.41.0') implementation 'com.google.cloud:google-cloud-storage-control' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-storage-control:2.40.1' +implementation 'com.google.cloud:google-cloud-storage-control:2.41.0' ``` #### Creating an authorized service object @@ -412,6 +412,10 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-storage/tree/ To get help, follow the instructions in the [shared Troubleshooting document][troubleshooting]. +## Transport + +Cloud Storage uses gRPC for the transport layer. + ## Supported Java Versions Java 8 or above is required for using this client. @@ -504,7 +508,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-storage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-storage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.40.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-storage/2.41.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/gapic-google-cloud-storage-v2/pom.xml b/gapic-google-cloud-storage-v2/pom.xml index a5ba5d2887..7a22e710e4 100644 --- a/gapic-google-cloud-storage-v2/pom.xml +++ b/gapic-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha gapic-google-cloud-storage-v2 GRPC library for gapic-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java b/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java index 1b42b3c7e2..3ced26a1a9 100644 --- a/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java +++ b/gapic-google-cloud-storage-v2/src/main/java/com/google/storage/v2/stub/StorageStubSettings.java @@ -23,6 +23,7 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; +import com.google.api.core.ObsoleteApi; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -637,6 +638,7 @@ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuild } /** Returns the default service endpoint. */ + @ObsoleteApi("Use getEndpoint() instead") public static String getDefaultEndpoint() { return "storage.googleapis.com:443"; } diff --git a/generation_config.yaml b/generation_config.yaml new file mode 100644 index 0000000000..ceae17f95f --- /dev/null +++ b/generation_config.yaml @@ -0,0 +1,25 @@ +gapic_generator_version: 2.43.0 +googleapis_commitish: d8fce50eea92bac3a6612ee61559989ce3b38776 +libraries_bom_version: 26.43.0 +libraries: + - api_shortname: storage + name_pretty: Cloud Storage + product_documentation: https://cloud.google.com/storage + client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-storage/latest/history + api_description: 'is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally.' + issue_tracker: https://issuetracker.google.com/savedsearches/559782 + release_level: stable + language: java + repo: googleapis/java-storage + repo_short: java-storage + distribution_name: com.google.cloud:google-cloud-storage + codeowner_team: '@googleapis/cloud-storage-dpe' + api_id: storage.googleapis.com + requires_billing: true + library_type: GAPIC_COMBO + extra_versioned_modules: gapic-google-cloud-storage-v2 + excluded_poms: google-cloud-storage-bom,google-cloud-storage + recommended_package: com.google.cloud.storage + GAPICs: + - proto_path: google/storage/v2 + - proto_path: google/storage/control/v2 diff --git a/google-cloud-storage-bom/pom.xml b/google-cloud-storage-bom/pom.xml index ef1657900c..28c2f85e85 100644 --- a/google-cloud-storage-bom/pom.xml +++ b/google-cloud-storage-bom/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud google-cloud-storage-bom - 2.40.1 + 2.41.0 pom com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 @@ -69,37 +69,37 @@ com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 diff --git a/google-cloud-storage-control/pom.xml b/google-cloud-storage-control/pom.xml index fd58a79375..fe1aa8edaf 100644 --- a/google-cloud-storage-control/pom.xml +++ b/google-cloud-storage-control/pom.xml @@ -5,13 +5,13 @@ 4.0.0 com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 google-cloud-storage-control GRPC library for google-cloud-storage-control com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java b/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java index eb717d3824..b9231bb537 100644 --- a/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java +++ b/google-cloud-storage-control/src/main/java/com/google/storage/control/v2/stub/StorageControlStubSettings.java @@ -21,6 +21,7 @@ import com.google.api.core.ApiFunction; import com.google.api.core.ApiFuture; +import com.google.api.core.ObsoleteApi; import com.google.api.gax.core.GaxProperties; import com.google.api.gax.core.GoogleCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; @@ -334,6 +335,7 @@ public static InstantiatingExecutorProvider.Builder defaultExecutorProviderBuild } /** Returns the default service endpoint. */ + @ObsoleteApi("Use getEndpoint() instead") public static String getDefaultEndpoint() { return "storage.googleapis.com:443"; } diff --git a/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json b/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json index 36399f5086..c10085e0b1 100644 --- a/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json +++ b/google-cloud-storage-control/src/main/resources/META-INF/native-image/com.google.storage.control.v2/reflect-config.json @@ -440,6 +440,24 @@ "allDeclaredClasses": true, "allPublicClasses": true }, + { + "name": "com.google.api.TypeReference", + "queryAllDeclaredConstructors": true, + "queryAllPublicConstructors": true, + "queryAllDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, + { + "name": "com.google.api.TypeReference$Builder", + "queryAllDeclaredConstructors": true, + "queryAllPublicConstructors": true, + "queryAllDeclaredMethods": true, + "allPublicMethods": true, + "allDeclaredClasses": true, + "allPublicClasses": true + }, { "name": "com.google.longrunning.CancelOperationRequest", "queryAllDeclaredConstructors": true, diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 7611c23326..fbc9e99af3 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-storage - 2.40.1 + 2.41.0 jar Google Cloud Storage https://github.com/googleapis/java-storage @@ -12,12 +12,12 @@ com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 google-cloud-storage 1.113.0 - 5.10.2 + 5.10.3 @@ -124,6 +124,53 @@ com.google.api.grpc gapic-google-cloud-storage-v2 + + + io.opentelemetry + opentelemetry-sdk + + + io.grpc + grpc-opentelemetry + + + io.opentelemetry + opentelemetry-api + + + + io.opentelemetry + opentelemetry-sdk-metrics + + + io.opentelemetry + opentelemetry-sdk-common + + + + io.opentelemetry + opentelemetry-sdk-extension-autoconfigure-spi + + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + com.google.cloud.opentelemetry + exporter-metrics + + + + io.opentelemetry.contrib + opentelemetry-gcp-resources + + + + org.checkerframework + checker-qual + @@ -159,10 +206,7 @@ grpc-googleapis runtime - - org.checkerframework - checker-qual - + org.junit.vintage:junit-vintage-engine + + + io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi + io.opentelemetry.semconv:opentelemetry-semconv diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java index cf85b7400d..b9a8df0b52 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/DefaultStorageRetryStrategy.java @@ -26,6 +26,7 @@ import com.google.gson.stream.MalformedJsonException; import java.io.IOException; import java.net.SocketException; +import java.net.UnknownHostException; import java.util.Set; import javax.net.ssl.SSLException; @@ -114,6 +115,8 @@ private RetryResult shouldRetryIOException(IOException ioException) { SocketException se = (SocketException) cause; return shouldRetryIOException(se); } + } else if (ioException instanceof UnknownHostException && idempotent) { + return RetryResult.RETRY; } if (BaseServiceException.isRetryable(idempotent, ioException)) { return RetryResult.RETRY; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java index 984c7bfd8f..e85363b227 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedFinalizeOnCloseResumableWritableByteChannel.java @@ -17,6 +17,7 @@ package com.google.cloud.storage; import static com.google.cloud.storage.GrpcUtils.contextWithBucketName; +import static com.google.cloud.storage.Utils.nullSafeList; import com.google.api.core.SettableApiFuture; import com.google.api.gax.grpc.GrpcCallContext; @@ -26,7 +27,6 @@ import com.google.cloud.storage.ChunkSegmenter.ChunkSegment; import com.google.cloud.storage.Crc32cValue.Crc32cLengthKnown; import com.google.cloud.storage.UnbufferedWritableByteChannelSession.UnbufferedWritableByteChannel; -import com.google.common.collect.ImmutableList; import com.google.protobuf.ByteString; import com.google.storage.v2.ChecksummedData; import com.google.storage.v2.ObjectChecksums; @@ -230,7 +230,7 @@ public void onError(Throwable t) { tmp.getCode(), tmp.getMessage(), tmp.getReason(), - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), null, context, t); @@ -251,7 +251,7 @@ public void onCompleted() { 0, "onComplete without preceding onNext, unable to determine success.", "invalid", - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), null, context, null)); @@ -263,11 +263,11 @@ public void onCompleted() { } else if (finalSize < totalSentBytes) { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_4_1.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } else { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_4_2.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } } else if (!finalizing || last.hasPersistedSize()) { // unexpected incremental response clientDetectedError( @@ -275,14 +275,14 @@ public void onCompleted() { 0, "Unexpected incremental response for finalizing request.", "invalid", - ImmutableList.of(lastWrittenRequest), + nullSafeList(lastWrittenRequest), last, context, null)); } else { clientDetectedError( ResumableSessionFailureScenario.SCENARIO_0.toStorageException( - ImmutableList.of(lastWrittenRequest), last, context, null)); + nullSafeList(lastWrittenRequest), last, context, null)); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedReadableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedReadableByteChannel.java index 6cecc8dded..ecaa3bc878 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedReadableByteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GapicUnbufferedReadableByteChannel.java @@ -30,11 +30,9 @@ import com.google.storage.v2.ReadObjectResponse; import java.io.Closeable; import java.io.IOException; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.ScatteringByteChannel; -import java.util.Arrays; import java.util.Iterator; final class GapicUnbufferedReadableByteChannel @@ -80,7 +78,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { throw new ClosedChannelException(); } - long totalBufferCapacity = Arrays.stream(dsts).mapToLong(Buffer::remaining).sum(); + long totalBufferCapacity = Buffers.totalRemaining(dsts, offset, length); ReadCursor c = new ReadCursor(blobOffset, blobOffset + totalBufferCapacity); while (c.hasRemaining()) { if (leftovers != null) { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java index 92081e03e0..64376cc107 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageOptions.java @@ -116,6 +116,9 @@ public final class GrpcStorageOptions extends StorageOptions private final GrpcRetryAlgorithmManager retryAlgorithmManager; private final Duration terminationAwaitDuration; private final boolean attemptDirectPath; + private final boolean enableGrpcClientMetrics; + + private final boolean grpcClientMetricsManuallyEnabled; private final GrpcInterceptorProvider grpcInterceptorProvider; private final BlobWriteSessionConfig blobWriteSessionConfig; @@ -129,6 +132,8 @@ private GrpcStorageOptions(Builder builder, GrpcStorageDefaults serviceDefaults) MoreObjects.firstNonNull( builder.terminationAwaitDuration, serviceDefaults.getTerminationAwaitDuration()); this.attemptDirectPath = builder.attemptDirectPath; + this.enableGrpcClientMetrics = builder.enableGrpcClientMetrics; + this.grpcClientMetricsManuallyEnabled = builder.grpcMetricsManuallyEnabled; this.grpcInterceptorProvider = builder.grpcInterceptorProvider; this.blobWriteSessionConfig = builder.blobWriteSessionConfig; } @@ -287,6 +292,16 @@ private Tuple> resolveSettingsAndOpts() throw if (scheme.equals("http")) { channelProviderBuilder.setChannelConfigurator(ManagedChannelBuilder::usePlaintext); } + + if (enableGrpcClientMetrics) { + OpenTelemetryBootstrappingUtils.enableGrpcMetrics( + channelProviderBuilder, + endpoint, + this.getProjectId(), + this.getUniverseDomain(), + !grpcClientMetricsManuallyEnabled); + } + builder.setTransportChannelProvider(channelProviderBuilder.build()); RetrySettings baseRetrySettings = getRetrySettings(); RetrySettings readRetrySettings = @@ -350,6 +365,7 @@ public int hashCode() { retryAlgorithmManager, terminationAwaitDuration, attemptDirectPath, + enableGrpcClientMetrics, grpcInterceptorProvider, blobWriteSessionConfig, baseHashCode()); @@ -365,6 +381,7 @@ public boolean equals(Object o) { } GrpcStorageOptions that = (GrpcStorageOptions) o; return attemptDirectPath == that.attemptDirectPath + && enableGrpcClientMetrics == that.enableGrpcClientMetrics && Objects.equals(retryAlgorithmManager, that.retryAlgorithmManager) && Objects.equals(terminationAwaitDuration, that.terminationAwaitDuration) && Objects.equals(grpcInterceptorProvider, that.grpcInterceptorProvider) @@ -408,11 +425,15 @@ public static final class Builder extends StorageOptions.Builder { private StorageRetryStrategy storageRetryStrategy; private Duration terminationAwaitDuration; private boolean attemptDirectPath = GrpcStorageDefaults.INSTANCE.isAttemptDirectPath(); + private boolean enableGrpcClientMetrics = + GrpcStorageDefaults.INSTANCE.isEnableGrpcClientMetrics(); private GrpcInterceptorProvider grpcInterceptorProvider = GrpcStorageDefaults.INSTANCE.grpcInterceptorProvider(); private BlobWriteSessionConfig blobWriteSessionConfig = GrpcStorageDefaults.INSTANCE.getDefaultStorageWriterConfig(); + private boolean grpcMetricsManuallyEnabled = false; + Builder() {} Builder(StorageOptions options) { @@ -421,6 +442,7 @@ public static final class Builder extends StorageOptions.Builder { this.storageRetryStrategy = gso.getRetryAlgorithmManager().retryStrategy; this.terminationAwaitDuration = gso.getTerminationAwaitDuration(); this.attemptDirectPath = gso.attemptDirectPath; + this.enableGrpcClientMetrics = gso.enableGrpcClientMetrics; this.grpcInterceptorProvider = gso.grpcInterceptorProvider; this.blobWriteSessionConfig = gso.blobWriteSessionConfig; } @@ -454,6 +476,21 @@ public GrpcStorageOptions.Builder setAttemptDirectPath(boolean attemptDirectPath this.attemptDirectPath = attemptDirectPath; return this; } + /** + * Option for whether this client should emit internal gRPC client internal metrics to Cloud + * Monitoring. To disable metric reporting, set this to false. True by default. Emitting metrics + * is free and requires minimal CPU and memory. + * + * @since 2.41.0 This new api is in preview and is subject to breaking changes. + */ + @BetaApi + public GrpcStorageOptions.Builder setEnableGrpcClientMetrics(boolean enableGrpcClientMetrics) { + this.enableGrpcClientMetrics = enableGrpcClientMetrics; + if (enableGrpcClientMetrics) { + grpcMetricsManuallyEnabled = true; + } + return this; + } /** @since 2.14.0 This new api is in preview and is subject to breaking changes. */ @BetaApi @@ -660,6 +697,12 @@ public boolean isAttemptDirectPath() { return false; } + /** @since 2.41.0 This new api is in preview and is subject to breaking changes. */ + @BetaApi + public boolean isEnableGrpcClientMetrics() { + return true; + } + /** @since 2.22.3 This new api is in preview and is subject to breaking changes. */ @BetaApi public GrpcInterceptorProvider grpcInterceptorProvider() { diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java new file mode 100644 index 0000000000..ed0ba544e5 --- /dev/null +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java @@ -0,0 +1,382 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import com.google.api.core.ApiFunction; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.api.gax.rpc.UnavailableException; +import com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter; +import com.google.cloud.opentelemetry.metric.MetricConfiguration; +import com.google.cloud.opentelemetry.metric.MonitoredResourceDescription; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import io.grpc.ManagedChannelBuilder; +import io.grpc.opentelemetry.GrpcOpenTelemetry; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.internal.StringUtils; +import io.opentelemetry.contrib.gcp.resource.GCPResourceProvider; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.common.export.MemoryMode; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import java.math.BigDecimal; +import java.math.MathContext; +import java.net.NoRouteToHostException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +final class OpenTelemetryBootstrappingUtils { + private static final Collection METRICS_TO_ENABLE = + ImmutableList.of( + "grpc.lb.wrr.rr_fallback", + "grpc.lb.wrr.endpoint_weight_not_yet_usable", + "grpc.lb.wrr.endpoint_weight_stale", + "grpc.lb.wrr.endpoint_weights", + "grpc.lb.rls.cache_entries", + "grpc.lb.rls.cache_size", + "grpc.lb.rls.default_target_picks", + "grpc.lb.rls.target_picks", + "grpc.lb.rls.failed_picks", + "grpc.xds_client.connected", + "grpc.xds_client.server_failure", + "grpc.xds_client.resource_updates_valid", + "grpc.xds_client.resource_updates_invalid", + "grpc.xds_client.resources"); + + private static final Collection METRICS_ENABLED_BY_DEFAULT = + ImmutableList.of( + "grpc.client.attempt.sent_total_compressed_message_size", + "grpc.client.attempt.rcvd_total_compressed_message_size", + "grpc.client.attempt.started", + "grpc.client.attempt.duration", + "grpc.client.call.duration"); + + static final Logger log = Logger.getLogger(OpenTelemetryBootstrappingUtils.class.getName()); + + static void enableGrpcMetrics( + InstantiatingGrpcChannelProvider.Builder channelProviderBuilder, + String endpoint, + String projectId, + String universeDomain, + boolean shouldSuppressExceptions) { + String metricServiceEndpoint = getCloudMonitoringEndpoint(endpoint, universeDomain); + SdkMeterProvider provider = + createMeterProvider(metricServiceEndpoint, projectId, shouldSuppressExceptions); + + OpenTelemetrySdk openTelemetrySdk = + OpenTelemetrySdk.builder().setMeterProvider(provider).build(); + GrpcOpenTelemetry grpcOpenTelemetry = + GrpcOpenTelemetry.newBuilder() + .sdk(openTelemetrySdk) + .enableMetrics(METRICS_TO_ENABLE) + .build(); + ApiFunction channelConfigurator = + channelProviderBuilder.getChannelConfigurator(); + channelProviderBuilder.setChannelConfigurator( + b -> { + grpcOpenTelemetry.configureChannelBuilder(b); + if (channelConfigurator != null) { + return channelConfigurator.apply(b); + } + return b; + }); + } + + @VisibleForTesting + static String getCloudMonitoringEndpoint(String endpoint, String universeDomain) { + String metricServiceEndpoint = "monitoring.googleapis.com"; + + // use contains instead of equals because endpoint has a port in it + if (universeDomain != null && endpoint.contains("storage." + universeDomain)) { + metricServiceEndpoint = "monitoring." + universeDomain; + } else if (!endpoint.contains("storage.googleapis.com")) { + String canonicalEndpoint = "storage.googleapis.com"; + String privateEndpoint = "private.googleapis.com"; + String restrictedEndpoint = "restricted.googleapis.com"; + if (universeDomain != null) { + canonicalEndpoint = "storage." + universeDomain; + privateEndpoint = "private." + universeDomain; + restrictedEndpoint = "restricted." + universeDomain; + } + String match = + ImmutableList.of(canonicalEndpoint, privateEndpoint, restrictedEndpoint).stream() + .filter(s -> endpoint.contains(s) || endpoint.contains("google-c2p:///" + s)) + .collect(Collectors.joining()); + if (!StringUtils.isNullOrEmpty(match)) { + metricServiceEndpoint = match; + } + } + return metricServiceEndpoint + ":" + endpoint.split(":")[1]; + } + + @VisibleForTesting + static SdkMeterProvider createMeterProvider( + String metricServiceEndpoint, String projectId, boolean shouldSuppressExceptions) { + GCPResourceProvider resourceProvider = new GCPResourceProvider(); + Attributes detectedAttributes = resourceProvider.getAttributes(); + + String detectedProjectId = detectedAttributes.get(AttributeKey.stringKey("cloud.account.id")); + String projectIdToUse = detectedProjectId == null ? projectId : detectedProjectId; + + if (!projectIdToUse.equals(projectId)) { + log.warning( + "The Project ID configured for metrics is " + + projectIdToUse + + ", but the Project ID of the storage client is " + + projectId + + ". Make sure that the service account in use has the required metric writing role " + + "(roles/monitoring.metricWriter) in the project " + + projectIdToUse + + ", or metrics will not be written."); + } + + MonitoredResourceDescription monitoredResourceDescription = + new MonitoredResourceDescription( + "storage.googleapis.com/Client", + ImmutableSet.of( + "project_id", "location", "cloud_platform", "host_id", "instance_id", "api")); + + MetricExporter cloudMonitoringExporter = + GoogleCloudMetricExporter.createWithConfiguration( + MetricConfiguration.builder() + .setMonitoredResourceDescription(monitoredResourceDescription) + .setInstrumentationLibraryLabelsEnabled(false) + .setMetricServiceEndpoint(metricServiceEndpoint) + .setPrefix("storage.googleapis.com/client") + .setUseServiceTimeSeries(true) + .setProjectId(projectIdToUse) + .build()); + + SdkMeterProviderBuilder providerBuilder = SdkMeterProvider.builder(); + + // This replaces the dots with slashes in each metric, which is the format needed for this + // monitored resource + for (String metric : + ImmutableList.copyOf(Iterables.concat(METRICS_TO_ENABLE, METRICS_ENABLED_BY_DEFAULT))) { + providerBuilder.registerView( + InstrumentSelector.builder().setName(metric).build(), + View.builder().setName(metric.replace(".", "/")).build()); + } + MetricExporter exporter = + shouldSuppressExceptions + ? new PermissionDeniedSingleReportMetricsExporter(cloudMonitoringExporter) + : cloudMonitoringExporter; + providerBuilder + .registerMetricReader( + PeriodicMetricReader.builder(exporter) + .setInterval(java.time.Duration.ofSeconds(60)) + .build()) + .setResource( + Resource.create( + Attributes.builder() + .put("gcp.resource_type", "storage.googleapis.com/Client") + .put("location", detectedAttributes.get(AttributeKey.stringKey("cloud.region"))) + .put("project_id", projectIdToUse) + .put( + "cloud_platform", + detectedAttributes.get(AttributeKey.stringKey("cloud.platform"))) + .put("host_id", detectedAttributes.get(AttributeKey.stringKey("host.id"))) + .put("instance_id", UUID.randomUUID().toString()) + .put("api", "grpc") + .build())); + + addHistogramView( + providerBuilder, latencyHistogramBoundaries(), "grpc/client/attempt/duration", "s"); + addHistogramView( + providerBuilder, + sizeHistogramBoundaries(), + "grpc/client/attempt/rcvd_total_compressed_message_size", + "By"); + addHistogramView( + providerBuilder, + sizeHistogramBoundaries(), + "grpc/client/attempt/sent_total_compressed_message_size", + "By"); + + return providerBuilder.build(); + } + + private static void addHistogramView( + SdkMeterProviderBuilder provider, List boundaries, String name, String unit) { + InstrumentSelector instrumentSelector = + InstrumentSelector.builder() + .setType(InstrumentType.HISTOGRAM) + .setUnit(unit) + .setName(name) + .setMeterName("grpc-java") + .setMeterSchemaUrl("") + .build(); + View view = + View.builder() + .setName(name) + .setDescription( + "A view of " + + name + + " with histogram boundaries more appropriate for Google Cloud Storage RPCs") + .setAggregation(Aggregation.explicitBucketHistogram(boundaries)) + .build(); + provider.registerView(instrumentSelector, view); + } + + private static List latencyHistogramBoundaries() { + List boundaries = new ArrayList<>(); + BigDecimal boundary = new BigDecimal(0, MathContext.UNLIMITED); + BigDecimal increment = new BigDecimal("0.002", MathContext.UNLIMITED); // 2ms + + // 2ms buckets for the first 100ms, so we can have higher resolution for uploads and downloads + // in the 100 KiB range + for (int i = 0; i != 50; i++) { + boundaries.add(boundary.doubleValue()); + boundary = boundary.add(increment); + } + + // For the remaining buckets do 10 10ms, 10 20ms, and so on, up until 5 minutes + increment = new BigDecimal("0.01", MathContext.UNLIMITED); // 10 ms + for (int i = 0; i != 150 && boundary.compareTo(new BigDecimal(300)) < 1; i++) { + boundaries.add(boundary.doubleValue()); + if (i != 0 && i % 10 == 0) { + increment = increment.multiply(new BigDecimal(2)); + } + boundary = boundary.add(increment); + } + + return boundaries; + } + + private static List sizeHistogramBoundaries() { + long kb = 1024; + long mb = 1024 * kb; + long gb = 1024 * mb; + + List boundaries = new ArrayList<>(); + long boundary = 0; + long increment = 128 * kb; + + // 128 KiB increments up to 4MiB, then exponential growth + while (boundaries.size() < 200 && boundary <= 16 * gb) { + boundaries.add((double) boundary); + boundary += increment; + if (boundary >= 4 * mb) { + increment *= 2; + } + } + return boundaries; + } + + private static final class PermissionDeniedSingleReportMetricsExporter implements MetricExporter { + private final MetricExporter delegate; + private final AtomicBoolean seenPermissionDenied = new AtomicBoolean(false); + private final AtomicBoolean seenNoRouteToHost = new AtomicBoolean(false); + + private PermissionDeniedSingleReportMetricsExporter(MetricExporter delegate) { + this.delegate = delegate; + } + + @Override + public CompletableResultCode export(Collection metrics) { + if (seenPermissionDenied.get() && seenNoRouteToHost.get()) { + return CompletableResultCode.ofFailure(); + } + + try { + return delegate.export(metrics); + } catch (PermissionDeniedException e) { + if (!seenPermissionDenied.get()) { + seenPermissionDenied.set(true); + throw e; + } + return CompletableResultCode.ofFailure(); + } catch (UnavailableException e) { + if (seenPermissionDenied.get() + && !seenNoRouteToHost.get() + && ultimateCause(e, NoRouteToHostException.class)) { + seenNoRouteToHost.set(true); + throw e; + } + return CompletableResultCode.ofFailure(); + } + } + + @Override + public Aggregation getDefaultAggregation(InstrumentType instrumentType) { + return delegate.getDefaultAggregation(instrumentType); + } + + @Override + public MemoryMode getMemoryMode() { + return delegate.getMemoryMode(); + } + + @Override + public CompletableResultCode flush() { + return delegate.flush(); + } + + @Override + public CompletableResultCode shutdown() { + return delegate.shutdown(); + } + + @Override + public void close() { + delegate.close(); + } + + @Override + public AggregationTemporality getAggregationTemporality(InstrumentType instrumentType) { + return delegate.getAggregationTemporality(instrumentType); + } + + @Override + public DefaultAggregationSelector with(InstrumentType instrumentType, Aggregation aggregation) { + return delegate.with(instrumentType, aggregation); + } + + private static boolean ultimateCause(Throwable t, Class c) { + if (t == null) { + return false; + } + + Throwable cause = t.getCause(); + if (cause != null && c.isAssignableFrom(cause.getClass())) { + return true; + } else { + return ultimateCause(cause, c); + } + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java index 88dbaf7bdc..aab5263397 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ResumableSessionFailureScenario.java @@ -169,6 +169,9 @@ static StorageException toStorageException( Map> extraHeaders = context.getExtraHeaders(); recordHeadersTo(extraHeaders, PREFIX_O, sb); int length = reqs.size(); + if (length == 0) { + sb.append("\n").append(PREFIX_O).append("[]"); + } for (int i = 0; i < length; i++) { if (i == 0) { sb.append("\n").append(PREFIX_O).append("["); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java index 03d9cc4627..ef215573fb 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RewindableContent.java @@ -22,7 +22,6 @@ import com.google.common.io.ByteStreams; import java.io.IOException; import java.io.OutputStream; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.GatheringByteChannel; @@ -173,7 +172,7 @@ private static final class ByteBufferContent extends RewindableContent { private ByteBufferContent(ByteBuffer[] buffers) { this.buffers = buffers; this.positions = Arrays.stream(buffers).mapToInt(Buffers::position).toArray(); - this.totalLength = Arrays.stream(buffers).mapToLong(Buffer::remaining).sum(); + this.totalLength = Buffers.totalRemaining(buffers, 0, buffers.length); this.dirty = false; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ThroughputSink.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ThroughputSink.java index 7d6d909dd6..4356df936d 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ThroughputSink.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ThroughputSink.java @@ -18,14 +18,12 @@ import com.google.common.base.MoreObjects; import java.io.IOException; -import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.channels.GatheringByteChannel; import java.nio.channels.WritableByteChannel; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; import java.util.logging.Logger; @@ -262,7 +260,7 @@ public int write(ByteBuffer src) throws IOException { @Override public long write(ByteBuffer[] srcs, int offset, int length) throws IOException { boolean exception = false; - long available = Arrays.stream(srcs).mapToLong(Buffer::remaining).sum(); + long available = Buffers.totalRemaining(srcs, offset, length); Instant begin = clock.instant(); try { return delegate.write(srcs, offset, length); @@ -271,7 +269,7 @@ public long write(ByteBuffer[] srcs, int offset, int length) throws IOException throw e; } finally { Instant end = clock.instant(); - long remaining = Arrays.stream(srcs).mapToLong(Buffer::remaining).sum(); + long remaining = Buffers.totalRemaining(srcs, offset, length); Record record = Record.of(available - remaining, begin, end, exception); sink.recordThroughput(record); } @@ -280,7 +278,7 @@ public long write(ByteBuffer[] srcs, int offset, int length) throws IOException @Override public long write(ByteBuffer[] srcs) throws IOException { boolean exception = false; - long available = Arrays.stream(srcs).mapToLong(Buffer::remaining).sum(); + long available = Buffers.totalRemaining(srcs, 0, srcs.length); Instant begin = clock.instant(); try { return delegate.write(srcs); @@ -289,7 +287,7 @@ public long write(ByteBuffer[] srcs) throws IOException { throw e; } finally { Instant end = clock.instant(); - long remaining = Arrays.stream(srcs).mapToLong(Buffer::remaining).sum(); + long remaining = Buffers.totalRemaining(srcs, 0, srcs.length); Record record = Record.of(available - remaining, begin, end, exception); sink.recordThroughput(record); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java index c24a68d4d6..c0374d3f06 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java @@ -25,6 +25,7 @@ import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.UnifiedOpts.NamedField; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import com.google.common.io.BaseEncoding; @@ -310,4 +311,12 @@ private static String crc32cEncode(int from) { static GrpcCallContext merge(@NonNull GrpcCallContext l, @NonNull GrpcCallContext r) { return (GrpcCallContext) l.merge(r); } + + static ImmutableList nullSafeList(@Nullable T t) { + if (t == null) { + return ImmutableList.of(); + } else { + return ImmutableList.of(t); + } + } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 363d9afdde..89a0986730 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -790,6 +790,9 @@ public byte[] load(StorageObject from, Map options) { .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); setEncryptionHeaders(getRequest.getRequestHeaders(), ENCRYPTION_KEY_PREFIX, options); + if (Option.RETURN_RAW_INPUT_STREAM.getBoolean(options) != null) { + getRequest.setReturnRawInputStream(Option.RETURN_RAW_INPUT_STREAM.getBoolean(options)); + } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.executeMedia().download(out); return out.toByteArray(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/ChunkedDownloadCallable.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/ChunkedDownloadCallable.java index 88c013a24a..f64e042be3 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/ChunkedDownloadCallable.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/ChunkedDownloadCallable.java @@ -58,9 +58,10 @@ final class ChunkedDownloadCallable implements Callable { @Override public DownloadSegment call() { long bytesCopied = -1L; - try (ReadChannel rc = storage.reader(originalBlob.getBlobId(), opts)) { - FileChannel wc = - FileChannel.open(destPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE); + try (ReadChannel rc = storage.reader(originalBlob.getBlobId(), opts); + FileChannel wc = + FileChannel.open(destPath, StandardOpenOption.WRITE, StandardOpenOption.CREATE)) { + rc.setChunkSize(0); rc.seek(startPosition); rc.limit(endPosition); wc.position(startPosition); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java index 0c7ec86a47..f2c8ba1776 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java @@ -52,14 +52,15 @@ public DownloadResult call() { Path path = TransferManagerUtils.createDestPath(parallelDownloadConfig, originalBlob); long bytesCopied = -1L; try (ReadChannel rc = - storage.reader( - BlobId.of(parallelDownloadConfig.getBucketName(), originalBlob.getName()), opts)) { - FileChannel wc = - FileChannel.open( - path, - StandardOpenOption.WRITE, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING); + storage.reader( + BlobId.of(parallelDownloadConfig.getBucketName(), originalBlob.getName()), opts); + FileChannel wc = + FileChannel.open( + path, + StandardOpenOption.WRITE, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING)) { + rc.setChunkSize(0); bytesCopied = ByteStreams.copy(rc, wc); if (originalBlob.getSize() != null) { if (bytesCopied != originalBlob.getSize()) { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java index f1c37b09d4..0f90c7524f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/DefaultRetryHandlingBehaviorTest.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.net.SocketException; import java.net.SocketTimeoutException; +import java.net.UnknownHostException; import java.security.cert.CertificateException; import java.util.Arrays; import java.util.Collection; @@ -342,6 +343,7 @@ enum ThrowableCategory { IO_EXCEPTION(new IOException("no retry")), AUTH_RETRYABLE_TRUE(new RetryableException(true)), AUTH_RETRYABLE_FALSE(new RetryableException(false)), + UNKNOWN_HOST_EXCEPTION(C.UNKNOWN_HOST_EXCEPTION), ; private final Throwable throwable; @@ -415,6 +417,8 @@ private static final class C { private static final MalformedJsonException GSON_MALFORMED_EXCEPTION = new MalformedJsonException("parse-exception"); private static final IOException IO_PREMATURE_EOF = new IOException("Premature EOF"); + private static final UnknownHostException UNKNOWN_HOST_EXCEPTION = + new UnknownHostException("fake.fake"); private static HttpResponseException newHttpResponseException( int httpStatusCode, String name) { @@ -1065,6 +1069,16 @@ private static ImmutableList getAllCases() { ThrowableCategory.AUTH_RETRYABLE_FALSE, HandlerCategory.NONIDEMPOTENT, ExpectRetry.NO, + Behavior.SAME), + new Case( + ThrowableCategory.UNKNOWN_HOST_EXCEPTION, + HandlerCategory.IDEMPOTENT, + ExpectRetry.YES, + Behavior.DEFAULT_MORE_PERMISSIBLE), + new Case( + ThrowableCategory.UNKNOWN_HOST_EXCEPTION, + HandlerCategory.NONIDEMPOTENT, + ExpectRetry.NO, Behavior.SAME)) .build(); } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java new file mode 100644 index 0000000000..f1a406ca6c --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcMetricsTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.SingleBackend; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@SingleBackend(Backend.PROD) +public class ITGrpcMetricsTest { + @Test + public void testGrpcMetrics() { + GrpcStorageOptions grpcStorageOptions = StorageOptions.grpc().build(); + assertThat( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.googleapis.com:443", "storage.googleapis.com")) + .isEqualTo("monitoring.googleapis.com:443"); + + SdkMeterProvider provider = + OpenTelemetryBootstrappingUtils.createMeterProvider( + "monitoring.googleapis.com:443", grpcStorageOptions.getProjectId(), false); + + /* + * SDKMeterProvider doesn't expose the relevant fields we want to test, but they are present in + * the String representation, so we'll check that instead. Most of the resources are auto-set, + * and will depend on environment, which could cause flakes to check. We're only responsible for + * setting the project ID, endpoint, and Histogram boundaries, so we'll just check those + */ + String result = provider.toString(); + + // What the project ID will be will depend on the environment, so we just make sure it's present + // and not null/empty + assertThat(result.contains("project_id")); + assertThat(result).doesNotContain("project_id=\"\""); + assertThat(result).doesNotContain("project_id=null"); + + // This is the check for the Seconds histogram boundary. We can't practically check for every + // boundary, + // but if *any* are present, that means they're different from the results and we successfully + // set them + assertThat(result).contains("1.2"); + + // This is the check for the Size boundary + assertThat(result).contains("131072"); + } + + @Test + public void testGrpcMetrics_universeDomain() { + assertThat("monitoring.my-universe-domain.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "storage.my-universe-domain.com:443", "my-universe-domain.com")); + } + + @Test + public void testGrpcMetrics_private() { + assertThat("private.googleapis.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "private.googleapis.com:443", null)); + } + + @Test + public void testGrpcMetrics_restricted() { + assertThat("restricted.googleapis.com:443") + .isEqualTo( + OpenTelemetryBootstrappingUtils.getCloudMonitoringEndpoint( + "restricted.googleapis.com:443", null)); + } +} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java index 4d6f0aa08e..3ac87cad0f 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGrpcTest.java @@ -251,12 +251,11 @@ public void lockBucketRetentionPolicy() { @Test public void testGrpcUniverseDomainMatchesHost() throws Exception { - Storage storage = - StorageOptions.grpc().setUniverseDomain("my-universe-domain.com").build().getService(); + StorageOptions storageOptions = + StorageOptions.grpc().setUniverseDomain("my-universe-domain.com").build(); assertAll( - () -> assertThat(storage.getOptions().getUniverseDomain().equals("my-universe-domain.com")), + () -> assertThat(storageOptions.getUniverseDomain().equals("my-universe-domain.com")), () -> - assertThat( - storage.getOptions().getHost().equals("https://storage.my-universe-domain.com"))); + assertThat(storageOptions.getHost().equals("https://storage.my-universe-domain.com"))); } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGzipTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGzipTest.java new file mode 100644 index 0000000000..5603762c47 --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITGzipTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.cloud.storage.TestUtils.xxd; +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobSourceOption; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.TestUtils; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.registry.Generator; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + backends = {Backend.PROD}, + transports = {Transport.HTTP, Transport.GRPC}) +public final class ITGzipTest { + + private static final ChecksummedTestContent CHECKSUMMED_TEST_CONTENT = + ChecksummedTestContent.of("Hello, to the world!!!"); + private static final byte[] GZIPPED_CONTENT = + TestUtils.gzipBytes(CHECKSUMMED_TEST_CONTENT.getBytes()); + + @Inject public Storage storage; + @Inject public BucketInfo bucket; + @Inject public Generator generator; + + @Test + public void noMetadata_noOption() { + String name = generator.randomObjectName(); + BlobInfo info = BlobInfo.newBuilder(bucket, name).build(); + Blob gen1 = storage.create(info, CHECKSUMMED_TEST_CONTENT.getBytes()); + assertThat(gen1.getContentEncoding()).isAnyOf(null, ""); // json null, grpc "" + byte[] actual = + storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.shouldReturnRawInputStream(true)); + assertThat(xxd(actual)).isEqualTo(xxd(CHECKSUMMED_TEST_CONTENT.getBytes())); + } + + @Test + public void yesMetadata_noOption() { + String name = generator.randomObjectName(); + BlobInfo info = BlobInfo.newBuilder(bucket, name).setContentEncoding("gzip").build(); + Blob gen1 = storage.create(info, GZIPPED_CONTENT); + assertThat(gen1.getContentEncoding()).isEqualTo("gzip"); + byte[] actual = + storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.shouldReturnRawInputStream(true)); + assertThat(xxd(actual)).isEqualTo(xxd(GZIPPED_CONTENT)); + } + + @Test + public void noMetadata_yesOption() { + String name = generator.randomObjectName(); + BlobInfo info = BlobInfo.newBuilder(bucket, name).build(); + Blob gen1 = + storage.create( + info, CHECKSUMMED_TEST_CONTENT.getBytes(), BlobTargetOption.disableGzipContent()); + assertThat(gen1.getContentEncoding()).isAnyOf(null, ""); // json null, grpc "" + byte[] actual = + storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.shouldReturnRawInputStream(true)); + assertThat(xxd(actual)).isEqualTo(xxd(CHECKSUMMED_TEST_CONTENT.getBytes())); + } + + @Test + public void yesMetadata_yesOption() { + String name = generator.randomObjectName(); + BlobInfo info = BlobInfo.newBuilder(bucket, name).setContentEncoding("gzip").build(); + Blob gen1 = storage.create(info, GZIPPED_CONTENT, BlobTargetOption.disableGzipContent()); + assertThat(gen1.getContentEncoding()).isEqualTo("gzip"); + byte[] actual = + storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.shouldReturnRawInputStream(true)); + assertThat(xxd(actual)).isEqualTo(xxd(GZIPPED_CONTENT)); + + byte[] actualUncompressed = + storage.readAllBytes(gen1.getBlobId(), BlobSourceOption.shouldReturnRawInputStream(false)); + assertThat(xxd(actualUncompressed)).isEqualTo(xxd(CHECKSUMMED_TEST_CONTENT.getBytes())); + } +} diff --git a/grpc-google-cloud-storage-control-v2/pom.xml b/grpc-google-cloud-storage-control-v2/pom.xml index 301d03a73f..cd85deba4b 100644 --- a/grpc-google-cloud-storage-control-v2/pom.xml +++ b/grpc-google-cloud-storage-control-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 grpc-google-cloud-storage-control-v2 GRPC library for google-cloud-storage com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/grpc-google-cloud-storage-v2/pom.xml b/grpc-google-cloud-storage-v2/pom.xml index 71ec72b7ea..2f788f3b84 100644 --- a/grpc-google-cloud-storage-v2/pom.xml +++ b/grpc-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha grpc-google-cloud-storage-v2 GRPC library for grpc-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/pom.xml b/pom.xml index 296f0041a8..eb1f4212d2 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-storage-parent pom - 2.40.1 + 2.41.0 Storage Parent https://github.com/googleapis/java-storage @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.32.0 + 3.33.0 @@ -54,7 +54,7 @@ UTF-8 github google-cloud-storage-parent - 3.25.0 + 3.31.0 @@ -65,10 +65,61 @@ ${google-cloud-shared-dependencies.version} pom import + + + + io.opentelemetry.semconv + opentelemetry-semconv + 1.25.0-alpha + + + com.google.cloud.opentelemetry + exporter-metrics + 0.31.0 + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + com.google.cloud.opentelemetry + detector-resources + 0.27.0-alpha + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + io.opentelemetry.instrumentation + opentelemetry-resources + 2.6.0-alpha + + + io.opentelemetry.semconv + opentelemetry-semconv + + + + + io.opentelemetry.contrib + opentelemetry-gcp-resources + 1.37.0-alpha - io.grpc - * + io.opentelemetry.semconv + opentelemetry-semconv @@ -76,12 +127,12 @@ com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 com.google.apis google-api-services-storage - v1-rev20240621-2.0.0 + v1-rev20240706-2.0.0 com.google.cloud @@ -98,32 +149,32 @@ com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.api.grpc grpc-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.api.grpc gapic-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha com.google.api.grpc grpc-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 com.google.cloud @@ -166,7 +217,7 @@ com.google.truth truth - 1.4.2 + 1.4.4 test diff --git a/proto-google-cloud-storage-control-v2/pom.xml b/proto-google-cloud-storage-control-v2/pom.xml index 4125fa00c9..b246d36a9f 100644 --- a/proto-google-cloud-storage-control-v2/pom.xml +++ b/proto-google-cloud-storage-control-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-control-v2 - 2.40.1 + 2.41.0 proto-google-cloud-storage-control-v2 Proto library for proto-google-cloud-storage-control-v2 com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/proto-google-cloud-storage-v2/pom.xml b/proto-google-cloud-storage-v2/pom.xml index 5cef367b2b..c5f35483eb 100644 --- a/proto-google-cloud-storage-v2/pom.xml +++ b/proto-google-cloud-storage-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-storage-v2 - 2.40.1-alpha + 2.41.0-alpha proto-google-cloud-storage-v2 PROTO library for proto-google-cloud-storage-v2 com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 diff --git a/renovate.json b/renovate.json index 23d26dbaa6..42380a2b69 100644 --- a/renovate.json +++ b/renovate.json @@ -46,6 +46,7 @@ "semanticCommitScope": null }, { + "groupName": "maven plugins", "packagePatterns": [ "^org.apache.maven", "^org.jacoco:", @@ -58,8 +59,15 @@ "semanticCommitScope": "deps" }, { + "groupName": "storage release dependencies", "packagePatterns": [ "^com.google.cloud:google-cloud-storage", + "^com.google.api.grpc:proto-google-cloud-storage-v2", + "^com.google.api.grpc:grpc-google-cloud-storage-v2", + "^com.google.api.grpc:gapic-google-cloud-storage-v2", + "^com.google.api.grpc:proto-google-cloud-storage-control-v2", + "^com.google.api.grpc:grpc-google-cloud-storage-control-v2", + "^com.google.cloud:google-cloud-storage-control", "^com.google.cloud:libraries-bom", "^com.google.cloud.samples:shared-configuration" ], @@ -67,8 +75,11 @@ "semanticCommitScope": "deps" }, { + "groupName": "test libraries", "packagePatterns": [ "^junit:junit", + "^org.junit", + "^net.jqwik", "^com.google.truth:truth", "^org.mockito:mockito-core", "^org.objenesis:objenesis", @@ -100,6 +111,16 @@ ], "semanticCommitType": "test", "semanticCommitScope": "deps" + }, + { + "groupName": "OpenTelemetry extended dependencies", + "packagePatterns": [ + "^io.opentelemetry.semconv:", + "^io.opentelemetry.instrumentation:", + "^io.opentelemetry.contrib:", + "^com.google.cloud.opentelemetry:", + "$com.google.cloud.opentelemetry:shared-resourcemapping" + ] } ], "semanticCommits": true, diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml index 1f54e417e1..d17cc74692 100644 --- a/samples/install-without-bom/pom.xml +++ b/samples/install-without-bom/pom.xml @@ -30,12 +30,12 @@ com.google.cloud google-cloud-storage - 2.40.0 + 2.40.1 com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 @@ -48,7 +48,7 @@ com.google.truth truth - 1.4.2 + 1.4.4 test @@ -72,7 +72,7 @@ com.google.cloud google-cloud-storage-control - 2.40.0 + 2.40.1 compile diff --git a/samples/native-image-sample/pom.xml b/samples/native-image-sample/pom.xml index 3391525543..2e15428c09 100644 --- a/samples/native-image-sample/pom.xml +++ b/samples/native-image-sample/pom.xml @@ -29,7 +29,7 @@ com.google.cloud libraries-bom - 26.42.0 + 26.43.0 pom import @@ -55,7 +55,7 @@ com.google.truth truth - 1.4.2 + 1.4.4 test @@ -67,7 +67,7 @@ com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 compile @@ -120,7 +120,7 @@ org.junit.vintage junit-vintage-engine - 5.10.2 + 5.10.3 test @@ -130,7 +130,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.3.0 + 3.3.1 **/*IT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index d8d687b89f..fd499ab676 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,12 +28,12 @@ com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 com.google.cloud google-cloud-storage-control - 2.40.1 + 2.41.0 compile @@ -46,7 +46,7 @@ com.google.truth truth - 1.4.2 + 1.4.4 test diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml index e6da8ec40d..de1be98b7c 100644 --- a/samples/snippets/pom.xml +++ b/samples/snippets/pom.xml @@ -31,7 +31,7 @@ com.google.cloud libraries-bom - 26.42.0 + 26.43.0 pom import @@ -58,7 +58,7 @@ com.google.truth truth - 1.4.2 + 1.4.4 test diff --git a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java index 847ec0d75a..60d918e842 100644 --- a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java +++ b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java @@ -26,6 +26,7 @@ import java.util.UUID; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/storage-shared-benchmarking/pom.xml b/storage-shared-benchmarking/pom.xml index b03c200be6..2dba989625 100644 --- a/storage-shared-benchmarking/pom.xml +++ b/storage-shared-benchmarking/pom.xml @@ -10,7 +10,7 @@ com.google.cloud google-cloud-storage-parent - 2.40.1 + 2.41.0 @@ -31,7 +31,7 @@ com.google.cloud google-cloud-storage - 2.40.1 + 2.41.0 tests diff --git a/versions.txt b/versions.txt index e316bf757c..fdc8027457 100644 --- a/versions.txt +++ b/versions.txt @@ -1,10 +1,10 @@ # Format: # module:released-version:current-version -google-cloud-storage:2.40.1:2.40.1 -gapic-google-cloud-storage-v2:2.40.1-alpha:2.40.1-alpha -grpc-google-cloud-storage-v2:2.40.1-alpha:2.40.1-alpha -proto-google-cloud-storage-v2:2.40.1-alpha:2.40.1-alpha -google-cloud-storage-control:2.40.1:2.40.1 -proto-google-cloud-storage-control-v2:2.40.1:2.40.1 -grpc-google-cloud-storage-control-v2:2.40.1:2.40.1 +google-cloud-storage:2.41.0:2.41.0 +gapic-google-cloud-storage-v2:2.41.0-alpha:2.41.0-alpha +grpc-google-cloud-storage-v2:2.41.0-alpha:2.41.0-alpha +proto-google-cloud-storage-v2:2.41.0-alpha:2.41.0-alpha +google-cloud-storage-control:2.41.0:2.41.0 +proto-google-cloud-storage-control-v2:2.41.0:2.41.0 +grpc-google-cloud-storage-control-v2:2.41.0:2.41.0