From 02a972d35fae6d05edfb26381f6a71e3b8f59d6d Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Tue, 24 Sep 2024 20:40:27 -0700
Subject: [PATCH 01/17] feat: add integration test for universe domain (#1346)

---
 .kokoro/build.sh                 |  8 +++++
 .kokoro/presubmit/system-3.8.cfg |  6 ++++
 owlbot.py                        | 11 ++++++-
 tests/system/_helpers.py         |  4 +++
 tests/system/conftest.py         | 53 ++++++++++++++++++++++++++++++++
 tests/system/test_client.py      | 27 ++++++++++++++++
 6 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/.kokoro/build.sh b/.kokoro/build.sh
index 5ac9f8a51..fdc6d0271 100755
--- a/.kokoro/build.sh
+++ b/.kokoro/build.sh
@@ -34,6 +34,14 @@ export API_VERSION_OVERRIDE
 export DUAL_REGION_LOC_1
 export DUAL_REGION_LOC_2
 
+# Setup universe domain testing needed environment variables.
+export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential)
+export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain)
+export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id)
+export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location)
+
+
+
 # Debug: show build environment
 env | grep KOKORO
 
diff --git a/.kokoro/presubmit/system-3.8.cfg b/.kokoro/presubmit/system-3.8.cfg
index f4bcee3db..6d3603eed 100644
--- a/.kokoro/presubmit/system-3.8.cfg
+++ b/.kokoro/presubmit/system-3.8.cfg
@@ -4,4 +4,10 @@
 env_vars: {
     key: "NOX_SESSION"
     value: "system-3.8"
+}
+
+# Credentials needed to test universe domain.
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "client-library-test-universe-domain-credential"
 }
\ No newline at end of file
diff --git a/owlbot.py b/owlbot.py
index a06ae8cc4..61871e3e4 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -46,6 +46,7 @@
         "noxfile.py",
         "CONTRIBUTING.rst",
         "README.rst",
+        ".kokoro/presubmit/system-3.8.cfg",
         ".kokoro/samples/python3.6", # remove python 3.6 support
         ".github/blunderbuss.yml", # blunderbuss assignment to python squad
         ".github/workflows", # exclude gh actions as credentials are needed for tests
@@ -66,7 +67,15 @@
 
 # Export dual region locations
 export DUAL_REGION_LOC_1
-export DUAL_REGION_LOC_2""")
+export DUAL_REGION_LOC_2
+
+# Setup universe domain testing needed environment variables.
+export TEST_UNIVERSE_DOMAIN_CREDENTIAL=$(realpath ${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential)
+export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain)
+export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id)
+export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location)
+
+""")
 
 s.replace(
     ".coveragerc",
diff --git a/tests/system/_helpers.py b/tests/system/_helpers.py
index a044c4ca8..7274610a8 100644
--- a/tests/system/_helpers.py
+++ b/tests/system/_helpers.py
@@ -31,6 +31,10 @@
 
 user_project = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT")
 testing_mtls = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
+test_universe_domain = os.getenv("TEST_UNIVERSE_DOMAIN")
+test_universe_project_id = os.getenv("TEST_UNIVERSE_PROJECT_ID")
+test_universe_location = os.getenv("TEST_UNIVERSE_LOCATION")
+test_universe_domain_credential = os.getenv("TEST_UNIVERSE_DOMAIN_CREDENTIAL")
 signing_blob_content = b"This time for sure, Rocky!"
 is_api_endpoint_override = (
     _get_default_storage_base_url() != "https://storage.googleapis.com"
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index a97b98648..4ec56176d 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -331,3 +331,56 @@ def keyring(storage_client, kms_bucket, kms_client):
         except exceptions.NotFound:
             key = {"purpose": purpose}
             kms_client.create_crypto_key(keyring_path, key_name, key)
+
+
+@pytest.fixture(scope="function")
+def test_universe_domain():
+    if _helpers.test_universe_domain is None:
+        pytest.skip("TEST_UNIVERSE_DOMAIN not set in environment.")
+    return _helpers.test_universe_domain
+
+
+@pytest.fixture(scope="function")
+def test_universe_project_id():
+    if _helpers.test_universe_project_id is None:
+        pytest.skip("TEST_UNIVERSE_PROJECT_ID not set in environment.")
+    return _helpers.test_universe_project_id
+
+
+@pytest.fixture(scope="function")
+def test_universe_location():
+    if _helpers.test_universe_location is None:
+        pytest.skip("TEST_UNIVERSE_LOCATION not set in environment.")
+    return _helpers.test_universe_location
+
+
+@pytest.fixture(scope="function")
+def test_universe_domain_credential():
+    if _helpers.test_universe_domain_credential is None:
+        pytest.skip("TEST_UNIVERSE_DOMAIN_CREDENTIAL not set in environment.")
+    return _helpers.test_universe_domain_credential
+
+
+@pytest.fixture(scope="function")
+def universe_domain_credential(test_universe_domain_credential):
+    from google.oauth2 import service_account
+
+    return service_account.Credentials.from_service_account_file(
+        test_universe_domain_credential
+    )
+
+
+@pytest.fixture(scope="function")
+def universe_domain_client(
+    test_universe_domain, test_universe_project_id, universe_domain_credential
+):
+    from google.cloud.storage import Client
+
+    client_options = {"universe_domain": test_universe_domain}
+    ud_storage_client = Client(
+        project=test_universe_project_id,
+        credentials=universe_domain_credential,
+        client_options=client_options,
+    )
+    with contextlib.closing(ud_storage_client):
+        yield ud_storage_client
diff --git a/tests/system/test_client.py b/tests/system/test_client.py
index 70f341851..baf4556b7 100644
--- a/tests/system/test_client.py
+++ b/tests/system/test_client.py
@@ -184,3 +184,30 @@ def test_download_blob_to_file_w_etag(
         if_etag_match=blob.etag,
     )
     assert buffer.getvalue() == payload
+
+
+def test_client_universe_domain(
+    universe_domain_client,
+    test_universe_location,
+    buckets_to_delete,
+    blobs_to_delete,
+):
+    bucket_name = _helpers.unique_name("gcp-systest-ud")
+    ud_bucket = universe_domain_client.create_bucket(
+        bucket_name, location=test_universe_location
+    )
+    buckets_to_delete.append(ud_bucket)
+
+    blob_name = _helpers.unique_name("gcp-systest-ud")
+    blob = ud_bucket.blob(blob_name)
+    payload = b"The quick brown fox jumps over the lazy dog"
+    blob.upload_from_string(payload)
+    blobs_to_delete.append(blob)
+
+    with tempfile.NamedTemporaryFile() as temp_f:
+        with open(temp_f.name, "wb") as file_obj:
+            universe_domain_client.download_blob_to_file(blob, file_obj)
+        with open(temp_f.name, "rb") as file_obj:
+            stored_contents = file_obj.read()
+
+    assert stored_contents == payload

From e3cfc4786209c77e3c879c9ff2978f4884a0d677 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Wed, 25 Sep 2024 17:13:50 -0700
Subject: [PATCH 02/17] chore: update secret manager in kokoro (#1350)

---
 .kokoro/continuous/common.cfg | 6 ++++++
 .kokoro/release/common.cfg    | 2 +-
 owlbot.py                     | 7 +++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg
index 51201dfab..fd7c8cc69 100644
--- a/.kokoro/continuous/common.cfg
+++ b/.kokoro/continuous/common.cfg
@@ -25,3 +25,9 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/python-storage/.kokoro/build.sh"
 }
+
+# Credentials needed to test universe domain.
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "client-library-test-universe-domain-credential"
+}
diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg
index a11679f43..3464807cf 100644
--- a/.kokoro/release/common.cfg
+++ b/.kokoro/release/common.cfg
@@ -36,7 +36,7 @@ before_action {
 # Tokens needed to report release status back to GitHub
 env_vars: {
   key: "SECRET_MANAGER_KEYS"
-  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"
+  value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem, client-library-test-universe-domain-credential"
 }
 
 # Store the packages we uploaded to PyPI.  That way, we have a record of exactly
diff --git a/owlbot.py b/owlbot.py
index 61871e3e4..93e8ceb1c 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -46,6 +46,7 @@
         "noxfile.py",
         "CONTRIBUTING.rst",
         "README.rst",
+        ".kokoro/continuous/common.cfg",
         ".kokoro/presubmit/system-3.8.cfg",
         ".kokoro/samples/python3.6", # remove python 3.6 support
         ".github/blunderbuss.yml", # blunderbuss assignment to python squad
@@ -83,6 +84,12 @@
     """omit =
   .nox/*""")
 
+s.replace(
+    ".kokoro/release/common.cfg",
+    'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"',
+    'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem, client-library-test-universe-domain-credential"'
+)
+
 python.py_samples(skip_readmes=True)
 
 s.shell.run(["nox", "-s", "blacken"], hide_output=False)

From 76316438f581b21bc229b13c5f1fb545f158dd77 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Fri, 27 Sep 2024 10:05:47 -0700
Subject: [PATCH 03/17] chore: update secret manager in kokoro (#1352)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore: update secret manager in kokoro

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
---
 .kokoro/continuous/common.cfg     | 6 ------
 .kokoro/continuous/continuous.cfg | 8 +++++++-
 owlbot.py                         | 2 +-
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg
index fd7c8cc69..51201dfab 100644
--- a/.kokoro/continuous/common.cfg
+++ b/.kokoro/continuous/common.cfg
@@ -25,9 +25,3 @@ env_vars: {
     key: "TRAMPOLINE_BUILD_FILE"
     value: "github/python-storage/.kokoro/build.sh"
 }
-
-# Credentials needed to test universe domain.
-env_vars: {
-  key: "SECRET_MANAGER_KEYS"
-  value: "client-library-test-universe-domain-credential"
-}
diff --git a/.kokoro/continuous/continuous.cfg b/.kokoro/continuous/continuous.cfg
index 8f43917d9..0cfe6b6e2 100644
--- a/.kokoro/continuous/continuous.cfg
+++ b/.kokoro/continuous/continuous.cfg
@@ -1 +1,7 @@
-# Format: //devtools/kokoro/config/proto/build.proto
\ No newline at end of file
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Credentials needed to test universe domain.
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "client-library-test-universe-domain-credential"
+}
diff --git a/owlbot.py b/owlbot.py
index 93e8ceb1c..8bd9de751 100644
--- a/owlbot.py
+++ b/owlbot.py
@@ -46,7 +46,7 @@
         "noxfile.py",
         "CONTRIBUTING.rst",
         "README.rst",
-        ".kokoro/continuous/common.cfg",
+        ".kokoro/continuous/continuous.cfg",
         ".kokoro/presubmit/system-3.8.cfg",
         ".kokoro/samples/python3.6", # remove python 3.6 support
         ".github/blunderbuss.yml", # blunderbuss assignment to python squad

From bebd97afaf0365693e97f88521e90dc60776e37f Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Fri, 27 Sep 2024 16:40:50 -0700
Subject: [PATCH 04/17] tests: unflake ud system test to only run in prod and
 hmac sample test (#1353)

* test: test universe domain client only in prod

* unflake hmac snippet test
---
 samples/snippets/hmac_samples_test.py | 5 ++++-
 tests/system/test_client.py           | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/samples/snippets/hmac_samples_test.py b/samples/snippets/hmac_samples_test.py
index 60eba2401..988b40305 100644
--- a/samples/snippets/hmac_samples_test.py
+++ b/samples/snippets/hmac_samples_test.py
@@ -64,7 +64,10 @@ def new_hmac_key():
     if not hmac_key.state == "INACTIVE":
         hmac_key.state = "INACTIVE"
         hmac_key.update()
-    hmac_key.delete()
+    try:
+        hmac_key.delete()
+    except google.api_core.exceptions.BadRequest:
+        pass
 
 
 def test_list_keys(capsys, new_hmac_key):
diff --git a/tests/system/test_client.py b/tests/system/test_client.py
index baf4556b7..c1b3858f2 100644
--- a/tests/system/test_client.py
+++ b/tests/system/test_client.py
@@ -186,6 +186,10 @@ def test_download_blob_to_file_w_etag(
     assert buffer.getvalue() == payload
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Credentials not yet supported in preprod testing.",
+)
 def test_client_universe_domain(
     universe_domain_client,
     test_universe_location,

From 1963de91f1e0e0fa331a59906c232738c8ebeaf3 Mon Sep 17 00:00:00 2001
From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com>
Date: Fri, 27 Sep 2024 18:24:10 -0700
Subject: [PATCH 05/17] build(python): release script update (#1345)

Source-Link: https://github.com/googleapis/synthtool/commit/71a72973dddbc66ea64073b53eda49f0d22e0942
Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: cojenco <cathyo@google.com>
---
 .github/.OwlBot.lock.yaml      |  4 ++--
 .kokoro/docker/docs/Dockerfile |  9 ++++-----
 .kokoro/publish-docs.sh        | 20 ++++++++++----------
 .kokoro/release.sh             |  2 +-
 .kokoro/release/common.cfg     |  2 +-
 5 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml
index f30cb3775..597e0c326 100644
--- a/.github/.OwlBot.lock.yaml
+++ b/.github/.OwlBot.lock.yaml
@@ -13,5 +13,5 @@
 # limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:52210e0e0559f5ea8c52be148b33504022e1faef4e95fbe4b32d68022af2fa7e
-# created: 2024-07-08T19:25:35.862283192Z
+  digest: sha256:e8dcfd7cbfd8beac3a3ff8d3f3185287ea0625d859168cc80faccfc9a7a00455
+# created: 2024-09-16T21:04:09.091105552Z
diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile
index 5205308b3..e5410e296 100644
--- a/.kokoro/docker/docs/Dockerfile
+++ b/.kokoro/docker/docs/Dockerfile
@@ -72,19 +72,18 @@ RUN tar -xvf Python-3.10.14.tgz
 RUN ./Python-3.10.14/configure --enable-optimizations
 RUN make altinstall
 
-RUN python3.10 -m venv /venv
-ENV PATH /venv/bin:$PATH
+ENV PATH /usr/local/bin/python3.10:$PATH
 
 ###################### Install pip
 RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
-  && python3 /tmp/get-pip.py \
+  && python3.10 /tmp/get-pip.py \
   && rm /tmp/get-pip.py
 
 # Test pip
-RUN python3 -m pip
+RUN python3.10 -m pip
 
 # Install build requirements
 COPY requirements.txt /requirements.txt
-RUN python3 -m pip install --require-hashes -r requirements.txt
+RUN python3.10 -m pip install --require-hashes -r requirements.txt
 
 CMD ["python3.10"]
diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh
index 38f083f05..233205d58 100755
--- a/.kokoro/publish-docs.sh
+++ b/.kokoro/publish-docs.sh
@@ -21,18 +21,18 @@ export PYTHONUNBUFFERED=1
 export PATH="${HOME}/.local/bin:${PATH}"
 
 # Install nox
-python3 -m pip install --require-hashes -r .kokoro/requirements.txt
-python3 -m nox --version
+python3.10 -m pip install --require-hashes -r .kokoro/requirements.txt
+python3.10 -m nox --version
 
 # build docs
 nox -s docs
 
 # create metadata
-python3 -m docuploader create-metadata \
+python3.10 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
-  --version=$(python3 setup.py --version) \
+  --version=$(python3.10 setup.py --version) \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
-  --distribution-name=$(python3 setup.py --name) \
+  --distribution-name=$(python3.10 setup.py --name) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
@@ -40,18 +40,18 @@ python3 -m docuploader create-metadata \
 cat docs.metadata
 
 # upload docs
-python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
+python3.10 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
 
 
 # docfx yaml files
 nox -s docfx
 
 # create metadata.
-python3 -m docuploader create-metadata \
+python3.10 -m docuploader create-metadata \
   --name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
-  --version=$(python3 setup.py --version) \
+  --version=$(python3.10 setup.py --version) \
   --language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
-  --distribution-name=$(python3 setup.py --name) \
+  --distribution-name=$(python3.10 setup.py --name) \
   --product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
   --github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
   --issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
@@ -59,4 +59,4 @@ python3 -m docuploader create-metadata \
 cat docs.metadata
 
 # upload docs
-python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
+python3.10 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"
diff --git a/.kokoro/release.sh b/.kokoro/release.sh
index c5fc555e1..a15b26b59 100755
--- a/.kokoro/release.sh
+++ b/.kokoro/release.sh
@@ -23,7 +23,7 @@ python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /
 export PYTHONUNBUFFERED=1
 
 # Move into the package, build the distribution and upload.
-TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1")
+TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-2")
 cd github/python-storage
 python3 setup.py sdist bdist_wheel
 twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/*
diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg
index 3464807cf..17918dc86 100644
--- a/.kokoro/release/common.cfg
+++ b/.kokoro/release/common.cfg
@@ -28,7 +28,7 @@ before_action {
   fetch_keystore {
     keystore_resource {
       keystore_config_id: 73713
-      keyname: "google-cloud-pypi-token-keystore-1"
+      keyname: "google-cloud-pypi-token-keystore-2"
     }
   }
 }

From 8edbec1b1d6e3165a196d3ff082fb65a2b697bd5 Mon Sep 17 00:00:00 2001
From: Mend Renovate <bot@renovateapp.com>
Date: Mon, 30 Sep 2024 18:13:30 +0200
Subject: [PATCH 06/17] chore(deps): update all dependencies (#1329)

Co-authored-by: Holt Skinner <13262395+holtskinner@users.noreply.github.com>
Co-authored-by: cojenco <cathyo@google.com>
---
 samples/snippets/requirements-test.txt | 2 +-
 samples/snippets/requirements.txt      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt
index 054670d8b..68fb21c1c 100644
--- a/samples/snippets/requirements-test.txt
+++ b/samples/snippets/requirements-test.txt
@@ -1,4 +1,4 @@
 pytest===7.4.4; python_version == '3.7'
-pytest==8.2.2; python_version >= '3.8'
+pytest==8.3.2; python_version >= '3.8'
 mock==5.1.0
 backoff==2.2.1
diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt
index c5b45a4a2..54f6f7806 100644
--- a/samples/snippets/requirements.txt
+++ b/samples/snippets/requirements.txt
@@ -1,5 +1,5 @@
-google-cloud-pubsub==2.22.0
-google-cloud-storage==2.17.0
+google-cloud-pubsub==2.23.0
+google-cloud-storage==2.18.0
 pandas===1.3.5; python_version == '3.7'
 pandas===2.0.3; python_version == '3.8'
 pandas==2.2.2; python_version >= '3.9'

From cea20e2362b463746344524eb70416044fe3b902 Mon Sep 17 00:00:00 2001
From: Mend Renovate <bot@renovateapp.com>
Date: Tue, 1 Oct 2024 01:59:47 +0200
Subject: [PATCH 07/17] chore(deps): update all dependencies (#1354)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* chore(deps): update all dependencies

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
---
 samples/snippets/requirements-test.txt | 2 +-
 samples/snippets/requirements.txt      | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt
index 68fb21c1c..a1dda582f 100644
--- a/samples/snippets/requirements-test.txt
+++ b/samples/snippets/requirements-test.txt
@@ -1,4 +1,4 @@
 pytest===7.4.4; python_version == '3.7'
-pytest==8.3.2; python_version >= '3.8'
+pytest==8.3.3; python_version >= '3.8'
 mock==5.1.0
 backoff==2.2.1
diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt
index 54f6f7806..4eb727236 100644
--- a/samples/snippets/requirements.txt
+++ b/samples/snippets/requirements.txt
@@ -1,5 +1,5 @@
-google-cloud-pubsub==2.23.0
-google-cloud-storage==2.18.0
+google-cloud-pubsub==2.25.1
+google-cloud-storage==2.18.2
 pandas===1.3.5; python_version == '3.7'
 pandas===2.0.3; python_version == '3.8'
-pandas==2.2.2; python_version >= '3.9'
+pandas==2.2.3; python_version >= '3.9'

From 8ec02c0e656a4e6786f256798f4b93b95b50acec Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Fri, 4 Oct 2024 11:26:21 -0700
Subject: [PATCH 08/17] fix: allow signed post policy v4 with service account
 and token (#1356)

---
 google/cloud/storage/client.py |  7 +++--
 tests/system/test__signing.py  | 49 ++++++++++++++++++++++++++++++++++
 tests/unit/test_client.py      | 44 ++++++++++++++++++++++++++++++
 3 files changed, 98 insertions(+), 2 deletions(-)

diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py
index b21ef7cef..bc2d1147e 100644
--- a/google/cloud/storage/client.py
+++ b/google/cloud/storage/client.py
@@ -1724,13 +1724,16 @@ def generate_signed_post_policy_v4(
             )
 
         credentials = self._credentials if credentials is None else credentials
-        ensure_signed_credentials(credentials)
+        client_email = service_account_email
+        if not access_token or not service_account_email:
+            ensure_signed_credentials(credentials)
+            client_email = credentials.signer_email
 
         # prepare policy conditions and fields
         timestamp, datestamp = get_v4_now_dtstamps()
 
         x_goog_credential = "{email}/{datestamp}/auto/storage/goog4_request".format(
-            email=credentials.signer_email, datestamp=datestamp
+            email=client_email, datestamp=datestamp
         )
         required_conditions = [
             {"bucket": bucket_name},
diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py
index 94930739e..8bcc46abc 100644
--- a/tests/system/test__signing.py
+++ b/tests/system/test__signing.py
@@ -415,6 +415,55 @@ def test_generate_signed_post_policy_v4(
     assert blob.download_as_bytes() == payload
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Test does not yet support endpoint override",
+)
+def test_generate_signed_post_policy_v4_access_token_sa_email(
+    storage_client, signing_bucket, blobs_to_delete, service_account, no_mtls
+):
+    client = iam_credentials_v1.IAMCredentialsClient()
+    service_account_email = service_account.service_account_email
+    name = path_template.expand(
+        "projects/{project}/serviceAccounts/{service_account}",
+        project="-",
+        service_account=service_account_email,
+    )
+    scope = [
+        "https://www.googleapis.com/auth/devstorage.read_write",
+        "https://www.googleapis.com/auth/iam",
+    ]
+    response = client.generate_access_token(name=name, scope=scope)
+
+    now = _NOW(_UTC).replace(tzinfo=None)
+    blob_name = "post_policy_obj_email2.txt"
+    payload = b"DEADBEEF"
+    with open(blob_name, "wb") as f:
+        f.write(payload)
+    policy = storage_client.generate_signed_post_policy_v4(
+        signing_bucket.name,
+        blob_name,
+        conditions=[
+            {"bucket": signing_bucket.name},
+            ["starts-with", "$Content-Type", "text/pla"],
+        ],
+        expiration=now + datetime.timedelta(hours=1),
+        fields={"content-type": "text/plain"},
+        service_account_email=service_account_email,
+        access_token=response.access_token,
+    )
+    with open(blob_name, "r") as f:
+        files = {"file": (blob_name, f)}
+        response = requests.post(policy["url"], data=policy["fields"], files=files)
+
+    os.remove(blob_name)
+    assert response.status_code == 204
+
+    blob = signing_bucket.get_blob(blob_name)
+    blobs_to_delete.append(blob)
+    assert blob.download_as_bytes() == payload
+
+
 def test_generate_signed_post_policy_v4_invalid_field(
     storage_client, buckets_to_delete, blobs_to_delete, service_account, no_mtls
 ):
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index b664e701d..5eb339acb 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -2849,6 +2849,50 @@ def test_get_signed_policy_v4_with_access_token(self):
         self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN)
         self.assertEqual(fields["policy"], EXPECTED_POLICY)
 
+    def test_get_signed_policy_v4_with_access_token_sa_email(self):
+        import datetime
+
+        BUCKET_NAME = "bucket-name"
+        BLOB_NAME = "object-name"
+        EXPECTED_SIGN = "0c4003044105"
+        EXPECTED_POLICY = "eyJjb25kaXRpb25zIjpbeyJidWNrZXQiOiJidWNrZXQtbmFtZSJ9LHsiYWNsIjoicHJpdmF0ZSJ9LFsic3RhcnRzLXdpdGgiLCIkQ29udGVudC1UeXBlIiwidGV4dC9wbGFpbiJdLHsiYnVja2V0IjoiYnVja2V0LW5hbWUifSx7ImtleSI6Im9iamVjdC1uYW1lIn0seyJ4LWdvb2ctZGF0ZSI6IjIwMjAwMzEyVDExNDcxNloifSx7IngtZ29vZy1jcmVkZW50aWFsIjoidGVzdEBtYWlsLmNvbS8yMDIwMDMxMi9hdXRvL3N0b3JhZ2UvZ29vZzRfcmVxdWVzdCJ9LHsieC1nb29nLWFsZ29yaXRobSI6IkdPT0c0LVJTQS1TSEEyNTYifV0sImV4cGlyYXRpb24iOiIyMDIwLTAzLTI2VDAwOjAwOjEwWiJ9"
+
+        project = "PROJECT"
+        credentials = _make_credentials(project=project)
+        client = self._make_one(credentials=credentials)
+
+        dtstamps_patch, now_patch, expire_secs_patch = _time_functions_patches()
+        with dtstamps_patch, now_patch, expire_secs_patch:
+            with mock.patch(
+                "google.cloud.storage.client._sign_message", return_value=b"DEADBEEF"
+            ):
+                policy = client.generate_signed_post_policy_v4(
+                    BUCKET_NAME,
+                    BLOB_NAME,
+                    expiration=datetime.datetime(2020, 3, 12),
+                    conditions=[
+                        {"bucket": BUCKET_NAME},
+                        {"acl": "private"},
+                        ["starts-with", "$Content-Type", "text/plain"],
+                    ],
+                    service_account_email="test@mail.com",
+                    access_token="token",
+                )
+        self.assertEqual(
+            policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME + "/"
+        )
+        fields = policy["fields"]
+
+        self.assertEqual(fields["key"], BLOB_NAME)
+        self.assertEqual(fields["x-goog-algorithm"], "GOOG4-RSA-SHA256")
+        self.assertEqual(fields["x-goog-date"], "20200312T114716Z")
+        self.assertEqual(
+            fields["x-goog-credential"],
+            "test@mail.com/20200312/auto/storage/goog4_request",
+        )
+        self.assertEqual(fields["x-goog-signature"], EXPECTED_SIGN)
+        self.assertEqual(fields["policy"], EXPECTED_POLICY)
+
 
 class Test__item_to_bucket(unittest.TestCase):
     def _call_fut(self, iterator, item):

From 42392ef8e38527ce4e50454cdd357425b3f57c87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rafa=C5=82=20Harabie=C5=84?= <rafalh92@outlook.com>
Date: Wed, 9 Oct 2024 18:53:12 +0200
Subject: [PATCH 09/17] fix: do not spam the log with checksum related INFO
 messages when downloading using transfer_manager (#1357)

* fix: do not spam the log with checksum related INFO messages when downloading using transfer_manager

`download_chunks_concurrently` function does not allow to set `checksum` field in `download_kwargs`. It also does not set it on its own so it takes the default value of `"md5"` (see `Blob._prep_and_do_download`). Because ranged downloads do not return checksums it results in a lot of INFO messages (tens/hundreds):
```
INFO google.resumable_media._helpers - No MD5 checksum was returned from the service while downloading ...
(which happens for composite objects), so client-side content integrity checking is not being performed.
```
To fix it set the `checksum` field to `None` which means no checksum checking for individual chunks. Note that `transfer_manager` has its own checksum checking logic (enabled by `crc32c_checksum` argument)

* fix tests
---
 google/cloud/storage/transfer_manager.py | 2 ++
 tests/unit/test_transfer_manager.py      | 7 +------
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/google/cloud/storage/transfer_manager.py b/google/cloud/storage/transfer_manager.py
index 1b48cd9cf..15325df56 100644
--- a/google/cloud/storage/transfer_manager.py
+++ b/google/cloud/storage/transfer_manager.py
@@ -885,6 +885,8 @@ def download_chunks_concurrently(
             "'checksum' is in download_kwargs, but is not supported because sliced downloads have a different checksum mechanism from regular downloads. Use the 'crc32c_checksum' argument on download_chunks_concurrently instead."
         )
 
+    download_kwargs = download_kwargs.copy()
+    download_kwargs["checksum"] = None
     download_kwargs["command"] = "tm.download_sharded"
 
     # We must know the size and the generation of the blob.
diff --git a/tests/unit/test_transfer_manager.py b/tests/unit/test_transfer_manager.py
index cee83ba54..09969b5eb 100644
--- a/tests/unit/test_transfer_manager.py
+++ b/tests/unit/test_transfer_manager.py
@@ -606,6 +606,7 @@ def test_download_chunks_concurrently():
 
     expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
     expected_download_kwargs["command"] = "tm.download_sharded"
+    expected_download_kwargs["checksum"] = None
 
     with mock.patch("google.cloud.storage.transfer_manager.open", mock.mock_open()):
         result = transfer_manager.download_chunks_concurrently(
@@ -636,9 +637,6 @@ def test_download_chunks_concurrently_with_crc32c():
     blob_mock.size = len(BLOB_CONTENTS)
     blob_mock.crc32c = "eOVVVw=="
 
-    expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
-    expected_download_kwargs["command"] = "tm.download_sharded"
-
     def write_to_file(f, *args, **kwargs):
         f.write(BLOB_CHUNK)
 
@@ -664,9 +662,6 @@ def test_download_chunks_concurrently_with_crc32c_failure():
     blob_mock.size = len(BLOB_CONTENTS)
     blob_mock.crc32c = "invalid"
 
-    expected_download_kwargs = EXPECTED_DOWNLOAD_KWARGS.copy()
-    expected_download_kwargs["command"] = "tm.download_sharded"
-
     def write_to_file(f, *args, **kwargs):
         f.write(BLOB_CHUNK)
 

From ab94efda83f68c974ec91d6b869b09047501031a Mon Sep 17 00:00:00 2001
From: Andrew Gorcester <gorcester@google.com>
Date: Tue, 29 Oct 2024 12:40:32 -0700
Subject: [PATCH 10/17] Feat: Add restore_bucket and handling for soft-deleted
 buckets (#1365)

---
 google/cloud/storage/_helpers.py |   3 +
 google/cloud/storage/bucket.py   |  66 +++++++++++++++-
 google/cloud/storage/client.py   | 131 ++++++++++++++++++++++++++++---
 tests/system/test_client.py      |  44 +++++++++++
 tests/unit/test_bucket.py        |  69 +++++++++++++++-
 tests/unit/test_client.py        | 117 ++++++++++++++++++++++++++-
 6 files changed, 412 insertions(+), 18 deletions(-)

diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py
index b90bf4eb2..3793a95f2 100644
--- a/google/cloud/storage/_helpers.py
+++ b/google/cloud/storage/_helpers.py
@@ -293,6 +293,9 @@ def reload(
         )
         if soft_deleted is not None:
             query_params["softDeleted"] = soft_deleted
+            # Soft delete reload requires a generation, even for targets
+            # that don't include them in default query params (buckets).
+            query_params["generation"] = self.generation
         headers = self._encryption_headers()
         _add_etag_match_headers(
             headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py
index ad1d0de5d..7cea15f4e 100644
--- a/google/cloud/storage/bucket.py
+++ b/google/cloud/storage/bucket.py
@@ -626,6 +626,10 @@ class Bucket(_PropertyMixin):
     :type user_project: str
     :param user_project: (Optional) the project ID to be billed for API
                          requests made via this instance.
+
+    :type generation: int
+    :param generation: (Optional) If present, selects a specific revision of
+                       this bucket.
     """
 
     _MAX_OBJECTS_FOR_ITERATION = 256
@@ -659,7 +663,7 @@ class Bucket(_PropertyMixin):
     )
     """Allowed values for :attr:`location_type`."""
 
-    def __init__(self, client, name=None, user_project=None):
+    def __init__(self, client, name=None, user_project=None, generation=None):
         """
         property :attr:`name`
             Get the bucket's name.
@@ -672,6 +676,9 @@ def __init__(self, client, name=None, user_project=None):
         self._label_removals = set()
         self._user_project = user_project
 
+        if generation is not None:
+            self._properties["generation"] = generation
+
     def __repr__(self):
         return f"<Bucket: {self.name}>"
 
@@ -726,6 +733,50 @@ def user_project(self):
         """
         return self._user_project
 
+    @property
+    def generation(self):
+        """Retrieve the generation for the bucket.
+
+        :rtype: int or ``NoneType``
+        :returns: The generation of the bucket or ``None`` if the bucket's
+                  resource has not been loaded from the server.
+        """
+        generation = self._properties.get("generation")
+        if generation is not None:
+            return int(generation)
+
+    @property
+    def soft_delete_time(self):
+        """If this bucket has been soft-deleted, returns the time at which it became soft-deleted.
+
+        :rtype: :class:`datetime.datetime` or ``NoneType``
+        :returns:
+            (readonly) The time that the bucket became soft-deleted.
+             Note this property is only set for soft-deleted buckets.
+        """
+        soft_delete_time = self._properties.get("softDeleteTime")
+        if soft_delete_time is not None:
+            return _rfc3339_nanos_to_datetime(soft_delete_time)
+
+    @property
+    def hard_delete_time(self):
+        """If this bucket has been soft-deleted, returns the time at which it will be permanently deleted.
+
+        :rtype: :class:`datetime.datetime` or ``NoneType``
+        :returns:
+            (readonly) The time that the bucket will be permanently deleted.
+            Note this property is only set for soft-deleted buckets.
+        """
+        hard_delete_time = self._properties.get("hardDeleteTime")
+        if hard_delete_time is not None:
+            return _rfc3339_nanos_to_datetime(hard_delete_time)
+
+    @property
+    def _query_params(self):
+        """Default query parameters."""
+        params = super()._query_params
+        return params
+
     @classmethod
     def from_string(cls, uri, client=None):
         """Get a constructor for bucket object by URI.
@@ -1045,6 +1096,7 @@ def reload(
         if_metageneration_match=None,
         if_metageneration_not_match=None,
         retry=DEFAULT_RETRY,
+        soft_deleted=None,
     ):
         """Reload properties from Cloud Storage.
 
@@ -1084,6 +1136,13 @@ def reload(
         :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
         :param retry:
             (Optional) How to retry the RPC. See: :ref:`configuring_retries`
+
+        :type soft_deleted: bool
+        :param soft_deleted: (Optional) If True, looks for a soft-deleted
+            bucket. Will only return the bucket metadata if the bucket exists
+            and is in a soft-deleted state. The bucket ``generation`` must be
+            set if ``soft_deleted`` is set to True.
+            See: https://cloud.google.com/storage/docs/soft-delete
         """
         super(Bucket, self).reload(
             client=client,
@@ -1094,6 +1153,7 @@ def reload(
             if_metageneration_match=if_metageneration_match,
             if_metageneration_not_match=if_metageneration_not_match,
             retry=retry,
+            soft_deleted=soft_deleted,
         )
 
     @create_trace_span(name="Storage.Bucket.patch")
@@ -2159,8 +2219,8 @@ def restore_blob(
         :param client: (Optional) The client to use. If not passed, falls back
                        to the ``client`` stored on the current bucket.
 
-        :type generation: long
-        :param generation: (Optional) If present, selects a specific revision of this object.
+        :type generation: int
+        :param generation: Selects the specific revision of the object.
 
         :type copy_source_acl: bool
         :param copy_source_acl: (Optional) If true, copy the soft-deleted object's access controls.
diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py
index bc2d1147e..b1f48f97e 100644
--- a/google/cloud/storage/client.py
+++ b/google/cloud/storage/client.py
@@ -30,6 +30,7 @@
 from google.cloud.client import ClientWithProject
 from google.cloud.exceptions import NotFound
 
+from google.cloud.storage._helpers import _add_generation_match_parameters
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
 from google.cloud.storage._helpers import _get_api_endpoint_override
 from google.cloud.storage._helpers import _get_environ_project
@@ -367,7 +368,7 @@ def get_service_account_email(
         api_response = self._get_resource(path, timeout=timeout, retry=retry)
         return api_response["email_address"]
 
-    def bucket(self, bucket_name, user_project=None):
+    def bucket(self, bucket_name, user_project=None, generation=None):
         """Factory constructor for bucket object.
 
         .. note::
@@ -381,10 +382,19 @@ def bucket(self, bucket_name, user_project=None):
         :param user_project: (Optional) The project ID to be billed for API
                              requests made via the bucket.
 
+        :type generation: int
+        :param generation: (Optional) If present, selects a specific revision of
+                           this bucket.
+
         :rtype: :class:`google.cloud.storage.bucket.Bucket`
         :returns: The bucket object created.
         """
-        return Bucket(client=self, name=bucket_name, user_project=user_project)
+        return Bucket(
+            client=self,
+            name=bucket_name,
+            user_project=user_project,
+            generation=generation,
+        )
 
     def batch(self, raise_exception=True):
         """Factory constructor for batch object.
@@ -789,7 +799,7 @@ def _delete_resource(
             _target_object=_target_object,
         )
 
-    def _bucket_arg_to_bucket(self, bucket_or_name):
+    def _bucket_arg_to_bucket(self, bucket_or_name, generation=None):
         """Helper to return given bucket or create new by name.
 
         Args:
@@ -798,17 +808,27 @@ def _bucket_arg_to_bucket(self, bucket_or_name):
                  str, \
             ]):
                 The bucket resource to pass or name to create.
+            generation (Optional[int]):
+                The bucket generation. If generation is specified,
+                bucket_or_name must be a name (str).
 
         Returns:
             google.cloud.storage.bucket.Bucket
                 The newly created bucket or the given one.
         """
         if isinstance(bucket_or_name, Bucket):
+            if generation:
+                raise ValueError(
+                    "The generation can only be specified if a "
+                    "name is used to specify a bucket, not a Bucket object. "
+                    "Create a new Bucket object with the correct generation "
+                    "instead."
+                )
             bucket = bucket_or_name
             if bucket.client is None:
                 bucket._client = self
         else:
-            bucket = Bucket(self, name=bucket_or_name)
+            bucket = Bucket(self, name=bucket_or_name, generation=generation)
         return bucket
 
     @create_trace_span(name="Storage.Client.getBucket")
@@ -819,6 +839,9 @@ def get_bucket(
         if_metageneration_match=None,
         if_metageneration_not_match=None,
         retry=DEFAULT_RETRY,
+        *,
+        generation=None,
+        soft_deleted=None,
     ):
         """Retrieve a bucket via a GET request.
 
@@ -837,12 +860,12 @@ def get_bucket(
                 Can also be passed as a tuple (connect_timeout, read_timeout).
                 See :meth:`requests.Session.request` documentation for details.
 
-            if_metageneration_match (Optional[long]):
+            if_metageneration_match (Optional[int]):
                 Make the operation conditional on whether the
-                blob's current metageneration matches the given value.
+                bucket's current metageneration matches the given value.
 
-            if_metageneration_not_match (Optional[long]):
-                Make the operation conditional on whether the blob's
+            if_metageneration_not_match (Optional[int]):
+                Make the operation conditional on whether the bucket's
                 current metageneration does not match the given value.
 
             retry (Optional[Union[google.api_core.retry.Retry, google.cloud.storage.retry.ConditionalRetryPolicy]]):
@@ -859,6 +882,19 @@ def get_bucket(
                 See the retry.py source code and docstrings in this package (google.cloud.storage.retry) for
                 information on retry types and how to configure them.
 
+            generation (Optional[int]):
+                The generation of the bucket. The generation can be used to
+                specify a specific soft-deleted version of the bucket, in
+                conjunction with the ``soft_deleted`` argument below. If
+                ``soft_deleted`` is not True, the generation is unused.
+
+            soft_deleted (Optional[bool]):
+                If True, looks for a soft-deleted bucket. Will only return
+                the bucket metadata if the bucket exists and is in a
+                soft-deleted state. The bucket ``generation`` is required if
+                ``soft_deleted`` is set to True.
+                See: https://cloud.google.com/storage/docs/soft-delete
+
         Returns:
             google.cloud.storage.bucket.Bucket
                 The bucket matching the name provided.
@@ -867,13 +903,14 @@ def get_bucket(
             google.cloud.exceptions.NotFound
                 If the bucket is not found.
         """
-        bucket = self._bucket_arg_to_bucket(bucket_or_name)
+        bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation)
         bucket.reload(
             client=self,
             timeout=timeout,
             if_metageneration_match=if_metageneration_match,
             if_metageneration_not_match=if_metageneration_not_match,
             retry=retry,
+            soft_deleted=soft_deleted,
         )
         return bucket
 
@@ -1386,6 +1423,8 @@ def list_buckets(
         page_size=None,
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
+        *,
+        soft_deleted=None,
     ):
         """Get all buckets in the project associated to the client.
 
@@ -1438,6 +1477,12 @@ def list_buckets(
         :param retry:
             (Optional) How to retry the RPC. See: :ref:`configuring_retries`
 
+        :type soft_deleted: bool
+        :param soft_deleted:
+            (Optional) If true, only soft-deleted buckets will be listed as distinct results in order of increasing
+            generation number. This parameter can only be used successfully if the bucket has a soft delete policy.
+            See: https://cloud.google.com/storage/docs/soft-delete
+
         :rtype: :class:`~google.api_core.page_iterator.Iterator`
         :raises ValueError: if both ``project`` is ``None`` and the client's
                             project is also ``None``.
@@ -1469,6 +1514,9 @@ def list_buckets(
         if fields is not None:
             extra_params["fields"] = fields
 
+        if soft_deleted is not None:
+            extra_params["softDeleted"] = soft_deleted
+
         return self._list_resource(
             "/b",
             _item_to_bucket,
@@ -1480,6 +1528,71 @@ def list_buckets(
             retry=retry,
         )
 
+    def restore_bucket(
+        self,
+        bucket_name,
+        generation,
+        projection="noAcl",
+        if_metageneration_match=None,
+        if_metageneration_not_match=None,
+        timeout=_DEFAULT_TIMEOUT,
+        retry=DEFAULT_RETRY,
+    ):
+        """Restores a soft-deleted bucket.
+
+        :type bucket_name: str
+        :param bucket_name: The name of the bucket to be restored.
+
+        :type generation: int
+        :param generation: Selects the specific revision of the bucket.
+
+        :type projection: str
+        :param projection:
+            (Optional) Specifies the set of properties to return. If used, must
+            be 'full' or 'noAcl'. Defaults to 'noAcl'.
+
+        if_metageneration_match (Optional[int]):
+            Make the operation conditional on whether the
+            blob's current metageneration matches the given value.
+
+        if_metageneration_not_match (Optional[int]):
+            Make the operation conditional on whether the blob's
+            current metageneration does not match the given value.
+
+        :type timeout: float or tuple
+        :param timeout:
+            (Optional) The amount of time, in seconds, to wait
+            for the server response.  See: :ref:`configuring_timeouts`
+
+        :type retry: google.api_core.retry.Retry or google.cloud.storage.retry.ConditionalRetryPolicy
+        :param retry:
+            (Optional) How to retry the RPC.
+
+            Users can configure non-default retry behavior. A ``None`` value will
+            disable retries. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout).
+
+        :rtype: :class:`google.cloud.storage.bucket.Bucket`
+        :returns: The restored Bucket.
+        """
+        query_params = {"generation": generation, "projection": projection}
+
+        _add_generation_match_parameters(
+            query_params,
+            if_metageneration_match=if_metageneration_match,
+            if_metageneration_not_match=if_metageneration_not_match,
+        )
+
+        bucket = self.bucket(bucket_name)
+        api_response = self._post_resource(
+            f"{bucket.path}/restore",
+            None,
+            query_params=query_params,
+            timeout=timeout,
+            retry=retry,
+        )
+        bucket._set_properties(api_response)
+        return bucket
+
     @create_trace_span(name="Storage.Client.createHmacKey")
     def create_hmac_key(
         self,
diff --git a/tests/system/test_client.py b/tests/system/test_client.py
index c1b3858f2..6b3798c83 100644
--- a/tests/system/test_client.py
+++ b/tests/system/test_client.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import datetime
 import io
 import re
 import os
@@ -215,3 +216,46 @@ def test_client_universe_domain(
             stored_contents = file_obj.read()
 
     assert stored_contents == payload
+
+
+def test_restore_bucket(
+    storage_client,
+    buckets_to_delete,
+):
+    from google.cloud.storage.bucket import SoftDeletePolicy
+
+    # Create a bucket with soft delete policy.
+    duration_secs = 7 * 86400
+    bucket = storage_client.bucket(_helpers.unique_name("w-soft-delete"))
+    bucket.soft_delete_policy.retention_duration_seconds = duration_secs
+    bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket)
+    buckets_to_delete.append(bucket)
+
+    policy = bucket.soft_delete_policy
+    assert isinstance(policy, SoftDeletePolicy)
+    assert policy.retention_duration_seconds == duration_secs
+    assert isinstance(policy.effective_time, datetime.datetime)
+
+    # Record the bucket's name and generation
+    name = bucket.name
+    generation = bucket.generation
+    assert generation is not None
+
+    # Delete the bucket, then use the generation to get a reference to it again.
+    _helpers.retry_429_503(bucket.delete)()
+    soft_deleted_bucket = _helpers.retry_429_503(storage_client.get_bucket)(
+        name, generation=generation, soft_deleted=True
+    )
+    assert soft_deleted_bucket.name == name
+    assert soft_deleted_bucket.generation == generation
+    assert soft_deleted_bucket.soft_delete_time is not None
+    assert soft_deleted_bucket.hard_delete_time is not None
+
+    # Restore the bucket.
+    restored_bucket = _helpers.retry_429_503(storage_client.restore_bucket)(
+        name, generation=generation
+    )
+    assert restored_bucket.name == name
+    assert restored_bucket.generation == generation
+    assert restored_bucket.soft_delete_time is None
+    assert restored_bucket.hard_delete_time is None
diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py
index 030fba72b..e6072ce5f 100644
--- a/tests/unit/test_bucket.py
+++ b/tests/unit/test_bucket.py
@@ -603,16 +603,24 @@ def _make_client(**kw):
         kw["api_endpoint"] = kw.get("api_endpoint") or _get_default_storage_base_url()
         return mock.create_autospec(Client, instance=True, **kw)
 
-    def _make_one(self, client=None, name=None, properties=None, user_project=None):
+    def _make_one(
+        self,
+        client=None,
+        name=None,
+        properties=None,
+        user_project=None,
+        generation=None,
+    ):
         if client is None:
             client = self._make_client()
         if user_project is None:
-            bucket = self._get_target_class()(client, name=name)
+            bucket = self._get_target_class()(client, name=name, generation=generation)
         else:
             bucket = self._get_target_class()(
-                client, name=name, user_project=user_project
+                client, name=name, user_project=user_project, generation=generation
             )
-        bucket._properties = properties or {}
+        if properties:
+            bucket._properties = {**bucket._properties, **properties}
         return bucket
 
     def test_ctor_w_invalid_name(self):
@@ -633,6 +641,9 @@ def test_ctor(self):
         self.assertIs(bucket._default_object_acl.bucket, bucket)
         self.assertEqual(list(bucket._label_removals), [])
         self.assertIsNone(bucket.user_project)
+        self.assertEqual(bucket.generation, None)
+        self.assertEqual(bucket.soft_delete_time, None)
+        self.assertEqual(bucket.hard_delete_time, None)
 
     def test_ctor_w_user_project(self):
         NAME = "name"
@@ -649,6 +660,31 @@ def test_ctor_w_user_project(self):
         self.assertEqual(list(bucket._label_removals), [])
         self.assertEqual(bucket.user_project, USER_PROJECT)
 
+    def test_ctor_w_generation_and_soft_delete_info(self):
+        from google.cloud._helpers import _RFC3339_MICROS
+
+        NAME = "name"
+        generation = 12345
+
+        soft_timestamp = datetime.datetime(2024, 1, 5, 20, 34, 37, tzinfo=_UTC)
+        soft_delete = soft_timestamp.strftime(_RFC3339_MICROS)
+        hard_timestamp = datetime.datetime(2024, 1, 15, 20, 34, 37, tzinfo=_UTC)
+        hard_delete = hard_timestamp.strftime(_RFC3339_MICROS)
+        properties = {"softDeleteTime": soft_delete, "hardDeleteTime": hard_delete}
+
+        bucket = self._make_one(name=NAME, generation=generation, properties=properties)
+        self.assertEqual(bucket.name, NAME)
+        self.assertEqual(list(bucket._changes), [])
+        self.assertFalse(bucket._acl.loaded)
+        self.assertIs(bucket._acl.bucket, bucket)
+        self.assertFalse(bucket._default_object_acl.loaded)
+        self.assertIs(bucket._default_object_acl.bucket, bucket)
+        self.assertEqual(list(bucket._label_removals), [])
+        self.assertIsNone(bucket.user_project)
+        self.assertEqual(bucket.generation, generation)
+        self.assertEqual(bucket.soft_delete_time, soft_timestamp)
+        self.assertEqual(bucket.hard_delete_time, hard_timestamp)
+
     def test_blob_wo_keys(self):
         from google.cloud.storage.blob import Blob
 
@@ -1994,6 +2030,31 @@ def test_reload_w_generation_match(self):
         with self.assertRaises(TypeError):
             bucket.reload(if_generation_match=6)
 
+    def test_reload_w_soft_deleted(self):
+        name = "name"
+        api_response = {"name": name}
+        client = mock.Mock(spec=["_get_resource"])
+        client._get_resource.return_value = api_response
+        bucket = self._make_one(client, name=name, generation=12345)
+
+        bucket.reload(soft_deleted=True)
+
+        expected_path = f"/b/{name}"
+        expected_query_params = {
+            "projection": "noAcl",
+            "softDeleted": True,
+            "generation": 12345,
+        }
+        expected_headers = {}
+        client._get_resource.assert_called_once_with(
+            expected_path,
+            query_params=expected_query_params,
+            headers=expected_headers,
+            timeout=self._get_default_timeout(),
+            retry=DEFAULT_RETRY,
+            _target_object=bucket,
+        )
+
     def test_update_w_metageneration_match(self):
         name = "name"
         metageneration_number = 9
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index 5eb339acb..df4578e09 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -532,13 +532,15 @@ def test_bucket(self):
         PROJECT = "PROJECT"
         CREDENTIALS = _make_credentials()
         BUCKET_NAME = "BUCKET_NAME"
+        GENERATION = 12345
 
         client = self._make_one(project=PROJECT, credentials=CREDENTIALS)
-        bucket = client.bucket(BUCKET_NAME)
+        bucket = client.bucket(BUCKET_NAME, generation=GENERATION)
         self.assertIsInstance(bucket, Bucket)
         self.assertIs(bucket.client, client)
         self.assertEqual(bucket.name, BUCKET_NAME)
         self.assertIsNone(bucket.user_project)
+        self.assertEqual(bucket.generation, GENERATION)
 
     def test_bucket_w_user_project(self):
         from google.cloud.storage.bucket import Bucket
@@ -958,6 +960,20 @@ def test__bucket_arg_to_bucket_w_bucket_w_client(self):
         self.assertIs(found, bucket)
         self.assertIs(found.client, other_client)
 
+    def test__bucket_arg_to_bucket_raises_on_generation(self):
+        from google.cloud.storage.bucket import Bucket
+
+        project = "PROJECT"
+        credentials = _make_credentials()
+        client = self._make_one(project=project, credentials=credentials)
+        other_client = mock.Mock(spec=[])
+        bucket_name = "w_client"
+
+        bucket = Bucket(other_client, name=bucket_name)
+
+        with self.assertRaises(ValueError):
+            client._bucket_arg_to_bucket(bucket, generation=12345)
+
     def test__bucket_arg_to_bucket_w_bucket_wo_client(self):
         from google.cloud.storage.bucket import Bucket
 
@@ -977,14 +993,16 @@ def test__bucket_arg_to_bucket_w_bucket_name(self):
         from google.cloud.storage.bucket import Bucket
 
         project = "PROJECT"
+        generation = 12345
         credentials = _make_credentials()
         client = self._make_one(project=project, credentials=credentials)
         bucket_name = "string-name"
 
-        found = client._bucket_arg_to_bucket(bucket_name)
+        found = client._bucket_arg_to_bucket(bucket_name, generation)
 
         self.assertIsInstance(found, Bucket)
         self.assertEqual(found.name, bucket_name)
+        self.assertEqual(found.generation, generation)
         self.assertIs(found.client, client)
 
     def test_get_bucket_miss_w_string_w_defaults(self):
@@ -1045,6 +1063,41 @@ def test_get_bucket_hit_w_string_w_timeout(self):
             _target_object=bucket,
         )
 
+    def test_get_bucket_hit_w_string_w_soft_deleted(self):
+        from google.cloud.storage.bucket import Bucket
+
+        project = "PROJECT"
+        bucket_name = "bucket-name"
+        generation = 12345
+        api_response = {"name": bucket_name, "generation": generation}
+        credentials = _make_credentials()
+        client = self._make_one(project=project, credentials=credentials)
+        client._get_resource = mock.Mock(return_value=api_response)
+
+        bucket = client.get_bucket(
+            bucket_name, generation=generation, soft_deleted=True
+        )
+
+        self.assertIsInstance(bucket, Bucket)
+        self.assertEqual(bucket.name, bucket_name)
+        self.assertEqual(bucket.generation, generation)
+
+        expected_path = f"/b/{bucket_name}"
+        expected_query_params = {
+            "generation": generation,
+            "projection": "noAcl",
+            "softDeleted": True,
+        }
+        expected_headers = {}
+        client._get_resource.assert_called_once_with(
+            expected_path,
+            query_params=expected_query_params,
+            headers=expected_headers,
+            timeout=60,
+            retry=DEFAULT_RETRY,
+            _target_object=bucket,
+        )
+
     def test_get_bucket_hit_w_string_w_metageneration_match(self):
         from google.cloud.storage.bucket import Bucket
 
@@ -2259,6 +2312,39 @@ def test_list_buckets_w_defaults(self):
             retry=DEFAULT_RETRY,
         )
 
+    def test_list_buckets_w_soft_deleted(self):
+        from google.cloud.storage.client import _item_to_bucket
+
+        project = "PROJECT"
+        credentials = _make_credentials()
+        client = self._make_one(project=project, credentials=credentials)
+        client._list_resource = mock.Mock(spec=[])
+
+        iterator = client.list_buckets(soft_deleted=True)
+
+        self.assertIs(iterator, client._list_resource.return_value)
+
+        expected_path = "/b"
+        expected_item_to_value = _item_to_bucket
+        expected_page_token = None
+        expected_max_results = None
+        expected_page_size = None
+        expected_extra_params = {
+            "project": project,
+            "projection": "noAcl",
+            "softDeleted": True,
+        }
+        client._list_resource.assert_called_once_with(
+            expected_path,
+            expected_item_to_value,
+            page_token=expected_page_token,
+            max_results=expected_max_results,
+            extra_params=expected_extra_params,
+            page_size=expected_page_size,
+            timeout=self._get_default_timeout(),
+            retry=DEFAULT_RETRY,
+        )
+
     def test_list_buckets_w_explicit(self):
         from google.cloud.storage.client import _item_to_bucket
 
@@ -2312,6 +2398,33 @@ def test_list_buckets_w_explicit(self):
             retry=retry,
         )
 
+    def test_restore_bucket(self):
+        from google.cloud.storage.bucket import Bucket
+
+        PROJECT = "PROJECT"
+        NAME = "my_deleted_bucket"
+        GENERATION = 12345
+
+        api_response = {"name": NAME}
+        credentials = _make_credentials()
+        client = self._make_one(project=PROJECT, credentials=credentials)
+        client._post_resource = mock.Mock(return_value=api_response)
+
+        bucket = client.restore_bucket(NAME, GENERATION)
+
+        self.assertIsInstance(bucket, Bucket)
+        self.assertEqual(bucket.name, NAME)
+
+        expected_path = f"/b/{NAME}/restore"
+        expected_query_params = {"generation": 12345, "projection": "noAcl"}
+        client._post_resource.assert_called_once_with(
+            expected_path,
+            None,
+            query_params=expected_query_params,
+            timeout=self._get_default_timeout(),
+            retry=DEFAULT_RETRY,
+        )
+
     def _create_hmac_key_helper(
         self,
         explicit_project=None,

From 06ed15b33dc884da6dffbef5119e47f0fc4e1285 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Wed, 30 Oct 2024 14:17:43 -0700
Subject: [PATCH 11/17] feat: add support for restore token (#1369)

* feat: add support for restore token

* add unit tests coverage

* update docstrings

* fix docs
---
 google/cloud/storage/_helpers.py | 10 ++++++
 google/cloud/storage/blob.py     | 23 ++++++++++++++
 google/cloud/storage/bucket.py   | 19 +++++++++++
 tests/system/test_bucket.py      | 54 +++++++++++++++++++++++++++++++-
 tests/unit/test_blob.py          | 18 +++++++++--
 tests/unit/test_bucket.py        | 19 +++++++++--
 6 files changed, 137 insertions(+), 6 deletions(-)

diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py
index 3793a95f2..8af5fd96c 100644
--- a/google/cloud/storage/_helpers.py
+++ b/google/cloud/storage/_helpers.py
@@ -226,6 +226,7 @@ def reload(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
+        restore_token=None,
     ):
         """Reload properties from Cloud Storage.
 
@@ -278,6 +279,13 @@ def reload(
             the object metadata if the object exists and is in a soft-deleted state.
             :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
+
+        :type restore_token: str
+        :param restore_token:
+            (Optional) The restore_token is required to retrieve a soft-deleted object only if
+            its name and generation value do not uniquely identify it, and hierarchical namespace
+            is enabled on the bucket. Otherwise, this parameter is optional.
+            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
         """
         client = self._require_client(client)
         query_params = self._query_params
@@ -296,6 +304,8 @@ def reload(
             # Soft delete reload requires a generation, even for targets
             # that don't include them in default query params (buckets).
             query_params["generation"] = self.generation
+        if restore_token is not None:
+            query_params["restoreToken"] = restore_token
         headers = self._encryption_headers()
         _add_etag_match_headers(
             headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
index e474f1681..6f2aab674 100644
--- a/google/cloud/storage/blob.py
+++ b/google/cloud/storage/blob.py
@@ -653,6 +653,7 @@ def exists(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
+        restore_token=None,
     ):
         """Determines whether or not this blob exists.
 
@@ -704,6 +705,13 @@ def exists(
             :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
 
+        :type restore_token: str
+        :param restore_token:
+            (Optional) The restore_token is required to retrieve a soft-deleted object only if
+            its name and generation value do not uniquely identify it, and hierarchical namespace
+            is enabled on the bucket. Otherwise, this parameter is optional.
+            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
+
         :rtype: bool
         :returns: True if the blob exists in Cloud Storage.
         """
@@ -714,6 +722,8 @@ def exists(
         query_params["fields"] = "name"
         if soft_deleted is not None:
             query_params["softDeleted"] = soft_deleted
+        if restore_token is not None:
+            query_params["restoreToken"] = restore_token
 
         _add_generation_match_parameters(
             query_params,
@@ -4794,6 +4804,19 @@ def hard_delete_time(self):
         if hard_delete_time is not None:
             return _rfc3339_nanos_to_datetime(hard_delete_time)
 
+    @property
+    def restore_token(self):
+        """The restore token, a universally unique identifier (UUID), along with the object's
+        name and generation value, uniquely identifies a soft-deleted object.
+        This field is only returned for soft-deleted objects in hierarchical namespace buckets.
+
+        :rtype: string or ``NoneType``
+        :returns:
+            (readonly) The restore token used to differentiate soft-deleted objects with the same name and generation.
+            This field is only returned for soft-deleted objects in hierarchical namespace buckets.
+        """
+        return self._properties.get("restoreToken")
+
 
 def _get_host_name(connection):
     """Returns the host name from the given connection.
diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py
index 7cea15f4e..a0018af91 100644
--- a/google/cloud/storage/bucket.py
+++ b/google/cloud/storage/bucket.py
@@ -1256,6 +1256,7 @@ def get_blob(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
+        restore_token=None,
         **kwargs,
     ):
         """Get a blob object by name.
@@ -1323,6 +1324,13 @@ def get_blob(
             Object ``generation`` is required if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
 
+        :type restore_token: str
+        :param restore_token:
+            (Optional) The restore_token is required to retrieve a soft-deleted object only if
+            its name and generation value do not uniquely identify it, and hierarchical namespace
+            is enabled on the bucket. Otherwise, this parameter is optional.
+            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
+
         :param kwargs: Keyword arguments to pass to the
                        :class:`~google.cloud.storage.blob.Blob` constructor.
 
@@ -1351,6 +1359,7 @@ def get_blob(
                 if_metageneration_not_match=if_metageneration_not_match,
                 retry=retry,
                 soft_deleted=soft_deleted,
+                restore_token=restore_token,
             )
         except NotFound:
             return None
@@ -2199,6 +2208,7 @@ def restore_blob(
         generation=None,
         copy_source_acl=None,
         projection=None,
+        restore_token=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -2229,6 +2239,13 @@ def restore_blob(
         :param projection: (Optional) Specifies the set of properties to return.
                            If used, must be 'full' or 'noAcl'.
 
+        :type restore_token: str
+        :param restore_token:
+            (Optional) The restore_token is required to restore a soft-deleted object
+            only if its name and generation value do not uniquely identify it, and hierarchical namespace
+            is enabled on the bucket. Otherwise, this parameter is optional.
+            See: https://cloud.google.com/storage/docs/json_api/v1/objects/restore
+
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -2276,6 +2293,8 @@ def restore_blob(
             query_params["copySourceAcl"] = copy_source_acl
         if projection is not None:
             query_params["projection"] = projection
+        if restore_token is not None:
+            query_params["restoreToken"] = restore_token
 
         _add_generation_match_parameters(
             query_params,
diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py
index 270a77ad1..7635388a5 100644
--- a/tests/system/test_bucket.py
+++ b/tests/system/test_bucket.py
@@ -1232,7 +1232,7 @@ def test_soft_delete_policy(
     assert restored_blob.generation != gen
 
     # Patch the soft delete policy on an existing bucket.
-    new_duration_secs = 10 * 86400
+    new_duration_secs = 0
     bucket.soft_delete_policy.retention_duration_seconds = new_duration_secs
     bucket.patch()
     assert bucket.soft_delete_policy.retention_duration_seconds == new_duration_secs
@@ -1265,3 +1265,55 @@ def test_new_bucket_with_hierarchical_namespace(
     bucket = storage_client.create_bucket(bucket_obj)
     buckets_to_delete.append(bucket)
     assert bucket.hierarchical_namespace_enabled is True
+
+
+def test_restore_token(
+    storage_client,
+    buckets_to_delete,
+    blobs_to_delete,
+):
+    # Create HNS bucket with soft delete policy.
+    duration_secs = 7 * 86400
+    bucket = storage_client.bucket(_helpers.unique_name("w-soft-delete"))
+    bucket.hierarchical_namespace_enabled = True
+    bucket.iam_configuration.uniform_bucket_level_access_enabled = True
+    bucket.soft_delete_policy.retention_duration_seconds = duration_secs
+    bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket)
+    buckets_to_delete.append(bucket)
+
+    # Insert an object and delete it to enter soft-deleted state.
+    payload = b"DEADBEEF"
+    blob_name = _helpers.unique_name("soft-delete")
+    blob = bucket.blob(blob_name)
+    blob.upload_from_string(payload)
+    # blob = bucket.get_blob(blob_name)
+    gen = blob.generation
+    blob.delete()
+
+    # Get the soft-deleted object and restore token.
+    blob = bucket.get_blob(blob_name, generation=gen, soft_deleted=True)
+    restore_token = blob.restore_token
+
+    # List and get soft-deleted object that includes restore token.
+    all_blobs = list(bucket.list_blobs(soft_deleted=True))
+    assert all_blobs[0].restore_token is not None
+    blob_w_restore_token = bucket.get_blob(
+        blob_name, generation=gen, soft_deleted=True, restore_token=restore_token
+    )
+    assert blob_w_restore_token.soft_delete_time is not None
+    assert blob_w_restore_token.hard_delete_time is not None
+    assert blob_w_restore_token.restore_token is not None
+
+    # Restore the soft-deleted object using the restore token.
+    restored_blob = bucket.restore_blob(
+        blob_name, generation=gen, restore_token=restore_token
+    )
+    blobs_to_delete.append(restored_blob)
+    assert restored_blob.exists() is True
+    assert restored_blob.generation != gen
+
+    # Patch the soft delete policy on the bucket.
+    new_duration_secs = 0
+    bucket.soft_delete_policy.retention_duration_seconds = new_duration_secs
+    bucket.patch()
+    assert bucket.soft_delete_policy.retention_duration_seconds == new_duration_secs
diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py
index b0ff4f07b..fc472a30f 100644
--- a/tests/unit/test_blob.py
+++ b/tests/unit/test_blob.py
@@ -784,21 +784,25 @@ def test_exists_hit_w_generation_w_retry(self):
             _target_object=None,
         )
 
-    def test_exists_hit_w_generation_w_soft_deleted(self):
+    def test_exists_hit_w_gen_soft_deleted_restore_token(self):
         blob_name = "blob-name"
         generation = 123456
+        restore_token = "88ba0d97-639e-5902"
         api_response = {"name": blob_name}
         client = mock.Mock(spec=["_get_resource"])
         client._get_resource.return_value = api_response
         bucket = _Bucket(client)
         blob = self._make_one(blob_name, bucket=bucket, generation=generation)
 
-        self.assertTrue(blob.exists(retry=None, soft_deleted=True))
+        self.assertTrue(
+            blob.exists(retry=None, soft_deleted=True, restore_token=restore_token)
+        )
 
         expected_query_params = {
             "fields": "name",
             "generation": generation,
             "softDeleted": True,
+            "restoreToken": restore_token,
         }
         expected_headers = {}
         client._get_resource.assert_called_once_with(
@@ -5870,6 +5874,16 @@ def test_soft_hard_delete_time_getter(self):
         self.assertEqual(blob.soft_delete_time, soft_timstamp)
         self.assertEqual(blob.hard_delete_time, hard_timstamp)
 
+    def test_restore_token_getter(self):
+        BLOB_NAME = "blob-name"
+        bucket = _Bucket()
+        restore_token = "88ba0d97-639e-5902"
+        properties = {
+            "restoreToken": restore_token,
+        }
+        blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties)
+        self.assertEqual(blob.restore_token, restore_token)
+
     def test_soft_hard_delte_time_unset(self):
         BUCKET = object()
         blob = self._make_one("blob-name", bucket=BUCKET)
diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py
index e6072ce5f..ac2bf44ee 100644
--- a/tests/unit/test_bucket.py
+++ b/tests/unit/test_bucket.py
@@ -1018,18 +1018,24 @@ def test_get_blob_hit_w_user_project(self):
             _target_object=blob,
         )
 
-    def test_get_blob_hit_w_generation_w_soft_deleted(self):
+    def test_get_blob_hit_w_gen_soft_deleted_restore_token(self):
         from google.cloud.storage.blob import Blob
 
         name = "name"
         blob_name = "blob-name"
         generation = 1512565576797178
+        restore_token = "88ba0d97-639e-5902"
         api_response = {"name": blob_name, "generation": generation}
         client = mock.Mock(spec=["_get_resource"])
         client._get_resource.return_value = api_response
         bucket = self._make_one(client, name=name)
 
-        blob = bucket.get_blob(blob_name, generation=generation, soft_deleted=True)
+        blob = bucket.get_blob(
+            blob_name,
+            generation=generation,
+            soft_deleted=True,
+            restore_token=restore_token,
+        )
 
         self.assertIsInstance(blob, Blob)
         self.assertIs(blob.bucket, bucket)
@@ -1041,6 +1047,7 @@ def test_get_blob_hit_w_generation_w_soft_deleted(self):
             "generation": generation,
             "projection": "noAcl",
             "softDeleted": True,
+            "restoreToken": restore_token,
         }
         expected_headers = {}
         client._get_resource.assert_called_once_with(
@@ -4217,8 +4224,10 @@ def test_restore_blob_w_explicit(self):
         user_project = "user-project-123"
         bucket_name = "restore_bucket"
         blob_name = "restore_blob"
+        new_generation = 987655
         generation = 123456
-        api_response = {"name": blob_name, "generation": generation}
+        restore_token = "88ba0d97-639e-5902"
+        api_response = {"name": blob_name, "generation": new_generation}
         client = mock.Mock(spec=["_post_resource"])
         client._post_resource.return_value = api_response
         bucket = self._make_one(
@@ -4233,6 +4242,8 @@ def test_restore_blob_w_explicit(self):
         restored_blob = bucket.restore_blob(
             blob_name,
             client=client,
+            generation=generation,
+            restore_token=restore_token,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,
@@ -4245,6 +4256,8 @@ def test_restore_blob_w_explicit(self):
         expected_path = f"/b/{bucket_name}/o/{blob_name}/restore"
         expected_data = None
         expected_query_params = {
+            "generation": generation,
+            "restoreToken": restore_token,
             "userProject": user_project,
             "projection": projection,
             "ifGenerationMatch": if_generation_match,

From 012eab4563d740443fbc24271b6eb86d0f256b1b Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Thu, 7 Nov 2024 10:24:21 -0800
Subject: [PATCH 12/17] chore(revert): Revert "feat: add support for restore
 token (#1369)" (#1373)

This reverts commit 06ed15b33dc884da6dffbef5119e47f0fc4e1285.
---
 google/cloud/storage/_helpers.py | 10 ------
 google/cloud/storage/blob.py     | 23 --------------
 google/cloud/storage/bucket.py   | 19 -----------
 tests/system/test_bucket.py      | 54 +-------------------------------
 tests/unit/test_blob.py          | 18 ++---------
 tests/unit/test_bucket.py        | 19 ++---------
 6 files changed, 6 insertions(+), 137 deletions(-)

diff --git a/google/cloud/storage/_helpers.py b/google/cloud/storage/_helpers.py
index 8af5fd96c..3793a95f2 100644
--- a/google/cloud/storage/_helpers.py
+++ b/google/cloud/storage/_helpers.py
@@ -226,7 +226,6 @@ def reload(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
-        restore_token=None,
     ):
         """Reload properties from Cloud Storage.
 
@@ -279,13 +278,6 @@ def reload(
             the object metadata if the object exists and is in a soft-deleted state.
             :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
-
-        :type restore_token: str
-        :param restore_token:
-            (Optional) The restore_token is required to retrieve a soft-deleted object only if
-            its name and generation value do not uniquely identify it, and hierarchical namespace
-            is enabled on the bucket. Otherwise, this parameter is optional.
-            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
         """
         client = self._require_client(client)
         query_params = self._query_params
@@ -304,8 +296,6 @@ def reload(
             # Soft delete reload requires a generation, even for targets
             # that don't include them in default query params (buckets).
             query_params["generation"] = self.generation
-        if restore_token is not None:
-            query_params["restoreToken"] = restore_token
         headers = self._encryption_headers()
         _add_etag_match_headers(
             headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match
diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
index 6f2aab674..e474f1681 100644
--- a/google/cloud/storage/blob.py
+++ b/google/cloud/storage/blob.py
@@ -653,7 +653,6 @@ def exists(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
-        restore_token=None,
     ):
         """Determines whether or not this blob exists.
 
@@ -705,13 +704,6 @@ def exists(
             :attr:`generation` is required to be set on the blob if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
 
-        :type restore_token: str
-        :param restore_token:
-            (Optional) The restore_token is required to retrieve a soft-deleted object only if
-            its name and generation value do not uniquely identify it, and hierarchical namespace
-            is enabled on the bucket. Otherwise, this parameter is optional.
-            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
-
         :rtype: bool
         :returns: True if the blob exists in Cloud Storage.
         """
@@ -722,8 +714,6 @@ def exists(
         query_params["fields"] = "name"
         if soft_deleted is not None:
             query_params["softDeleted"] = soft_deleted
-        if restore_token is not None:
-            query_params["restoreToken"] = restore_token
 
         _add_generation_match_parameters(
             query_params,
@@ -4804,19 +4794,6 @@ def hard_delete_time(self):
         if hard_delete_time is not None:
             return _rfc3339_nanos_to_datetime(hard_delete_time)
 
-    @property
-    def restore_token(self):
-        """The restore token, a universally unique identifier (UUID), along with the object's
-        name and generation value, uniquely identifies a soft-deleted object.
-        This field is only returned for soft-deleted objects in hierarchical namespace buckets.
-
-        :rtype: string or ``NoneType``
-        :returns:
-            (readonly) The restore token used to differentiate soft-deleted objects with the same name and generation.
-            This field is only returned for soft-deleted objects in hierarchical namespace buckets.
-        """
-        return self._properties.get("restoreToken")
-
 
 def _get_host_name(connection):
     """Returns the host name from the given connection.
diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py
index a0018af91..7cea15f4e 100644
--- a/google/cloud/storage/bucket.py
+++ b/google/cloud/storage/bucket.py
@@ -1256,7 +1256,6 @@ def get_blob(
         timeout=_DEFAULT_TIMEOUT,
         retry=DEFAULT_RETRY,
         soft_deleted=None,
-        restore_token=None,
         **kwargs,
     ):
         """Get a blob object by name.
@@ -1324,13 +1323,6 @@ def get_blob(
             Object ``generation`` is required if ``soft_deleted`` is set to True.
             See: https://cloud.google.com/storage/docs/soft-delete
 
-        :type restore_token: str
-        :param restore_token:
-            (Optional) The restore_token is required to retrieve a soft-deleted object only if
-            its name and generation value do not uniquely identify it, and hierarchical namespace
-            is enabled on the bucket. Otherwise, this parameter is optional.
-            See: https://cloud.google.com/storage/docs/json_api/v1/objects/get
-
         :param kwargs: Keyword arguments to pass to the
                        :class:`~google.cloud.storage.blob.Blob` constructor.
 
@@ -1359,7 +1351,6 @@ def get_blob(
                 if_metageneration_not_match=if_metageneration_not_match,
                 retry=retry,
                 soft_deleted=soft_deleted,
-                restore_token=restore_token,
             )
         except NotFound:
             return None
@@ -2208,7 +2199,6 @@ def restore_blob(
         generation=None,
         copy_source_acl=None,
         projection=None,
-        restore_token=None,
         if_generation_match=None,
         if_generation_not_match=None,
         if_metageneration_match=None,
@@ -2239,13 +2229,6 @@ def restore_blob(
         :param projection: (Optional) Specifies the set of properties to return.
                            If used, must be 'full' or 'noAcl'.
 
-        :type restore_token: str
-        :param restore_token:
-            (Optional) The restore_token is required to restore a soft-deleted object
-            only if its name and generation value do not uniquely identify it, and hierarchical namespace
-            is enabled on the bucket. Otherwise, this parameter is optional.
-            See: https://cloud.google.com/storage/docs/json_api/v1/objects/restore
-
         :type if_generation_match: long
         :param if_generation_match:
             (Optional) See :ref:`using-if-generation-match`
@@ -2293,8 +2276,6 @@ def restore_blob(
             query_params["copySourceAcl"] = copy_source_acl
         if projection is not None:
             query_params["projection"] = projection
-        if restore_token is not None:
-            query_params["restoreToken"] = restore_token
 
         _add_generation_match_parameters(
             query_params,
diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py
index 7635388a5..270a77ad1 100644
--- a/tests/system/test_bucket.py
+++ b/tests/system/test_bucket.py
@@ -1232,7 +1232,7 @@ def test_soft_delete_policy(
     assert restored_blob.generation != gen
 
     # Patch the soft delete policy on an existing bucket.
-    new_duration_secs = 0
+    new_duration_secs = 10 * 86400
     bucket.soft_delete_policy.retention_duration_seconds = new_duration_secs
     bucket.patch()
     assert bucket.soft_delete_policy.retention_duration_seconds == new_duration_secs
@@ -1265,55 +1265,3 @@ def test_new_bucket_with_hierarchical_namespace(
     bucket = storage_client.create_bucket(bucket_obj)
     buckets_to_delete.append(bucket)
     assert bucket.hierarchical_namespace_enabled is True
-
-
-def test_restore_token(
-    storage_client,
-    buckets_to_delete,
-    blobs_to_delete,
-):
-    # Create HNS bucket with soft delete policy.
-    duration_secs = 7 * 86400
-    bucket = storage_client.bucket(_helpers.unique_name("w-soft-delete"))
-    bucket.hierarchical_namespace_enabled = True
-    bucket.iam_configuration.uniform_bucket_level_access_enabled = True
-    bucket.soft_delete_policy.retention_duration_seconds = duration_secs
-    bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket)
-    buckets_to_delete.append(bucket)
-
-    # Insert an object and delete it to enter soft-deleted state.
-    payload = b"DEADBEEF"
-    blob_name = _helpers.unique_name("soft-delete")
-    blob = bucket.blob(blob_name)
-    blob.upload_from_string(payload)
-    # blob = bucket.get_blob(blob_name)
-    gen = blob.generation
-    blob.delete()
-
-    # Get the soft-deleted object and restore token.
-    blob = bucket.get_blob(blob_name, generation=gen, soft_deleted=True)
-    restore_token = blob.restore_token
-
-    # List and get soft-deleted object that includes restore token.
-    all_blobs = list(bucket.list_blobs(soft_deleted=True))
-    assert all_blobs[0].restore_token is not None
-    blob_w_restore_token = bucket.get_blob(
-        blob_name, generation=gen, soft_deleted=True, restore_token=restore_token
-    )
-    assert blob_w_restore_token.soft_delete_time is not None
-    assert blob_w_restore_token.hard_delete_time is not None
-    assert blob_w_restore_token.restore_token is not None
-
-    # Restore the soft-deleted object using the restore token.
-    restored_blob = bucket.restore_blob(
-        blob_name, generation=gen, restore_token=restore_token
-    )
-    blobs_to_delete.append(restored_blob)
-    assert restored_blob.exists() is True
-    assert restored_blob.generation != gen
-
-    # Patch the soft delete policy on the bucket.
-    new_duration_secs = 0
-    bucket.soft_delete_policy.retention_duration_seconds = new_duration_secs
-    bucket.patch()
-    assert bucket.soft_delete_policy.retention_duration_seconds == new_duration_secs
diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py
index fc472a30f..b0ff4f07b 100644
--- a/tests/unit/test_blob.py
+++ b/tests/unit/test_blob.py
@@ -784,25 +784,21 @@ def test_exists_hit_w_generation_w_retry(self):
             _target_object=None,
         )
 
-    def test_exists_hit_w_gen_soft_deleted_restore_token(self):
+    def test_exists_hit_w_generation_w_soft_deleted(self):
         blob_name = "blob-name"
         generation = 123456
-        restore_token = "88ba0d97-639e-5902"
         api_response = {"name": blob_name}
         client = mock.Mock(spec=["_get_resource"])
         client._get_resource.return_value = api_response
         bucket = _Bucket(client)
         blob = self._make_one(blob_name, bucket=bucket, generation=generation)
 
-        self.assertTrue(
-            blob.exists(retry=None, soft_deleted=True, restore_token=restore_token)
-        )
+        self.assertTrue(blob.exists(retry=None, soft_deleted=True))
 
         expected_query_params = {
             "fields": "name",
             "generation": generation,
             "softDeleted": True,
-            "restoreToken": restore_token,
         }
         expected_headers = {}
         client._get_resource.assert_called_once_with(
@@ -5874,16 +5870,6 @@ def test_soft_hard_delete_time_getter(self):
         self.assertEqual(blob.soft_delete_time, soft_timstamp)
         self.assertEqual(blob.hard_delete_time, hard_timstamp)
 
-    def test_restore_token_getter(self):
-        BLOB_NAME = "blob-name"
-        bucket = _Bucket()
-        restore_token = "88ba0d97-639e-5902"
-        properties = {
-            "restoreToken": restore_token,
-        }
-        blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties)
-        self.assertEqual(blob.restore_token, restore_token)
-
     def test_soft_hard_delte_time_unset(self):
         BUCKET = object()
         blob = self._make_one("blob-name", bucket=BUCKET)
diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py
index ac2bf44ee..e6072ce5f 100644
--- a/tests/unit/test_bucket.py
+++ b/tests/unit/test_bucket.py
@@ -1018,24 +1018,18 @@ def test_get_blob_hit_w_user_project(self):
             _target_object=blob,
         )
 
-    def test_get_blob_hit_w_gen_soft_deleted_restore_token(self):
+    def test_get_blob_hit_w_generation_w_soft_deleted(self):
         from google.cloud.storage.blob import Blob
 
         name = "name"
         blob_name = "blob-name"
         generation = 1512565576797178
-        restore_token = "88ba0d97-639e-5902"
         api_response = {"name": blob_name, "generation": generation}
         client = mock.Mock(spec=["_get_resource"])
         client._get_resource.return_value = api_response
         bucket = self._make_one(client, name=name)
 
-        blob = bucket.get_blob(
-            blob_name,
-            generation=generation,
-            soft_deleted=True,
-            restore_token=restore_token,
-        )
+        blob = bucket.get_blob(blob_name, generation=generation, soft_deleted=True)
 
         self.assertIsInstance(blob, Blob)
         self.assertIs(blob.bucket, bucket)
@@ -1047,7 +1041,6 @@ def test_get_blob_hit_w_gen_soft_deleted_restore_token(self):
             "generation": generation,
             "projection": "noAcl",
             "softDeleted": True,
-            "restoreToken": restore_token,
         }
         expected_headers = {}
         client._get_resource.assert_called_once_with(
@@ -4224,10 +4217,8 @@ def test_restore_blob_w_explicit(self):
         user_project = "user-project-123"
         bucket_name = "restore_bucket"
         blob_name = "restore_blob"
-        new_generation = 987655
         generation = 123456
-        restore_token = "88ba0d97-639e-5902"
-        api_response = {"name": blob_name, "generation": new_generation}
+        api_response = {"name": blob_name, "generation": generation}
         client = mock.Mock(spec=["_post_resource"])
         client._post_resource.return_value = api_response
         bucket = self._make_one(
@@ -4242,8 +4233,6 @@ def test_restore_blob_w_explicit(self):
         restored_blob = bucket.restore_blob(
             blob_name,
             client=client,
-            generation=generation,
-            restore_token=restore_token,
             if_generation_match=if_generation_match,
             if_generation_not_match=if_generation_not_match,
             if_metageneration_match=if_metageneration_match,
@@ -4256,8 +4245,6 @@ def test_restore_blob_w_explicit(self):
         expected_path = f"/b/{bucket_name}/o/{blob_name}/restore"
         expected_data = None
         expected_query_params = {
-            "generation": generation,
-            "restoreToken": restore_token,
             "userProject": user_project,
             "projection": projection,
             "ifGenerationMatch": if_generation_match,

From 63cff046f0d82d3261fac654e206c7f77dca48b3 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Thu, 7 Nov 2024 11:17:29 -0800
Subject: [PATCH 13/17] chore: add Cloud Trace adoption attributes (#1374)

---
 .../cloud/storage/_opentelemetry_tracing.py   |  7 ++++++
 tests/unit/test__opentelemetry_tracing.py     | 23 ++++++++-----------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/google/cloud/storage/_opentelemetry_tracing.py b/google/cloud/storage/_opentelemetry_tracing.py
index ac4c43e07..3416081cd 100644
--- a/google/cloud/storage/_opentelemetry_tracing.py
+++ b/google/cloud/storage/_opentelemetry_tracing.py
@@ -54,6 +54,12 @@
     "user_agent.original": f"gcloud-python/{__version__}",
 }
 
+_cloud_trace_adoption_attrs = {
+    "gcp.client.service": "storage",
+    "gcp.client.version": __version__,
+    "gcp.client.repo": "googleapis/python-storage",
+}
+
 
 @contextmanager
 def create_trace_span(name, attributes=None, client=None, api_request=None, retry=None):
@@ -79,6 +85,7 @@ def create_trace_span(name, attributes=None, client=None, api_request=None, retr
 
 def _get_final_attributes(attributes=None, client=None, api_request=None, retry=None):
     collected_attr = _default_attributes.copy()
+    collected_attr.update(_cloud_trace_adoption_attrs)
     if api_request:
         collected_attr.update(_set_api_request_attr(api_request, client))
     if isinstance(retry, api_retry.Retry):
diff --git a/tests/unit/test__opentelemetry_tracing.py b/tests/unit/test__opentelemetry_tracing.py
index 631ac9f82..bdbb40fd2 100644
--- a/tests/unit/test__opentelemetry_tracing.py
+++ b/tests/unit/test__opentelemetry_tracing.py
@@ -89,11 +89,8 @@ def test_enable_trace_call(setup, setup_optin):
     extra_attributes = {
         "attribute1": "value1",
     }
-    expected_attributes = {
-        "rpc.service": "CloudStorage",
-        "rpc.system": "http",
-        "user_agent.original": f"gcloud-python/{__version__}",
-    }
+    expected_attributes = _opentelemetry_tracing._default_attributes.copy()
+    expected_attributes.update(_opentelemetry_tracing._cloud_trace_adoption_attrs)
     expected_attributes.update(extra_attributes)
 
     with _opentelemetry_tracing.create_trace_span(
@@ -114,11 +111,8 @@ def test_enable_trace_error(setup, setup_optin):
     extra_attributes = {
         "attribute1": "value1",
     }
-    expected_attributes = {
-        "rpc.service": "CloudStorage",
-        "rpc.system": "http",
-        "user_agent.original": f"gcloud-python/{__version__}",
-    }
+    expected_attributes = _opentelemetry_tracing._default_attributes.copy()
+    expected_attributes.update(_opentelemetry_tracing._cloud_trace_adoption_attrs)
     expected_attributes.update(extra_attributes)
 
     with pytest.raises(GoogleAPICallError):
@@ -157,6 +151,7 @@ def test_get_final_attributes(setup, setup_optin):
         "connect_timeout,read_timeout": (100, 100),
         "retry": f"multiplier{retry_obj._multiplier}/deadline{retry_obj._deadline}/max{retry_obj._maximum}/initial{retry_obj._initial}/predicate{retry_obj._predicate}",
     }
+    expected_attributes.update(_opentelemetry_tracing._cloud_trace_adoption_attrs)
 
     with mock.patch("google.cloud.storage.client.Client") as test_client:
         test_client.project = "test_project"
@@ -185,12 +180,12 @@ def test_set_conditional_retry_attr(setup, setup_optin):
         retry_policy, conditional_predicate, required_kwargs
     )
 
-    expected_attributes = {
-        "rpc.service": "CloudStorage",
-        "rpc.system": "http",
-        "user_agent.original": f"gcloud-python/{__version__}",
+    retry_attrs = {
         "retry": f"multiplier{retry_policy._multiplier}/deadline{retry_policy._deadline}/max{retry_policy._maximum}/initial{retry_policy._initial}/predicate{conditional_predicate}",
     }
+    expected_attributes = _opentelemetry_tracing._default_attributes.copy()
+    expected_attributes.update(_opentelemetry_tracing._cloud_trace_adoption_attrs)
+    expected_attributes.update(retry_attrs)
 
     with _opentelemetry_tracing.create_trace_span(
         test_span_name,

From 0cfddf4ba101ebbcd6a026687a4f1b67f98bdf96 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Tue, 12 Nov 2024 15:50:15 -0800
Subject: [PATCH 14/17] chore: remove debugger comment (#1381)

---
 google/cloud/storage/blob.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
index e474f1681..42b044824 100644
--- a/google/cloud/storage/blob.py
+++ b/google/cloud/storage/blob.py
@@ -2344,7 +2344,6 @@ def _do_resumable_upload(
             "upload.checksum": f"{checksum}",
         }
         args = {"timeout": timeout}
-        # import pdb; pdb.set_trace()
         with create_trace_span(
             name="Storage.ResumableUpload/transmitNextChunk",
             attributes=extra_attributes,

From abc80615ee00a14bc0e6b095252f6d1eb09c4b45 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Wed, 20 Nov 2024 14:50:42 -0800
Subject: [PATCH 15/17] feat: IAM signBlob retry and universe domain support
 (#1380)

* feat: IAM signBlob retries

* support universe domain and update tests

* update test credentials

* use ud signing bucket fixture
---
 google/cloud/storage/_signing.py | 34 ++++++++++++++++++++++++--------
 google/cloud/storage/blob.py     |  4 ++++
 tests/system/conftest.py         | 30 ++++++++++++++++++++++++++++
 tests/system/test__signing.py    | 29 +++++++++++++++++++++++++++
 tests/unit/test_blob.py          |  3 +++
 5 files changed, 92 insertions(+), 8 deletions(-)

diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py
index ecf110769..9f47e1a6e 100644
--- a/google/cloud/storage/_signing.py
+++ b/google/cloud/storage/_signing.py
@@ -28,8 +28,10 @@
 from google.auth import exceptions
 from google.auth.transport import requests
 from google.cloud import _helpers
+from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN
 from google.cloud.storage._helpers import _NOW
 from google.cloud.storage._helpers import _UTC
+from google.cloud.storage.retry import DEFAULT_RETRY
 
 
 # `google.cloud.storage._signing.NOW` is deprecated.
@@ -271,6 +273,7 @@ def generate_signed_url_v2(
     query_parameters=None,
     service_account_email=None,
     access_token=None,
+    universe_domain=None,
 ):
     """Generate a V2 signed URL to provide query-string auth'n to a resource.
 
@@ -384,7 +387,9 @@ def generate_signed_url_v2(
     # See https://github.com/googleapis/google-cloud-python/issues/922
     # Set the right query parameters.
     if access_token and service_account_email:
-        signature = _sign_message(string_to_sign, access_token, service_account_email)
+        signature = _sign_message(
+            string_to_sign, access_token, service_account_email, universe_domain
+        )
         signed_query_params = {
             "GoogleAccessId": service_account_email,
             "Expires": expiration_stamp,
@@ -432,6 +437,7 @@ def generate_signed_url_v4(
     query_parameters=None,
     service_account_email=None,
     access_token=None,
+    universe_domain=None,
     _request_timestamp=None,  # for testing only
 ):
     """Generate a V4 signed URL to provide query-string auth'n to a resource.
@@ -623,7 +629,9 @@ def generate_signed_url_v4(
     string_to_sign = "\n".join(string_elements)
 
     if access_token and service_account_email:
-        signature = _sign_message(string_to_sign, access_token, service_account_email)
+        signature = _sign_message(
+            string_to_sign, access_token, service_account_email, universe_domain
+        )
         signature_bytes = base64.b64decode(signature)
         signature = binascii.hexlify(signature_bytes).decode("ascii")
     else:
@@ -647,7 +655,12 @@ def get_v4_now_dtstamps():
     return timestamp, datestamp
 
 
-def _sign_message(message, access_token, service_account_email):
+def _sign_message(
+    message,
+    access_token,
+    service_account_email,
+    universe_domain=_DEFAULT_UNIVERSE_DOMAIN,
+):
     """Signs a message.
 
     :type message: str
@@ -669,17 +682,22 @@ def _sign_message(message, access_token, service_account_email):
     message = _helpers._to_bytes(message)
 
     method = "POST"
-    url = "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob?alt=json".format(
-        service_account_email
-    )
+    url = f"https://iamcredentials.{universe_domain}/v1/projects/-/serviceAccounts/{service_account_email}:signBlob?alt=json"
     headers = {
         "Authorization": "Bearer " + access_token,
         "Content-type": "application/json",
     }
     body = json.dumps({"payload": base64.b64encode(message).decode("utf-8")})
-
     request = requests.Request()
-    response = request(url=url, method=method, body=body, headers=headers)
+
+    def retriable_request():
+        response = request(url=url, method=method, body=body, headers=headers)
+        return response
+
+    # Apply the default retry object to the signBlob call.
+    retry = DEFAULT_RETRY
+    call = retry(retriable_request)
+    response = call()
 
     if response.status != http.client.OK:
         raise exceptions.TransportError(
diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py
index 42b044824..1cd71bdb7 100644
--- a/google/cloud/storage/blob.py
+++ b/google/cloud/storage/blob.py
@@ -607,6 +607,9 @@ def generate_signed_url(
             client = self._require_client(client)  # May be redundant, but that's ok.
             credentials = client._credentials
 
+        client = self._require_client(client)
+        universe_domain = client.universe_domain
+
         if version == "v2":
             helper = generate_signed_url_v2
         else:
@@ -638,6 +641,7 @@ def generate_signed_url(
             query_parameters=query_parameters,
             service_account_email=service_account_email,
             access_token=access_token,
+            universe_domain=universe_domain,
         )
 
     @create_trace_span(name="Storage.Blob.exists")
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 4ec56176d..588f66f79 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -384,3 +384,33 @@ def universe_domain_client(
     )
     with contextlib.closing(ud_storage_client):
         yield ud_storage_client
+
+
+@pytest.fixture(scope="function")
+def universe_domain_bucket(universe_domain_client, test_universe_location):
+    bucket_name = _helpers.unique_name("gcp-systest-ud")
+    bucket = universe_domain_client.create_bucket(
+        bucket_name, location=test_universe_location
+    )
+
+    blob = bucket.blob("README.txt")
+    blob.upload_from_string(_helpers.signing_blob_content)
+
+    yield bucket
+
+    _helpers.delete_bucket(bucket)
+
+
+@pytest.fixture(scope="function")
+def universe_domain_iam_client(
+    test_universe_domain, test_universe_project_id, universe_domain_credential
+):
+    from google.cloud import iam_credentials_v1
+
+    client_options = {"universe_domain": test_universe_domain}
+    iam_client = iam_credentials_v1.IAMCredentialsClient(
+        credentials=universe_domain_credential,
+        client_options=client_options,
+    )
+
+    return iam_client
diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py
index 8bcc46abc..ee7a85fb7 100644
--- a/tests/system/test__signing.py
+++ b/tests/system/test__signing.py
@@ -287,6 +287,35 @@ def test_create_signed_read_url_v4_w_access_token(
     )
 
 
+def test_create_signed_read_url_v4_w_access_token_universe_domain(
+    universe_domain_iam_client,
+    universe_domain_client,
+    test_universe_location,
+    universe_domain_credential,
+    universe_domain_bucket,
+    no_mtls,
+):
+    service_account_email = universe_domain_credential.service_account_email
+    name = path_template.expand(
+        "projects/{project}/serviceAccounts/{service_account}",
+        project="-",
+        service_account=service_account_email,
+    )
+    scope = [
+        "https://www.googleapis.com/auth/devstorage.read_write",
+        "https://www.googleapis.com/auth/iam",
+    ]
+    response = universe_domain_iam_client.generate_access_token(name=name, scope=scope)
+
+    _create_signed_read_url_helper(
+        universe_domain_client,
+        universe_domain_bucket,
+        version="v4",
+        service_account_email=service_account_email,
+        access_token=response.access_token,
+    )
+
+
 def _create_signed_delete_url_helper(client, bucket, version="v2", expiration=None):
     expiration = _morph_expiration(version, expiration)
 
diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py
index b0ff4f07b..d805017b9 100644
--- a/tests/unit/test_blob.py
+++ b/tests/unit/test_blob.py
@@ -487,6 +487,8 @@ def _generate_signed_url_helper(
             expected_creds = credentials
             client = self._make_client(_credentials=object())
 
+        expected_universe_domain = client.universe_domain
+
         bucket = _Bucket(client)
         blob = self._make_one(blob_name, bucket=bucket, encryption_key=encryption_key)
 
@@ -564,6 +566,7 @@ def _generate_signed_url_helper(
             "query_parameters": query_parameters,
             "access_token": access_token,
             "service_account_email": service_account_email,
+            "universe_domain": expected_universe_domain,
         }
         signer.assert_called_once_with(expected_creds, **expected_kwargs)
 

From a92542715a2969e04c944acf180c468504a772b2 Mon Sep 17 00:00:00 2001
From: cojenco <cathyo@google.com>
Date: Thu, 21 Nov 2024 15:56:03 -0800
Subject: [PATCH 16/17] tests: skip universe domain test in preprod (#1386)

---
 tests/system/test__signing.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py
index ee7a85fb7..cdf718d90 100644
--- a/tests/system/test__signing.py
+++ b/tests/system/test__signing.py
@@ -287,6 +287,10 @@ def test_create_signed_read_url_v4_w_access_token(
     )
 
 
+@pytest.mark.skipif(
+    _helpers.is_api_endpoint_override,
+    reason="Credentials not yet supported in preprod testing.",
+)
 def test_create_signed_read_url_v4_w_access_token_universe_domain(
     universe_domain_iam_client,
     universe_domain_client,

From 309bad16072c1d660799c2eed8f46434bc0a2f1d Mon Sep 17 00:00:00 2001
From: "release-please[bot]"
 <55107282+release-please[bot]@users.noreply.github.com>
Date: Wed, 4 Dec 2024 17:28:24 -0800
Subject: [PATCH 17/17] chore(main): release 2.19.0 (#1348)

Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com>
---
 CHANGELOG.md                    | 16 ++++++++++++++++
 google/cloud/storage/version.py |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06c80ebae..9f3883ec3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,22 @@
 
 [1]: https://pypi.org/project/google-cloud-storage/#history
 
+## [2.19.0](https://github.com/googleapis/python-storage/compare/v2.18.2...v2.19.0) (2024-11-21)
+
+
+### Features
+
+* Add integration test for universe domain ([#1346](https://github.com/googleapis/python-storage/issues/1346)) ([02a972d](https://github.com/googleapis/python-storage/commit/02a972d35fae6d05edfb26381f6a71e3b8f59d6d))
+* Add restore_bucket and handling for soft-deleted buckets ([#1365](https://github.com/googleapis/python-storage/issues/1365)) ([ab94efd](https://github.com/googleapis/python-storage/commit/ab94efda83f68c974ec91d6b869b09047501031a))
+* Add support for restore token ([#1369](https://github.com/googleapis/python-storage/issues/1369)) ([06ed15b](https://github.com/googleapis/python-storage/commit/06ed15b33dc884da6dffbef5119e47f0fc4e1285))
+* IAM signBlob retry and universe domain support ([#1380](https://github.com/googleapis/python-storage/issues/1380)) ([abc8061](https://github.com/googleapis/python-storage/commit/abc80615ee00a14bc0e6b095252f6d1eb09c4b45))
+
+
+### Bug Fixes
+
+* Allow signed post policy v4 with service account and token ([#1356](https://github.com/googleapis/python-storage/issues/1356)) ([8ec02c0](https://github.com/googleapis/python-storage/commit/8ec02c0e656a4e6786f256798f4b93b95b50acec))
+* Do not spam the log with checksum related INFO messages when downloading using transfer_manager ([#1357](https://github.com/googleapis/python-storage/issues/1357)) ([42392ef](https://github.com/googleapis/python-storage/commit/42392ef8e38527ce4e50454cdd357425b3f57c87))
+
 ## [2.18.2](https://github.com/googleapis/python-storage/compare/v2.18.1...v2.18.2) (2024-08-08)
 
 
diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py
index bbe5b63fe..2605c08a3 100644
--- a/google/cloud/storage/version.py
+++ b/google/cloud/storage/version.py
@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "2.18.2"
+__version__ = "2.19.0"