goblet · anovis · Jan 9, 2024 · Jan 5, 2024 · Jan 5, 2024 · Jan 5, 2024
diff --git a/docs/source/backends.rst b/docs/source/backends.rst
@@ -29,7 +29,48 @@ Goblet's default backend is first-generation cloud functions. However, Goblet su
 * For cloudfunctions v1, python version must be at least python3.8, and for cloudfunctionv2, python version must be at least python3.8.
   To specify a python version for your cloudfunction, you can set the runtime field in config.json as such:
 
-{"cloudfunction": {"runtime": "python38"}}
+.. code:: json
+
+    {"cloudfunction": {"runtime": "python38"}}
+
+
+To set a custom GCS Bucket to upload the Cloudfunction sourcecode, use the `artifact_bucket` configuration in the `deploy` configuration key.
+
+.. code:: json
+
+    {
+        "deploy":{
+            "artifact_bucket": "name-of-the-bucket"
+        }
+    }
+
+or the ``GOBLET_ARTIFACT_BUCKET`` environment variable.
+
+
+
+To tag the `.zip` file with the code uploaded to the GCS Bucket, use the ``GOBLET_BUILD_TAGS`` environment variable with the tag.
+The Cloudfunction will be created using the file uploaded to this custom location.
+
+.. code:: bash
+
+    GOBLET_BUILD_TAGS=1cac04f3b
+
+In order to use ``GOBLET_BUILD_TAGS``, ``GOBLET_ARTIFACT_BUCKET`` must also have a value.
+
+
+To deploy a Cloudfunction from a tagged `.zip` file use the `artifact_tag` configuration in the `deploy` configuration key
+
+.. code:: json
+
+    {
+        "deploy":{
+            "artifact_tag": "1cac04f3b",
+        }
+    }
+
+or use the ``GOBLET_ARTIFACT_TAG`` environment variable.
+When using an `artifact_tag` to deploy, ``GOBLET_ARTIFACT_BUCKET`` must also have a value.
+
 
 * Goblet does not currently support eventarc triggers for cloudfunctions
 
@@ -130,7 +171,7 @@ This can be done by running:
 
 Here the service account from `project_b` is granted permissions to read from artifact registry en `project_a`
 
-For adding custom tags to be created in cloud build and pushed to artifact registry, use the `GOBLET_BUILD_TAGS` environment variable with a comma-sepparated list of tags to be created:
+To set the tags used to tag the image created in CloudBuild and pushed to artifact registry, use the `GOBLET_BUILD_TAGS` environment variable with a comma-sepparated list of tags to be created:
 
 .. code:: bash
 

diff --git a/goblet/backends/backend.py b/goblet/backends/backend.py
@@ -94,15 +94,46 @@ def delta(self, zip_path=None):
     def _checksum(self):
         raise NotImplementedError("_checksum")
 
-    def _gcs_upload(self, client, headers, upload_client=None, force=False):
+    def _gcs_upload(self, client, headers, upload_client=None, force=False, tag=None):
         self.log.info("zipping source code")
         self.zip()
         if not force and self.get() and not self.delta():
             self.log.info("No changes detected....")
             return None, False
         self.log.info("uploading source zip to gs......")
+
+        if tag:
+            return self._upload_tagged_zip(upload_client or client, tag, headers), True
+
         return self._upload_zip(upload_client or client, headers), True
 
+    def _upload_tagged_zip(self, client, tag, headers=None) -> dict:
+        self.zipf.close()
+        bucket_name = (
+            self.config.deploy.artifact_bucket or os.environ["GOBLET_ARTIFACT_BUCKET"]
+        )
+        try:
+            client.execute(
+                "insert",
+                params={
+                    "bucket": bucket_name,
+                    "uploadType": "media",
+                    "media_body": f".goblet/{self.name}.zip",
+                    "media_mime_type": "application/zip",
+                    "body": {
+                        "name": f"{self.name}-{tag}.zip",
+                    },
+                },
+            )
+
+        except requests.exceptions.HTTPError as e:
+            if not os.environ.get("G_HTTP_TEST") == "REPLAY":
+                raise e
+
+        self.log.info("source code uploaded")
+
+        return {"uploadUrl": f"gs://{bucket_name}/{self.name}-{tag}.zip"}
+
     def _upload_zip(self, client, headers=None) -> dict:
         """Uploads zipped cloudfunction using generateUploadUrl endpoint"""
         self.zipf.close()

diff --git a/goblet/backends/cloudfunctionv1.py b/goblet/backends/cloudfunctionv1.py
@@ -1,3 +1,5 @@
+import os
+
 from requests import request
 import base64
 
@@ -38,6 +40,7 @@ class CloudFunctionV1(Backend):
 
     def __init__(self, app):
         self.client = VersionedClients().cloudfunctions
+        self.storage_client = VersionedClients().storage_objects
         self.func_path = f"projects/{get_default_project()}/locations/{get_default_location()}/functions/{app.function_name}"
         super().__init__(app, self.client, self.func_path)
 
@@ -46,12 +49,42 @@ def deploy(self, force=False):
             "content-type": "application/zip",
             "x-goog-content-length-range": "0,104857600",
         }
-        source, changes = self._gcs_upload(self.client, put_headers, force=force)
-        if not changes:
-            return None
+
+        artifact_tag = self.config.deploy.artifact_tag or os.getenv(
+            "GOBLET_ARTIFACT_TAG", None
+        )
+
+        if not artifact_tag:
+            build_tag = None
+            upload_client = None
+            upload_method = None
+            build_tags = os.getenv("GOBLET_BUILD_TAGS", None)
+            if build_tags:
+                build_tag = build_tags.split(",")[0]
+                upload_client = self.storage_client
+                upload_method = "sourceArchiveUrl"
+
+            source, changes = self._gcs_upload(
+                self.client,
+                put_headers,
+                force=force,
+                upload_client=upload_client,
+                tag=build_tag,
+            )
+            if not changes:
+                return None
+        else:
+            bucket_name = (
+                self.config.deploy.artifact_bucket
+                or os.environ["GOBLET_ARTIFACT_BUCKET"]
+            )
+            source = {"uploadUrl": f"gs://{bucket_name}/{self.name}-{artifact_tag}.zip"}
+            upload_method = "sourceArchiveUrl"
 
         if self.app.is_http():
-            client, params = self._get_upload_params(source)
+            client, params = self._get_upload_params(
+                source, upload_method=upload_method
+            )
             create_cloudfunctionv1(client, params, config=self.config)
 
         return source
@@ -61,20 +94,23 @@ def destroy(self, all=False):
         if all:
             destroy_cloudfunction_artifacts(self.name)
 
-    def _get_upload_params(self, source):
+    def _get_upload_params(self, source, upload_method=None):
         user_configs = self.config.cloudfunction or {}
         params = {
             "body": {
                 "name": self.func_path,
                 "description": self.config.description or "created by goblet",
                 "entryPoint": "goblet_entrypoint",
-                "sourceUploadUrl": source["uploadUrl"],
                 "httpsTrigger": {},
                 "runtime": get_function_runtime(self.client, self.config),
                 "labels": {**self.config.labels},
                 **user_configs,
             }
         }
+        if not upload_method:
+            upload_method = "sourceUploadUrl"
+        params["body"][upload_method] = source["uploadUrl"]
+
         return self.client, params
 
     def _checksum(self):

diff --git a/...ZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy9BOGVQS3AtM3RHbw_1.json b/...ZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy9BOGVQS3AtM3RHbw_1.json
@@ -0,0 +1,48 @@
+{
+  "headers": {},
+  "body": {
+    "name": "operations/cHJlbWlzZS1kZXZlbG9wZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy9BOGVQS3AtM3RHbw",
+    "metadata": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.OperationMetadataV1",
+      "target": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "type": "CREATE_FUNCTION",
+      "request": {
+        "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+        "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+        "description": "created by goblet",
+        "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+        "httpsTrigger": {},
+        "entryPoint": "goblet_entrypoint",
+        "runtime": "python310"
+      },
+      "versionId": "1",
+      "updateTime": "2024-01-08T05:38:30Z",
+      "buildId": "99ac93f2-6815-4c62-a3ca-48ee88350564",
+      "buildName": "projects/test_project/locations/us-central1/builds/99ac93f2-6815-4c62-a3ca-48ee88350564"
+    },
+    "done": true,
+    "response": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+      "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "description": "created by goblet",
+      "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+      "httpsTrigger": {
+        "url": "https://us-central1-test_project.cloudfunctions.net/cloudfunction-build-tags",
+        "securityLevel": "SECURE_OPTIONAL"
+      },
+      "status": "ACTIVE",
+      "entryPoint": "goblet_entrypoint",
+      "timeout": "60s",
+      "availableMemoryMb": 256,
+      "serviceAccountEmail": "[email protected]",
+      "updateTime": "2024-01-08T05:38:30.323Z",
+      "versionId": "1",
+      "runtime": "python310",
+      "ingressSettings": "ALLOW_ALL",
+      "buildId": "99ac93f2-6815-4c62-a3ca-48ee88350564",
+      "buildName": "projects/test_project/locations/us-central1/builds/99ac93f2-6815-4c62-a3ca-48ee88350564",
+      "dockerRegistry": "CONTAINER_REGISTRY",
+      "automaticUpdatePolicy": {}
+    }
+  }
+}
diff --git a/...cloudfunction-artifact-tag/post-v1-projects-goblet-locations-us-central1-functions_1.json b/...cloudfunction-artifact-tag/post-v1-projects-goblet-locations-us-central1-functions_1.json
@@ -0,0 +1,22 @@
+{
+  "headers": {},
+  "body": {
+    "name": "operations/cHJlbWlzZS1kZXZlbG9wZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy9BOGVQS3AtM3RHbw",
+    "metadata": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.OperationMetadataV1",
+      "target": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "type": "CREATE_FUNCTION",
+      "request": {
+        "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+        "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+        "description": "created by goblet",
+        "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+        "httpsTrigger": {},
+        "entryPoint": "goblet_entrypoint",
+        "runtime": "python310"
+      },
+      "versionId": "1",
+      "updateTime": "2024-01-08T05:37:08Z"
+    }
+  }
+}
diff --git a/...ZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy83ZXFqdHhvcjBicw_1.json b/...ZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy83ZXFqdHhvcjBicw_1.json
@@ -0,0 +1,48 @@
+{
+  "headers": {},
+  "body": {
+    "name": "operations/cHJlbWlzZS1kZXZlbG9wZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy83ZXFqdHhvcjBicw",
+    "metadata": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.OperationMetadataV1",
+      "target": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "type": "UPDATE_FUNCTION",
+      "request": {
+        "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+        "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+        "description": "created by goblet",
+        "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+        "httpsTrigger": {},
+        "entryPoint": "goblet_entrypoint",
+        "runtime": "python310"
+      },
+      "versionId": "5",
+      "updateTime": "2024-01-05T19:26:44Z",
+      "buildId": "4a029f5f-0b13-42f0-9e31-8e4969167dfe",
+      "buildName": "projects/test_project/locations/us-central1/builds/4a029f5f-0b13-42f0-9e31-8e4969167dfe"
+    },
+    "done": true,
+    "response": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+      "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "description": "created by goblet",
+      "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+      "httpsTrigger": {
+        "url": "https://us-central1-goblet.cloudfunctions.net/cloudfunction-build-tags",
+        "securityLevel": "SECURE_OPTIONAL"
+      },
+      "status": "ACTIVE",
+      "entryPoint": "goblet_entrypoint",
+      "timeout": "60s",
+      "availableMemoryMb": 256,
+      "serviceAccountEmail": "[email protected]",
+      "updateTime": "2024-01-05T19:26:43.958Z",
+      "versionId": "5",
+      "runtime": "python310",
+      "ingressSettings": "ALLOW_ALL",
+      "buildId": "4a029f5f-0b13-42f0-9e31-8e4969167dfe",
+      "buildName": "projects/test_project/locations/us-central1/builds/4a029f5f-0b13-42f0-9e31-8e4969167dfe",
+      "dockerRegistry": "CONTAINER_REGISTRY",
+      "automaticUpdatePolicy": {}
+    }
+  }
+}
diff --git a/.../patch-v1-projects-goblet-locations-us-central1-functions-cloudfunction-build-tags_1.json b/.../patch-v1-projects-goblet-locations-us-central1-functions-cloudfunction-build-tags_1.json
@@ -0,0 +1,22 @@
+{
+  "headers": {},
+  "body": {
+    "name": "operations/cHJlbWlzZS1kZXZlbG9wZXItcG9ydGFsLXJkL3VzLWNlbnRyYWwxL2Nsb3VkZnVuY3Rpb24tYnVpbGQtdGFncy83ZXFqdHhvcjBicw",
+    "metadata": {
+      "@type": "type.googleapis.com/google.cloud.functions.v1.OperationMetadataV1",
+      "target": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+      "type": "UPDATE_FUNCTION",
+      "request": {
+        "@type": "type.googleapis.com/google.cloud.functions.v1.CloudFunction",
+        "name": "projects/test_project/locations/us-central1/functions/cloudfunction-build-tags",
+        "description": "created by goblet",
+        "sourceArchiveUrl": "gs://bucket/cloudfunction-build-tags-test.zip",
+        "httpsTrigger": {},
+        "entryPoint": "goblet_entrypoint",
+        "runtime": "python310"
+      },
+      "versionId": "5",
+      "updateTime": "2024-01-05T19:25:56Z"
+    }
+  }
+}
diff --git a/goblet/tests/data/http/cloudfunction-build-tags/post-upload-storage-v1-b-bucket-o_1.json b/goblet/tests/data/http/cloudfunction-build-tags/post-upload-storage-v1-b-bucket-o_1.json
@@ -0,0 +1,22 @@
+{
+  "headers": {},
+  "body": {
+    "kind": "storage#object",
+    "id": "bucket/cloudfunction-build-tags-test.zip/1704482754811322",
+    "selfLink": "https://www.googleapis.com/storage/v1/b/bucket/o/cloudfunction-build-tags-test.zip",
+    "mediaLink": "https://storage.googleapis.com/download/storage/v1/b/bucket/o/cloudfunction-build-tags-test.zip?generation=1704482754811322&alt=media",
+    "name": "cloudfunction-build-tags-test.zip",
+    "bucket": "bucket",
+    "generation": "1704482754811322",
+    "metageneration": "1",
+    "contentType": "application/zip",
+    "storageClass": "STANDARD",
+    "size": "36054",
+    "md5Hash": "HBI8hp2fuAgx9tPZb7iOMw==",
+    "crc32c": "XV7ndg==",
+    "etag": "CLqrgYn9xoMDEAE=",
+    "timeCreated": "2024-01-05T19:25:54.964Z",
+    "updated": "2024-01-05T19:25:54.964Z",
+    "timeStorageClassUpdated": "2024-01-05T19:25:54.964Z"
+  }
+}
diff --git a/...p/cloudfunction-build-tags/post-v1-projects-goblet-locations-us-central1-functions_1.json b/...p/cloudfunction-build-tags/post-v1-projects-goblet-locations-us-central1-functions_1.json
@@ -0,0 +1,10 @@
+{
+  "headers": {},
+  "body": {
+    "error": {
+      "code": 409,
+      "message": "Function cloudfunction-build-tags in region us-central1 in project goblet already exists",
+      "status": "ALREADY_EXISTS"
+    }
+  }
+}