diff --git a/goblet/common_cloud_actions.py b/goblet/common_cloud_actions.py index 48f218cc..d7edbea8 100644 --- a/goblet/common_cloud_actions.py +++ b/goblet/common_cloud_actions.py @@ -324,6 +324,28 @@ def getDefaultRegistryName(): return f"projects/{get_default_project()}/locations/{get_default_location()}/repositories/cloud-run-source-deploy" +def get_artifact_image_name(client, artifact_name, config): + """retrieves name of image stored in artifact registry""" + # Get artifact tag from env variable or config + try: + artifact_tag = os.environ["GOBLET_ARTIFACT_TAG"] + except KeyError: + artifact_tag = config.deploy.get("artifact_tag", None) + + if artifact_tag: + artifact_registry = config.deploy.get( + "artifact_registry", None + ) or getDefaultRegistry(artifact_name) + image_name = ( + artifact_registry + + ("@" if "sha256" in artifact_tag else ":") + + artifact_tag + ) + else: + image_name = getCloudbuildArtifact(client, artifact_name, config=config) + return image_name + + # calls latest build and checks for its artifact to avoid image:latest behavior with cloud run revisions def getCloudbuildArtifact(client, artifactName, config): defaultProject = get_default_project() diff --git a/goblet/handlers/jobs.py b/goblet/handlers/jobs.py index 94c6923e..a9fd4113 100644 --- a/goblet/handlers/jobs.py +++ b/goblet/handlers/jobs.py @@ -2,7 +2,7 @@ import os from goblet.handlers.handler import Handler -from goblet.common_cloud_actions import getCloudbuildArtifact +from goblet.common_cloud_actions import get_artifact_image_name from goblet.permissions import gcp_generic_resource_permissions from googleapiclient.errors import HttpError @@ -57,7 +57,7 @@ def _deploy(self, source=None, entrypoint=None): if not self.resources: return - artifact = getCloudbuildArtifact( + artifact = get_artifact_image_name( self.versioned_clients.cloudbuild, self.name, config=self.config ) diff --git a/goblet/revision.py b/goblet/revision.py index c1ec2812..9a20d3fb 100644 --- a/goblet/revision.py +++ b/goblet/revision.py @@ -7,11 +7,7 @@ ) from goblet_gcp_client.client import get_default_location -from goblet.common_cloud_actions import ( - deploy_cloudrun, - getCloudbuildArtifact, - getDefaultRegistry, -) +from goblet.common_cloud_actions import deploy_cloudrun, get_artifact_image_name from goblet.config import GConfig log = logging.getLogger("goblet.deployer") @@ -107,25 +103,9 @@ def deployRevision(self): region = get_default_location() project = get_default_project_number() - # Get artifact tag from env variable or config - try: - artifact_tag = os.environ["GOBLET_ARTIFACT_TAG"] - except KeyError: - artifact_tag = self.config.deploy.get("artifact_tag", None) - - if artifact_tag: - artifact_registry = self.config.deploy.get( - "artifact_registry", None - ) or getDefaultRegistry(self.name) - self.artifactToDeploy = ( - artifact_registry - + ("@" if "sha256" in artifact_tag else ":") - + artifact_tag - ) - else: - self.artifactToDeploy = getCloudbuildArtifact( - self.versioned_clients.cloudbuild, self.name, config=self.config - ) + self.artifactToDeploy = get_artifact_image_name( + self.versioned_clients.cloudbuild, self.name, config=self.config + ) self.req_body = { "template": { diff --git a/goblet/tests/data/http/jobs/deploy-from-artifact-tag/get-v2-projects-goblet-locations-us-central1-operations-269d9939-1325-4a1c-b9aa-06f42a1601e9_1.json b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/get-v2-projects-goblet-locations-us-central1-operations-269d9939-1325-4a1c-b9aa-06f42a1601e9_1.json new file mode 100644 index 00000000..7bcaf6a7 --- /dev/null +++ b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/get-v2-projects-goblet-locations-us-central1-operations-269d9939-1325-4a1c-b9aa-06f42a1601e9_1.json @@ -0,0 +1,99 @@ +{ + "headers": {}, + "body": { + "name": "projects/goblet/locations/us-central1/operations/269d9939-1325-4a1c-b9aa-06f42a1601e9", + "metadata": { + "@type": "type.googleapis.com/google.cloud.run.v2.Job", + "name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag", + "uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205", + "generation": "1", + "createTime": "2023-11-22T19:40:29.178576Z", + "updateTime": "2023-11-22T19:50:13.872450Z", + "launchStage": "GA", + "template": { + "taskCount": 1, + "template": { + "containers": [ + { + "image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a", + "command": [ + "goblet", + "job", + "run", + "goblet-jobs-test-artifact-tag" + ], + "resources": { + "limits": { + "memory": "512Mi", + "cpu": "1000m" + } + } + } + ], + "maxRetries": 3, + "timeout": "600s", + "serviceAccount": "1234567890-compute@developer.gserviceaccount.com", + "executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2" + } + }, + "observedGeneration": "1", + "terminalCondition": { + "type": "Ready", + "state": "CONDITION_SUCCEEDED", + "lastTransitionTime": "2023-11-22T19:40:29.399832Z" + }, + "latestCreatedExecution": { + "createTime": "1970-01-01T00:00:00Z", + "completionTime": "1970-01-01T00:00:00Z" + }, + "etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\"" + }, + "done": true, + "response": { + "@type": "type.googleapis.com/google.cloud.run.v2.Job", + "name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag", + "uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205", + "generation": "1", + "createTime": "2023-11-22T19:40:29.178576Z", + "updateTime": "2023-11-22T19:50:13.872450Z", + "launchStage": "GA", + "template": { + "taskCount": 1, + "template": { + "containers": [ + { + "image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a", + "command": [ + "goblet", + "job", + "run", + "goblet-jobs-test-artifact-tag" + ], + "resources": { + "limits": { + "memory": "512Mi", + "cpu": "1000m" + } + } + } + ], + "maxRetries": 3, + "timeout": "600s", + "serviceAccount": "1234567890-compute@developer.gserviceaccount.com", + "executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2" + } + }, + "observedGeneration": "1", + "terminalCondition": { + "type": "Ready", + "state": "CONDITION_SUCCEEDED", + "lastTransitionTime": "2023-11-22T19:40:29.399832Z" + }, + "latestCreatedExecution": { + "createTime": "1970-01-01T00:00:00Z", + "completionTime": "1970-01-01T00:00:00Z" + }, + "etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\"" + } + } +} \ No newline at end of file diff --git a/goblet/tests/data/http/jobs/deploy-from-artifact-tag/patch-v2-projects-goblet-locations-us-central1-jobs-goblet-jobs-test-artifact-tag_1.json b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/patch-v2-projects-goblet-locations-us-central1-jobs-goblet-jobs-test-artifact-tag_1.json new file mode 100644 index 00000000..7bcaf6a7 --- /dev/null +++ b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/patch-v2-projects-goblet-locations-us-central1-jobs-goblet-jobs-test-artifact-tag_1.json @@ -0,0 +1,99 @@ +{ + "headers": {}, + "body": { + "name": "projects/goblet/locations/us-central1/operations/269d9939-1325-4a1c-b9aa-06f42a1601e9", + "metadata": { + "@type": "type.googleapis.com/google.cloud.run.v2.Job", + "name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag", + "uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205", + "generation": "1", + "createTime": "2023-11-22T19:40:29.178576Z", + "updateTime": "2023-11-22T19:50:13.872450Z", + "launchStage": "GA", + "template": { + "taskCount": 1, + "template": { + "containers": [ + { + "image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a", + "command": [ + "goblet", + "job", + "run", + "goblet-jobs-test-artifact-tag" + ], + "resources": { + "limits": { + "memory": "512Mi", + "cpu": "1000m" + } + } + } + ], + "maxRetries": 3, + "timeout": "600s", + "serviceAccount": "1234567890-compute@developer.gserviceaccount.com", + "executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2" + } + }, + "observedGeneration": "1", + "terminalCondition": { + "type": "Ready", + "state": "CONDITION_SUCCEEDED", + "lastTransitionTime": "2023-11-22T19:40:29.399832Z" + }, + "latestCreatedExecution": { + "createTime": "1970-01-01T00:00:00Z", + "completionTime": "1970-01-01T00:00:00Z" + }, + "etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\"" + }, + "done": true, + "response": { + "@type": "type.googleapis.com/google.cloud.run.v2.Job", + "name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag", + "uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205", + "generation": "1", + "createTime": "2023-11-22T19:40:29.178576Z", + "updateTime": "2023-11-22T19:50:13.872450Z", + "launchStage": "GA", + "template": { + "taskCount": 1, + "template": { + "containers": [ + { + "image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a", + "command": [ + "goblet", + "job", + "run", + "goblet-jobs-test-artifact-tag" + ], + "resources": { + "limits": { + "memory": "512Mi", + "cpu": "1000m" + } + } + } + ], + "maxRetries": 3, + "timeout": "600s", + "serviceAccount": "1234567890-compute@developer.gserviceaccount.com", + "executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2" + } + }, + "observedGeneration": "1", + "terminalCondition": { + "type": "Ready", + "state": "CONDITION_SUCCEEDED", + "lastTransitionTime": "2023-11-22T19:40:29.399832Z" + }, + "latestCreatedExecution": { + "createTime": "1970-01-01T00:00:00Z", + "completionTime": "1970-01-01T00:00:00Z" + }, + "etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\"" + } + } +} \ No newline at end of file diff --git a/goblet/tests/data/http/jobs/deploy-from-artifact-tag/post-v2-projects-goblet-locations-us-central1-jobs_1.json b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/post-v2-projects-goblet-locations-us-central1-jobs_1.json new file mode 100644 index 00000000..94622a41 --- /dev/null +++ b/goblet/tests/data/http/jobs/deploy-from-artifact-tag/post-v2-projects-goblet-locations-us-central1-jobs_1.json @@ -0,0 +1,10 @@ +{ + "headers": {}, + "body": { + "error": { + "code": 409, + "message": "Resource 'goblet-jobs-test-artifact-tag' already exists.", + "status": "ALREADY_EXISTS" + } + } +} \ No newline at end of file diff --git a/goblet/tests/test_services.py b/goblet/tests/test_common_actions.py similarity index 53% rename from goblet/tests/test_services.py rename to goblet/tests/test_common_actions.py index 965a06e5..091242ba 100644 --- a/goblet/tests/test_services.py +++ b/goblet/tests/test_common_actions.py @@ -2,8 +2,39 @@ from goblet import Goblet +from goblet.common_cloud_actions import get_artifact_image_name + + +class TestCommonActions: + def test_get_artifact_image_name(self, monkeypatch): + monkeypatch.setenv("GOBLET_ARTIFACT_TAG", "test-tag-1") + monkeypatch.setenv("GOOGLE_PROJECT", "goblet") + monkeypatch.setenv("GOOGLE_LOCATION", "us-central1") + image_name = get_artifact_image_name(None, "test-artifact", Goblet().config) + assert ( + image_name + == "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/test-artifact:test-tag-1" + ) + + def test_get_artifact_image_name_with_registry(self, monkeypatch): + monkeypatch.setenv("GOBLET_ARTIFACT_TAG", "test-tag-1") + monkeypatch.setenv("GOOGLE_PROJECT", "goblet") + monkeypatch.setenv("GOOGLE_LOCATION", "us-central1") + + app = Goblet( + config={ + "deploy": { + "artifact_registry": "us-central1-docker.pkg.dev/goblet/repo-name/test-artifact" + } + }, + ) + + image_name = get_artifact_image_name(None, "test-artifact", app.config) + assert ( + image_name + == "us-central1-docker.pkg.dev/goblet/repo-name/test-artifact:test-tag-1" + ) -class TestServices: def test_check_service_apis_status(self, monkeypatch): app = Goblet(function_name="test-goblet-services") monkeypatch.setenv("G_TEST_NAME", "services-check") diff --git a/goblet/tests/test_jobs.py b/goblet/tests/test_jobs.py index 78745978..283cdec4 100644 --- a/goblet/tests/test_jobs.py +++ b/goblet/tests/test_jobs.py @@ -201,3 +201,26 @@ def test_job_iam_bindings(self, monkeypatch): == "serviceAccount:test@goblet.iam.gserviceaccount.com" ) assert len(schedule_bindings["body"]["bindings"][0]["members"]) == 2 + + def test_deploy_jobs_from_artifact_tag(self, monkeypatch): + artifact_tag = ( + "sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a" + ) + monkeypatch.setenv("GOOGLE_PROJECT", "goblet") + monkeypatch.setenv("GOOGLE_LOCATION", "us-central1") + monkeypatch.setenv("G_TEST_NAME", "jobs/deploy-from-artifact-tag") + monkeypatch.setenv("G_HTTP_TEST", "REPLAY") + monkeypatch.setenv("GOBLET_ARTIFACT_TAG", artifact_tag) + + app = Goblet(function_name="goblet-jobs", backend="cloudrun") + app.job("test-artifact-tag")(dummy_function) + app.deploy() + + responses = get_responses("jobs/deploy-from-artifact-tag") + assert len(responses) == 3 + assert ( + artifact_tag + in responses[1]["body"]["metadata"]["template"]["template"]["containers"][ + 0 + ]["image"] + )