Thanks to visit codestin.com
Credit goes to github.com

Skip to content

[jobs] deploy from artifact tag #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anovis merged 3 commits into from
Nov 22, 2023
Merged

[jobs] deploy from artifact tag #462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2260ae0
deploy cloud run job from artifact tag
quajones Nov 22, 2023
1f8a404
fix naming covention
quajones Nov 22, 2023
cd90fd8
Merge branch 'main' into get-artifact-image-name
quajones Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions goblet/common_cloud_actions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,28 @@ def getDefaultRegistryName():
return f"projects/{get_default_project()}/locations/{get_default_location()}/repositories/cloud-run-source-deploy"


def get_artifact_image_name(client, artifact_name, config):
"""retrieves name of image stored in artifact registry"""
# Get artifact tag from env variable or config
try:
artifact_tag = os.environ["GOBLET_ARTIFACT_TAG"]
except KeyError:
artifact_tag = config.deploy.get("artifact_tag", None)

if artifact_tag:
artifact_registry = config.deploy.get(
"artifact_registry", None
) or getDefaultRegistry(artifact_name)
image_name = (
artifact_registry
+ ("@" if "sha256" in artifact_tag else ":")
+ artifact_tag
)
else:
image_name = getCloudbuildArtifact(client, artifact_name, config=config)
return image_name


# calls latest build and checks for its artifact to avoid image:latest behavior with cloud run revisions
def getCloudbuildArtifact(client, artifactName, config):
defaultProject = get_default_project()
Expand Down
4 changes: 2 additions & 2 deletions goblet/handlers/jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os

from goblet.handlers.handler import Handler
from goblet.common_cloud_actions import getCloudbuildArtifact
from goblet.common_cloud_actions import get_artifact_image_name
from goblet.permissions import gcp_generic_resource_permissions

from googleapiclient.errors import HttpError
Expand Down Expand Up @@ -57,7 +57,7 @@ def _deploy(self, source=None, entrypoint=None):
if not self.resources:
return

artifact = getCloudbuildArtifact(
artifact = get_artifact_image_name(
self.versioned_clients.cloudbuild, self.name, config=self.config
)

Expand Down
28 changes: 4 additions & 24 deletions goblet/revision.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@
)

from goblet_gcp_client.client import get_default_location
from goblet.common_cloud_actions import (
deploy_cloudrun,
getCloudbuildArtifact,
getDefaultRegistry,
)
from goblet.common_cloud_actions import deploy_cloudrun, get_artifact_image_name
from goblet.config import GConfig

log = logging.getLogger("goblet.deployer")
Expand Down Expand Up @@ -107,25 +103,9 @@ def deployRevision(self):
region = get_default_location()
project = get_default_project_number()

# Get artifact tag from env variable or config
try:
artifact_tag = os.environ["GOBLET_ARTIFACT_TAG"]
except KeyError:
artifact_tag = self.config.deploy.get("artifact_tag", None)

if artifact_tag:
artifact_registry = self.config.deploy.get(
"artifact_registry", None
) or getDefaultRegistry(self.name)
self.artifactToDeploy = (
artifact_registry
+ ("@" if "sha256" in artifact_tag else ":")
+ artifact_tag
)
else:
self.artifactToDeploy = getCloudbuildArtifact(
self.versioned_clients.cloudbuild, self.name, config=self.config
)
self.artifactToDeploy = get_artifact_image_name(
self.versioned_clients.cloudbuild, self.name, config=self.config
)

self.req_body = {
"template": {
Expand Down
99 changes: 99 additions & 0 deletions ...jects-goblet-locations-us-central1-operations-269d9939-1325-4a1c-b9aa-06f42a1601e9_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
{
"headers": {},
"body": {
"name": "projects/goblet/locations/us-central1/operations/269d9939-1325-4a1c-b9aa-06f42a1601e9",
"metadata": {
"@type": "type.googleapis.com/google.cloud.run.v2.Job",
"name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag",
"uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205",
"generation": "1",
"createTime": "2023-11-22T19:40:29.178576Z",
"updateTime": "2023-11-22T19:50:13.872450Z",
"launchStage": "GA",
"template": {
"taskCount": 1,
"template": {
"containers": [
{
"image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a",
"command": [
"goblet",
"job",
"run",
"goblet-jobs-test-artifact-tag"
],
"resources": {
"limits": {
"memory": "512Mi",
"cpu": "1000m"
}
}
}
],
"maxRetries": 3,
"timeout": "600s",
"serviceAccount": "[email protected]",
"executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2"
}
},
"observedGeneration": "1",
"terminalCondition": {
"type": "Ready",
"state": "CONDITION_SUCCEEDED",
"lastTransitionTime": "2023-11-22T19:40:29.399832Z"
},
"latestCreatedExecution": {
"createTime": "1970-01-01T00:00:00Z",
"completionTime": "1970-01-01T00:00:00Z"
},
"etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\""
},
"done": true,
"response": {
"@type": "type.googleapis.com/google.cloud.run.v2.Job",
"name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag",
"uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205",
"generation": "1",
"createTime": "2023-11-22T19:40:29.178576Z",
"updateTime": "2023-11-22T19:50:13.872450Z",
"launchStage": "GA",
"template": {
"taskCount": 1,
"template": {
"containers": [
{
"image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a",
"command": [
"goblet",
"job",
"run",
"goblet-jobs-test-artifact-tag"
],
"resources": {
"limits": {
"memory": "512Mi",
"cpu": "1000m"
}
}
}
],
"maxRetries": 3,
"timeout": "600s",
"serviceAccount": "[email protected]",
"executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2"
}
},
"observedGeneration": "1",
"terminalCondition": {
"type": "Ready",
"state": "CONDITION_SUCCEEDED",
"lastTransitionTime": "2023-11-22T19:40:29.399832Z"
},
"latestCreatedExecution": {
"createTime": "1970-01-01T00:00:00Z",
"completionTime": "1970-01-01T00:00:00Z"
},
"etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\""
}
}
}
99 changes: 99 additions & 0 deletions .../patch-v2-projects-goblet-locations-us-central1-jobs-goblet-jobs-test-artifact-tag_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
{
"headers": {},
"body": {
"name": "projects/goblet/locations/us-central1/operations/269d9939-1325-4a1c-b9aa-06f42a1601e9",
"metadata": {
"@type": "type.googleapis.com/google.cloud.run.v2.Job",
"name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag",
"uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205",
"generation": "1",
"createTime": "2023-11-22T19:40:29.178576Z",
"updateTime": "2023-11-22T19:50:13.872450Z",
"launchStage": "GA",
"template": {
"taskCount": 1,
"template": {
"containers": [
{
"image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a",
"command": [
"goblet",
"job",
"run",
"goblet-jobs-test-artifact-tag"
],
"resources": {
"limits": {
"memory": "512Mi",
"cpu": "1000m"
}
}
}
],
"maxRetries": 3,
"timeout": "600s",
"serviceAccount": "[email protected]",
"executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2"
}
},
"observedGeneration": "1",
"terminalCondition": {
"type": "Ready",
"state": "CONDITION_SUCCEEDED",
"lastTransitionTime": "2023-11-22T19:40:29.399832Z"
},
"latestCreatedExecution": {
"createTime": "1970-01-01T00:00:00Z",
"completionTime": "1970-01-01T00:00:00Z"
},
"etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\""
},
"done": true,
"response": {
"@type": "type.googleapis.com/google.cloud.run.v2.Job",
"name": "projects/goblet/locations/us-central1/jobs/goblet-jobs-test-artifact-tag",
"uid": "eb8d6a69-6a35-48b0-8ac4-e807be9eb205",
"generation": "1",
"createTime": "2023-11-22T19:40:29.178576Z",
"updateTime": "2023-11-22T19:50:13.872450Z",
"launchStage": "GA",
"template": {
"taskCount": 1,
"template": {
"containers": [
{
"image": "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/goblet-jobs@sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a",
"command": [
"goblet",
"job",
"run",
"goblet-jobs-test-artifact-tag"
],
"resources": {
"limits": {
"memory": "512Mi",
"cpu": "1000m"
}
}
}
],
"maxRetries": 3,
"timeout": "600s",
"serviceAccount": "[email protected]",
"executionEnvironment": "EXECUTION_ENVIRONMENT_GEN2"
}
},
"observedGeneration": "1",
"terminalCondition": {
"type": "Ready",
"state": "CONDITION_SUCCEEDED",
"lastTransitionTime": "2023-11-22T19:40:29.399832Z"
},
"latestCreatedExecution": {
"createTime": "1970-01-01T00:00:00Z",
"completionTime": "1970-01-01T00:00:00Z"
},
"etag": "\"CPW2-aoGENCPgqAD/cHJvamVjdHMvcHJlbWlzZS13YWxsZXQtZGV2L2xvY2F0aW9ucy91cy1jZW50cmFsMS9qb2JzL2dvYmxldC1qb2JzLXRlc3QtYXJ0aWZhY3QtdGFn\""
}
}
}
10 changes: 10 additions & 0 deletions ...p/jobs/deploy-from-artifact-tag/post-v2-projects-goblet-locations-us-central1-jobs_1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"headers": {},
"body": {
"error": {
"code": 409,
"message": "Resource 'goblet-jobs-test-artifact-tag' already exists.",
"status": "ALREADY_EXISTS"
}
}
}
33 changes: 32 additions & 1 deletion goblet/tests/test_services.py → goblet/tests/test_common_actions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,39 @@

from goblet import Goblet

from goblet.common_cloud_actions import get_artifact_image_name


class TestCommonActions:
def test_get_artifact_image_name(self, monkeypatch):
monkeypatch.setenv("GOBLET_ARTIFACT_TAG", "test-tag-1")
monkeypatch.setenv("GOOGLE_PROJECT", "goblet")
monkeypatch.setenv("GOOGLE_LOCATION", "us-central1")
image_name = get_artifact_image_name(None, "test-artifact", Goblet().config)
assert (
image_name
== "us-central1-docker.pkg.dev/goblet/cloud-run-source-deploy/test-artifact:test-tag-1"
)

def test_get_artifact_image_name_with_registry(self, monkeypatch):
monkeypatch.setenv("GOBLET_ARTIFACT_TAG", "test-tag-1")
monkeypatch.setenv("GOOGLE_PROJECT", "goblet")
monkeypatch.setenv("GOOGLE_LOCATION", "us-central1")

app = Goblet(
config={
"deploy": {
"artifact_registry": "us-central1-docker.pkg.dev/goblet/repo-name/test-artifact"
}
},
)

image_name = get_artifact_image_name(None, "test-artifact", app.config)
assert (
image_name
== "us-central1-docker.pkg.dev/goblet/repo-name/test-artifact:test-tag-1"
)

class TestServices:
def test_check_service_apis_status(self, monkeypatch):
app = Goblet(function_name="test-goblet-services")
monkeypatch.setenv("G_TEST_NAME", "services-check")
Expand Down
23 changes: 23 additions & 0 deletions goblet/tests/test_jobs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,26 @@ def test_job_iam_bindings(self, monkeypatch):
== "serviceAccount:[email protected]"
)
assert len(schedule_bindings["body"]["bindings"][0]["members"]) == 2

def test_deploy_jobs_from_artifact_tag(self, monkeypatch):
artifact_tag = (
"sha256:0a05e8ee3a7a3527dee34999247e29f19c4cf7941750a3267bb9b1a2f37b724a"
)
monkeypatch.setenv("GOOGLE_PROJECT", "goblet")
monkeypatch.setenv("GOOGLE_LOCATION", "us-central1")
monkeypatch.setenv("G_TEST_NAME", "jobs/deploy-from-artifact-tag")
monkeypatch.setenv("G_HTTP_TEST", "REPLAY")
monkeypatch.setenv("GOBLET_ARTIFACT_TAG", artifact_tag)

app = Goblet(function_name="goblet-jobs", backend="cloudrun")
app.job("test-artifact-tag")(dummy_function)
app.deploy()

responses = get_responses("jobs/deploy-from-artifact-tag")
assert len(responses) == 3
assert (
artifact_tag
in responses[1]["body"]["metadata"]["template"]["template"]["containers"][
0
]["image"]
)