Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Fix sync and default alert type #475

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
anovis merged 2 commits into from
Jan 4, 2024
Merged

Fix sync and default alert type #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a7c3181
Fix sync and handlers
anovis Jan 4, 2024
918a09c
Fix test
anovis Jan 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions goblet/alerts/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
BackendAlert,
PubSubDLQAlert,
UptimeAlert,
AlertType,
Alert,
)
from goblet.alerts.alert_conditions import (
MetricCondition,
Expand Down
46 changes: 35 additions & 11 deletions goblet/alerts/alerts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
import os
import goblet.globals as g

from enum import Enum

from goblet.permissions import gcp_generic_resource_permissions
from goblet.client import VersionedClients
from goblet.errors import GobletValidationError
Expand Down Expand Up @@ -81,20 +83,40 @@ def destroy(self, alert_type):

def sync(self, dryrun=False):
# Does not sync custom metrics
for alert_name, alert in self.gcp_deployed_alerts.values():
for alert_name, alert in self.gcp_deployed_alerts.items():
if not self.resources.get(alert_name):
log.info(f"Detected unused alert {alert_name}")
if not dryrun:
self.resources[alert_name].destroy(alert["name"])
try:
self.versioned_clients.monitoring_alert.execute(
"delete",
parent_key="name",
parent_schema=alert["name"],
)
log.info(f"Destroying alert {self.name}......")
except HttpError as e:
if e.resp.status == 404:
log.info(f"Alert {self.name} already destroyed")
else:
raise e


class AlertType(Enum):
INFRA = "infra"
BACKEND = "backend"
HANDLER = "handler"
DEFAULT = "default"


class Alert:
"""Cloud Monitoring Alert Policies that can trigger notification channels based on built in or custom metrics.
https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.alertPolicies. Alerts and Alert conditions contain a
few common defaults, that are used by GCP. These can be overriden by passing in the correct params.

The default alert_type is Default
"""

alert_type = None
alert_type = AlertType.DEFAULT
extras = {}

def __init__(self, name, conditions, channels=[], extras=None, **kwargs) -> None:
Expand Down Expand Up @@ -128,32 +150,34 @@ def deploy(self, app_name, gcp_deployed_alerts):

default_alert_kwargs.update(self.kwargs)

alert_name = f"{self.app_name}-{self.name}"

body = {
"displayName": f"{self.app_name}-{self.name}",
"displayName": alert_name,
"conditions": formatted_conditions,
"notificationChannels": self.notification_channels,
"combiner": "OR",
**default_alert_kwargs,
}
# check if exists
if self.name in gcp_deployed_alerts:
if alert_name in gcp_deployed_alerts:
# patch
self.versioned_clients.monitoring_alert.execute(
"patch",
parent_key="name",
parent_schema=gcp_deployed_alerts[self.name]["name"],
parent_schema=gcp_deployed_alerts[alert_name]["name"],
params={"updateMask": ",".join(body.keys()), "body": body},
)

log.info(f"updated alert: {self.name}")
log.info(f"updated alert: {alert_name}")
else:
# deploy
self.versioned_clients.monitoring_alert.execute(
"create",
parent_key="name",
params={"body": body},
)
log.info(f"created alert: {self.name}")
log.info(f"created alert: {alert_name}")

def destroy(self, app_name, full_alert_name):
self.app_name = app_name
Expand Down Expand Up @@ -190,7 +214,7 @@ def update_extras(self, extras):


class BackendAlert(Alert):
alert_type = "backend"
alert_type = AlertType.BACKEND

def validate_extras(self):
return list(self.extras.keys()) == [
Expand All @@ -209,7 +233,7 @@ def _condition_arguments(self):


class PubSubDLQAlert(Alert):
alert_type = "handler"
alert_type = AlertType.HANDLER

def validate_extras(self):
return "topic" in self.extras.keys()
Expand All @@ -222,7 +246,7 @@ def _condition_arguments(self):


class UptimeAlert(Alert):
alert_type = "handler"
alert_type = AlertType.HANDLER

def validate_extras(self):
return "check_name" in self.extras.keys()
Expand Down
19 changes: 12 additions & 7 deletions goblet/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from goblet.config import GConfig
from goblet.decorators import Goblet_Decorators
from goblet.resource_manager import Resource_Manager
from goblet.alerts import AlertType

logging.basicConfig()

Expand Down Expand Up @@ -102,7 +103,7 @@ def deploy(
self.deploy_infrastructure(infras)

if not skip_alerts:
self.deploy_alerts(alert_type="infra")
self.deploy_alerts(alert_type=AlertType.INFRA)

infra_config = self.get_infrastructure_config()
backend.update_config(infra_config, write_config, stage)
Expand All @@ -112,7 +113,7 @@ def deploy(
source = backend.deploy(force=force)

if not skip_alerts:
self.deploy_alerts(alert_type="backend")
self.deploy_alerts(alert_type=AlertType.BACKEND)

registered_handlers = self.get_registered_handler_resource_types()

Expand All @@ -132,7 +133,8 @@ def deploy(
self.deploy_handlers(source, handlers)

if not skip_alerts:
self.deploy_alerts(alert_type="handler")
self.deploy_alerts(alert_type=AlertType.HANDLER)
self.deploy_alerts(alert_type=AlertType.DEFAULT)

def destroy(
self,
Expand All @@ -151,23 +153,26 @@ def destroy(
self.destroy_handlers(handlers)

if not skip_alerts:
log.info("destroying default alerts")
self.destroy_alerts(AlertType.DEFAULT)

log.info("destroying handler alerts")
self.destroy_alerts("handler")
self.destroy_alerts(AlertType.HANDLER)

if not skip_backend:
self.backend.destroy(all=all)

if not skip_alerts:
log.info("destroying backend alerts")
self.destroy_alerts("backend")
self.destroy_alerts(AlertType.BACKEND)

if infras or not skip_infra:
log.info("destroying infrastructure")
self.destroy_infrastructure(infras)

if not skip_alerts:
log.info("destroying infra alerts")
self.destroy_alerts("infra")
self.destroy_alerts(AlertType.INFRA)

def sync(
self,
Expand All @@ -186,7 +191,7 @@ def sync(
self.sync_handlers(dryrun, handlers)
if not skip_alerts:
log.info("syncing alerts")
self.alerts.sync()
self.alerts.sync(dryrun=dryrun)

def check_or_enable_services(self, enable=False):
self.backend._check_or_enable_service(enable)
Expand Down
12 changes: 0 additions & 12 deletions goblet/tests/test_alerts.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,21 +196,9 @@ def test_destroy_alerts(self, monkeypatch):
],
)

# pubsub_dlq = PubSubDLQAlert(
# "pubsubdlq",
# conditions=[
# PubSubDLQCondition(
# "pubsubdlq",
# subscription_id="pubsub-deploy-subscription",
# )
# ],
# extras={"topic": "subscription"},
# )

app.alert(metric_alert)
app.alert(log_alert)
app.alert(custom_alert)
# app.alert(pubsub_dlq)

app.destroy()

Expand Down