target · dctalbot · Mar 8, 2022 · Feb 2, 2022 · Feb 7, 2022 · Feb 7, 2022
diff --git a/engine/engine.go b/engine/engine.go
@@ -14,6 +14,7 @@ import (
 	"github.com/target/goalert/engine/escalationmanager"
 	"github.com/target/goalert/engine/heartbeatmanager"
 	"github.com/target/goalert/engine/message"
+	"github.com/target/goalert/engine/metricsmanager"
 	"github.com/target/goalert/engine/npcyclemanager"
 	"github.com/target/goalert/engine/processinglock"
 	"github.com/target/goalert/engine/rotationmanager"
@@ -123,6 +124,10 @@ func NewEngine(ctx context.Context, db *sql.DB, c *Config) (*Engine, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "cleanup backend")
 	}
+	metricsMgr, err := metricsmanager.NewDB(ctx, db)
+	if err != nil {
+		return nil, errors.Wrap(err, "metrics management backend")
+	}
 
 	p.modules = []updater{
 		rotMgr,
@@ -133,6 +138,7 @@ func NewEngine(ctx context.Context, db *sql.DB, c *Config) (*Engine, error) {
 		verifyMgr,
 		hbMgr,
 		cleanMgr,
+		metricsMgr,
 	}
 
 	p.msg, err = message.NewDB(ctx, db, c.AlertLogStore, p.mgr)

diff --git a/engine/metricsmanager/db.go b/engine/metricsmanager/db.go
@@ -0,0 +1,75 @@
+package metricsmanager
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/target/goalert/engine/processinglock"
+	"github.com/target/goalert/util"
+)
+
+const engineVersion = 1
+
+// DB handles updating metrics
+type DB struct {
+	db   *sql.DB
+	lock *processinglock.Lock
+
+	highAlertID *sql.Stmt
+	lowAlertID  *sql.Stmt
+
+	recentlyClosed *sql.Stmt
+	scanAlerts     *sql.Stmt
+	insertMetrics  *sql.Stmt
+}
+
+// Name returns the name of the module.
+func (db *DB) Name() string { return "Engine.MetricsManager" }
+
+// NewDB creates a new DB.
+func NewDB(ctx context.Context, db *sql.DB) (*DB, error) {
+	lock, err := processinglock.NewLock(ctx, db, processinglock.Config{
+		Version: engineVersion,
+		Type:    processinglock.TypeMetrics,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	p := &util.Prepare{Ctx: ctx, DB: db}
+
+	return &DB{
+		db:   db,
+		lock: lock,
+
+		highAlertID: p.P(`select max(id) from alerts where status = 'closed'`),
+		lowAlertID:  p.P(`select min(id) from alerts where status = 'closed'`),
+
+		recentlyClosed: p.P(`
+			select distinct log.alert_id
+			from alert_logs log
+			left join alert_metrics m on m.alert_id = log.alert_id
+			where m isnull and log.event = 'closed' and log.timestamp >= now() - '1 hour'::interval
+			limit 500
+		`),
+
+		scanAlerts: p.P(`
+			select a.id
+			from alerts a
+			left join alert_metrics m on m.alert_id = a.id
+			where m isnull and a.status = 'closed' and a.id between $1 and $2
+		`),
+
+		insertMetrics: p.P(`
+			insert into alert_metrics (alert_id, service_id, time_to_ack, time_to_close, escalated)
+			select
+				a.id,
+				a.service_id,
+				(select timestamp - a.created_at from alert_logs where alert_id = a.id and event = 'acknowledged' order by timestamp limit 1),
+				(select timestamp - a.created_at from alert_logs where alert_id = a.id and event = 'closed'       order by timestamp limit 1),
+				(select count(*) > 1             from alert_logs where alert_id = a.id and event = 'escalated')
+			from alerts a
+			where a.id = any($1) and a.service_id is not null
+		`),
+	}, p.Err
+}
diff --git a/engine/metricsmanager/update.go b/engine/metricsmanager/update.go
@@ -0,0 +1,139 @@
+package metricsmanager
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/target/goalert/permission"
+	"github.com/target/goalert/util/log"
+	"github.com/target/goalert/util/sqlutil"
+)
+
+type State struct {
+	V1 struct {
+		NextAlertID int
+	}
+}
+
+/*
+	Theory of Operation:
+
+	1. Aquire processing lock
+	2. Look for recently closed alerts without a metrics entry
+	3. If any, insert metrics for them and exit
+	4. If no state, start scan from last closed alert id
+	5. If state, resume scan until min closed alert id
+
+*/
+
+// UpdateAll will update the alert metrics table
+func (db *DB) UpdateAll(ctx context.Context) error {
+	err := permission.LimitCheckAny(ctx, permission.System)
+	if err != nil {
+		return err
+	}
+	log.Debugf(ctx, "Running metrics operations.")
+
+	tx, lockState, err := db.lock.BeginTxWithState(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("begin tx: %w", err)
+	}
+	defer tx.Rollback()
+
+	rows, err := tx.StmtContext(ctx, db.recentlyClosed).QueryContext(ctx)
+	if err != nil {
+		return fmt.Errorf("query recently closed alerts: %w", err)
+	}
+	defer rows.Close()
+
+	var alertIDs []int
+	for rows.Next() {
+		var alertID int
+		err = rows.Scan(&alertID)
+		if err != nil {
+			return fmt.Errorf("scan alert id: %w", err)
+		}
+		alertIDs = append(alertIDs, alertID)
+	}
+
+	if len(alertIDs) > 0 {
+		_, err = tx.StmtContext(ctx, db.insertMetrics).ExecContext(ctx, sqlutil.IntArray(alertIDs))
+		if err != nil {
+			return fmt.Errorf("insert metrics: %w", err)
+		}
+		err = tx.Commit()
+		if err != nil {
+			return fmt.Errorf("commit: %w", err)
+		}
+		return nil
+	}
+
+	// fetch min alert id from db for later
+	var minAlertID sql.NullInt64
+	err = tx.StmtContext(ctx, db.lowAlertID).QueryRowContext(ctx).Scan(&minAlertID)
+	if err != nil {
+		return fmt.Errorf("query min alert id: %w", err)
+	}
+
+	if !minAlertID.Valid {
+		// no alerts
+		return nil
+	}
+
+	var state State
+	err = lockState.Load(ctx, &state)
+	if err != nil {
+		return fmt.Errorf("load state: %w", err)
+	}
+
+	if state.V1.NextAlertID == 0 || state.V1.NextAlertID < int(minAlertID.Int64) {
+		// no state, or reset, set to the highest alert id from the db
+		err = tx.StmtContext(ctx, db.highAlertID).QueryRowContext(ctx).Scan(&state.V1.NextAlertID)
+		if err != nil {
+			return fmt.Errorf("query high alert id: %w", err)
+		}
+	}
+
+	// clamp min alert ID 500 below next
+	if int(minAlertID.Int64) < state.V1.NextAlertID-500 {
+		minAlertID.Int64 = int64(state.V1.NextAlertID) - 500
+	}
+
+	// fetch alerts to update
+	rows, err = tx.StmtContext(ctx, db.scanAlerts).QueryContext(ctx, minAlertID, state.V1.NextAlertID)
+	if err != nil {
+		return fmt.Errorf("query alerts: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var alertID int
+		err = rows.Scan(&alertID)
+		if err != nil {
+			return fmt.Errorf("scan alert id: %w", err)
+		}
+		alertIDs = append(alertIDs, alertID)
+	}
+
+	if len(alertIDs) > 0 {
+		_, err = tx.StmtContext(ctx, db.insertMetrics).ExecContext(ctx, sqlutil.IntArray(alertIDs))
+		if err != nil {
+			return fmt.Errorf("insert metrics: %w", err)
+		}
+	}
+
+	// update and save state
+	state.V1.NextAlertID = int(minAlertID.Int64) - 1
+	err = lockState.Save(ctx, &state)
+	if err != nil {
+		return fmt.Errorf("save state: %w", err)
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		return fmt.Errorf("commit: %w", err)
+	}
+
+	return nil
+}
diff --git a/engine/processinglock/lock.go b/engine/processinglock/lock.go
@@ -17,6 +17,9 @@ type Lock struct {
 	db       *sql.DB
 	lockStmt *sql.Stmt
 
+	loadState *sql.Stmt
+	saveState *sql.Stmt
+
 	advLockStmt *sql.Stmt
 }
 type txBeginner interface {
@@ -36,6 +39,8 @@ func NewLock(ctx context.Context, db *sql.DB, cfg Config) (*Lock, error) {
 			where type_id = $1
 			for update nowait
 		`),
+		loadState: p.P(`select state from engine_processing_versions where type_id = $1 for update nowait`),
+		saveState: p.P(`update engine_processing_versions set state = $2 where type_id = $1`),
 	}, p.Err
 }
 

diff --git a/engine/processinglock/state.go b/engine/processinglock/state.go
@@ -0,0 +1,53 @@
+package processinglock
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+
+	"github.com/target/goalert/util/jsonutil"
+)
+
+// State manages the state value for a processing lock.
+type State struct {
+	data []byte
+	tx   *sql.Tx
+	l    *Lock
+}
+
+// BeginTxWithState will start a transaction, returning a State object.
+func (l *Lock) BeginTxWithState(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, *State, error) {
+	tx, err := l.BeginTx(ctx, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return tx, &State{tx: tx, l: l}, nil
+}
+
+// Load will load the JSON state from the database.
+func (s *State) Load(ctx context.Context, v interface{}) error {
+	err := s.tx.StmtContext(ctx, s.l.loadState).QueryRowContext(ctx, s.l.cfg.Type).Scan(&s.data)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(s.data, v)
+}
+
+// Save will save the JSON state to the database, taking care to ensure that
+// existing unknown fields are preserved.
+func (s *State) Save(ctx context.Context, v interface{}) error {
+	data, err := jsonutil.Apply(s.data, v)
+	if err != nil {
+		return err
+	}
+	s.data = data
+
+	_, err = s.tx.StmtContext(ctx, s.l.saveState).ExecContext(ctx, s.l.cfg.Type, s.data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/engine/processinglock/type.go b/engine/processinglock/type.go
@@ -1,11 +1,5 @@
 package processinglock
 
-import (
-	"database/sql/driver"
-	"fmt"
-	"github.com/target/goalert/validation/validate"
-)
-
 // Type indicates the lock type. For TypeMessage, the RegionID is used.
 type Type string
 
@@ -20,62 +14,5 @@ const (
 	TypeVerify       Type = "verify"
 	TypeMessage      Type = "message"
 	TypeCleanup      Type = "cleanup"
+	TypeMetrics      Type = "metrics"
 )
-
-func (t Type) validate() error {
-	return validate.OneOf("Type", t,
-		TypeEscalation,
-		TypeHeartbeat,
-		TypeNPCycle,
-		TypeRotation,
-		TypeSchedule,
-		TypeStatusUpdate,
-		TypeVerify,
-		TypeMessage,
-		TypeCleanup,
-	)
-}
-
-// Value will return the DB enum value of the Type.
-func (t Type) Value() (driver.Value, error) {
-	return string(t), t.validate()
-}
-
-// Scan will scan a DB enum value into Type.
-func (t *Type) Scan(value interface{}) error {
-	switch _t := value.(type) {
-	case []byte:
-		*t = Type(_t)
-	case string:
-		*t = Type(_t)
-	default:
-		return fmt.Errorf("could not process unknown type for Type(%T)", t)
-	}
-	return t.validate()
-}
-
-// LockID returns the int value used for the advisory lock for the Type.
-func (t Type) LockID() int {
-	switch t {
-	case TypeEscalation:
-		return 0x1000 // 4096
-	case TypeHeartbeat:
-		return 0x1010 // 4112
-	case TypeNPCycle:
-		return 0x1020 // 4128
-	case TypeRotation:
-		return 0x1030 // 4144
-	case TypeSchedule:
-		return 0x1040 // 4160
-	case TypeStatusUpdate:
-		return 0x1050 // 4176
-	case TypeVerify:
-		return 0x1060 // 4192
-	case TypeMessage:
-		return 0x1070 // 4208
-	case TypeCleanup:
-		return 0x1080 // 4224
-	}
-
-	panic("invalid type")
-}
diff --git a/migrate/migrations/20220307103153-add-metrics-proctype.sql b/migrate/migrations/20220307103153-add-metrics-proctype.sql
@@ -0,0 +1,4 @@
+-- +migrate Up notransaction
+ALTER TYPE engine_processing_type ADD VALUE IF NOT EXISTS 'metrics';
+
+-- +migrate Down