add another metric for retries

Emyrk · Emyrk · commit c7e53cd4af2d · 2024-10-24T11:53:59.000-04:00
diff --git a/coderd/coderdtest/promhelp/metrics.go b/coderd/coderdtest/promhelp/metrics.go
@@ -39,26 +39,34 @@ func Compare(reg prometheus.Gatherer, compare string, metricNames ...string) err
 
 // HistogramValue returns the value of a histogram metric with the given name and labels.
 func HistogramValue(t testing.TB, reg prometheus.Gatherer, metricName string, labels prometheus.Labels) *io_prometheus_client.Histogram {
+	t.Helper()
+
 	labeled := MetricValue(t, reg, metricName, labels)
 	require.NotNilf(t, labeled, "metric %q with labels %v not found", metricName, labels)
 	return labeled.GetHistogram()
 }
 
 // GaugeValue returns the value of a gauge metric with the given name and labels.
 func GaugeValue(t testing.TB, reg prometheus.Gatherer, metricName string, labels prometheus.Labels) int {
+	t.Helper()
+
 	labeled := MetricValue(t, reg, metricName, labels)
 	require.NotNilf(t, labeled, "metric %q with labels %v not found", metricName, labels)
 	return int(labeled.GetGauge().GetValue())
 }
 
 // CounterValue returns the value of a counter metric with the given name and labels.
 func CounterValue(t testing.TB, reg prometheus.Gatherer, metricName string, labels prometheus.Labels) int {
+	t.Helper()
+
 	labeled := MetricValue(t, reg, metricName, labels)
 	require.NotNilf(t, labeled, "metric %q with labels %v not found", metricName, labels)
 	return int(labeled.GetCounter().GetValue())
 }
 
 func MetricValue(t testing.TB, reg prometheus.Gatherer, metricName string, labels prometheus.Labels) *io_prometheus_client.Metric {
+	t.Helper()
+
 	metrics, err := reg.Gather()
 	require.NoError(t, err)
 
diff --git a/coderd/database/dbmetrics/dbmetrics.go b/coderd/database/dbmetrics/dbmetrics.go
@@ -14,8 +14,11 @@ import (
 
 type metricsStore struct {
 	database.Store
-	logger     slog.Logger
+	logger slog.Logger
+	// txDuration is how long transactions take to execute.
 	txDuration *prometheus.HistogramVec
+	// txRetries is how many retries we are seeing for a given tx.
+	txRetries *prometheus.CounterVec
 }
 
 // NewDBMetrics returns a database.Store that registers metrics for the database
@@ -27,6 +30,21 @@ func NewDBMetrics(s database.Store, logger slog.Logger, reg prometheus.Registere
 	if slices.Contains(s.Wrappers(), wrapname) {
 		return s
 	}
+	txRetries := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "coderd",
+		Subsystem: "db",
+		Name:      "tx_executions_count",
+		Help:      "Total count of transactions executed. 'retries' is expected to be 0 for a successful transaction.",
+	}, []string{
+		"success", // Did the InTx function return an error?
+		// Number of executions, since we have retry logic on serialization errors.
+		// retries = Executions - 1 (as 1 execute is expected)
+		"retries",
+		// Uniquely naming some transactions can help debug reoccurring errors.
+		"id",
+	})
+	reg.MustRegister(txRetries)
+
 	txDuration := prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "coderd",
 		Subsystem: "db",
@@ -35,15 +53,14 @@ func NewDBMetrics(s database.Store, logger slog.Logger, reg prometheus.Registere
 		Buckets:   prometheus.DefBuckets,
 	}, []string{
 		"success", // Did the InTx function return an error?
-		// Number of executions, since we have retry logic on serialization errors.
-		"executions",
 		// Uniquely naming some transactions can help debug reoccurring errors.
 		"id",
 	})
 	reg.MustRegister(txDuration)
 	return &metricsStore{
 		Store:      s,
 		txDuration: txDuration,
+		txRetries:  txRetries,
 		logger:     logger,
 	}
 }
@@ -61,14 +78,19 @@ func (m metricsStore) InTx(f func(database.Store) error, options *database.TxOpt
 	err := m.Store.InTx(f, options)
 	dur := time.Since(start)
 	// The number of unique label combinations is
-	// 2 x 3 (retry count) x #IDs
+	// 2 x #IDs x #of buckets
 	// So IDs should be used sparingly to prevent too much bloat.
 	m.txDuration.With(prometheus.Labels{
-		"success":    strconv.FormatBool(err == nil),
-		"executions": strconv.FormatInt(int64(options.ExecutionCount()), 10),
-		"id":         options.TxIdentifier, // Can be empty string for unlabeled
+		"success": strconv.FormatBool(err == nil),
+		"id":      options.TxIdentifier, // Can be empty string for unlabeled
 	}).Observe(dur.Seconds())
 
+	m.txRetries.With(prometheus.Labels{
+		"success": strconv.FormatBool(err == nil),
+		"retries": strconv.FormatInt(int64(options.ExecutionCount()-1), 10),
+		"id":      options.TxIdentifier, // Can be empty string for unlabeled
+	}).Inc()
+
 	// Log all serializable transactions that are retried.
 	// This is expected to happen in production, but should be kept
 	// to a minimum. If these logs happen frequently, something is wrong.
diff --git a/coderd/database/dbmetrics/dbmetrics_test.go b/coderd/database/dbmetrics/dbmetrics_test.go
@@ -2,7 +2,6 @@ package dbmetrics_test
 
 import (
 	"bytes"
-	"fmt"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -22,11 +21,11 @@ func TestInTxMetrics(t *testing.T) {
 	t.Parallel()
 
 	successLabels := prometheus.Labels{
-		"success":    "true",
-		"executions": "1",
-		"id":         "",
+		"success": "true",
+		"id":      "",
 	}
-	const inTxMetricName = "coderd_db_tx_duration_seconds"
+	const inTxHistMetricName = "coderd_db_tx_duration_seconds"
+	const inTxCountMetricName = "coderd_db_tx_executions_count"
 	t.Run("QueryMetrics", func(t *testing.T) {
 		t.Parallel()
 
@@ -40,7 +39,7 @@ func TestInTxMetrics(t *testing.T) {
 		require.NoError(t, err)
 
 		// Check that the metrics are registered
-		inTxMetric := promhelp.HistogramValue(t, reg, inTxMetricName, successLabels)
+		inTxMetric := promhelp.HistogramValue(t, reg, inTxHistMetricName, successLabels)
 		require.NotNil(t, inTxMetric)
 		require.Equal(t, uint64(1), inTxMetric.GetSampleCount())
 	})
@@ -58,7 +57,7 @@ func TestInTxMetrics(t *testing.T) {
 		require.NoError(t, err)
 
 		// Check that the metrics are registered
-		inTxMetric := promhelp.HistogramValue(t, reg, inTxMetricName, successLabels)
+		inTxMetric := promhelp.HistogramValue(t, reg, inTxHistMetricName, successLabels)
 		require.NotNil(t, inTxMetric)
 		require.Equal(t, uint64(1), inTxMetric.GetSampleCount())
 	})
@@ -85,21 +84,26 @@ func TestInTxMetrics(t *testing.T) {
 		require.Error(t, err)
 
 		// Check that the metrics are registered
-		inTxMetric := promhelp.HistogramValue(t, reg, inTxMetricName, prometheus.Labels{
-			"success":    "false",
-			"executions": "2",
-			"id":         id,
+		inTxHistMetric := promhelp.HistogramValue(t, reg, inTxHistMetricName, prometheus.Labels{
+			"success": "false",
+			"id":      id,
 		})
-		require.NotNil(t, inTxMetric)
-		require.Equal(t, uint64(1), inTxMetric.GetSampleCount())
+		require.NotNil(t, inTxHistMetric)
+		require.Equal(t, uint64(1), inTxHistMetric.GetSampleCount())
+
+		inTxCountMetric := promhelp.CounterValue(t, reg, inTxCountMetricName, prometheus.Labels{
+			"success": "false",
+			"retries": "1",
+			"id":      id,
+		})
+		require.NotNil(t, inTxCountMetric)
+		require.Equal(t, 1, inTxCountMetric)
 
 		// Also check the logs
 		require.Contains(t, output.String(), "some dumb error")
 		require.Contains(t, output.String(), "database transaction hit serialization error and had to retry")
 		require.Contains(t, output.String(), "success=false")
 		require.Contains(t, output.String(), "executions=2")
 		require.Contains(t, output.String(), "id="+id)
-
-		fmt.Println(promhelp.RegistryDump(reg))
 	})
 }
