From 62c055105236ac3068446d468f230cd50aae6cfe Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 19 Mar 2024 08:20:14 +0000 Subject: [PATCH 1/5] Export metric indicating each experiment's status Signed-off-by: Danny Kopping --- cli/server.go | 4 +++ coderd/prometheusmetrics/prometheusmetrics.go | 25 +++++++++++++ .../prometheusmetrics_test.go | 35 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/cli/server.go b/cli/server.go index f57d761da7481..82ecb5edc2afa 100644 --- a/cli/server.go +++ b/cli/server.go @@ -892,6 +892,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("register agents prometheus metric: %w", err) } defer closeAgentsFunc() + + if err = prometheusmetrics.Experiments(logger, options.PrometheusRegistry, options.DeploymentValues.Experiments.Value(), codersdk.ExperimentsAll); err != nil { + return xerrors.Errorf("register experiments metric: %w", err) + } } client := codersdk.New(localURL) diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go index 36bf9b887d2c0..147b2d95c50cc 100644 --- a/coderd/prometheusmetrics/prometheusmetrics.go +++ b/coderd/prometheusmetrics/prometheusmetrics.go @@ -516,6 +516,31 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R }, nil } +// Experiments registers a metric which indicates whether each experiment is enabled or not. +func Experiments(_ slog.Logger, registerer prometheus.Registerer, exps []string, all codersdk.Experiments) error { + experimentsGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Name: "experiments", + Help: "Indicates whether each experiment is enabled (1) or not (0)", + }, []string{"experiment"}) + if err := registerer.Register(experimentsGauge); err != nil { + return err + } + + for _, exp := range all { + var val float64 + for _, enabled := range exps { + if string(exp) == enabled { + val = 1 + } + } + + experimentsGauge.WithLabelValues(string(exp)).Set(val) + } + + return nil +} + // filterAcceptableAgentLabels handles a slightly messy situation whereby `prometheus-aggregate-agent-stats-by` can control on // which labels agent stats are aggregated, but for these specific metrics in this file there is no `template` label value, // and therefore we have to exclude it from the list of acceptable labels. diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go index 3992792266c65..c962f03f7c2cb 100644 --- a/coderd/prometheusmetrics/prometheusmetrics_test.go +++ b/coderd/prometheusmetrics/prometheusmetrics_test.go @@ -500,6 +500,41 @@ func TestAgentStats(t *testing.T) { assert.EqualValues(t, golden, collected) } +func TestExperimentsMetric(t *testing.T) { + t.Parallel() + + log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) + reg := prometheus.NewRegistry() + + const ( + a codersdk.Experiment = "a" + b codersdk.Experiment = "b" + c codersdk.Experiment = "c" + ) + allExps := codersdk.Experiments{a, b, c} + require.NoError(t, prometheusmetrics.Experiments(log, reg, []string{string(b), string(c)}, allExps)) + + expectation := map[codersdk.Experiment]float64{ + a: 0, + b: 1, + c: 1, + } + + out, err := reg.Gather() + require.NoError(t, err) + require.Lenf(t, out, 1, "unexpected number of registered metrics") + + for _, metric := range out[0].GetMetric() { + labels := metric.GetLabel() + require.Lenf(t, labels, 1, "unexpected number of labels") + + experiment := labels[0].GetValue() + expected, found := expectation[codersdk.Experiment(experiment)] + require.Truef(t, found, "did not find experiment %q in expectations", experiment) + require.EqualValues(t, expected, metric.GetGauge().GetValue()) + } +} + func prepareWorkspaceAndAgent(t *testing.T, client *codersdk.Client, user codersdk.CreateFirstUserResponse, workspaceNum int) *agentsdk.Client { authToken := uuid.NewString() From c93e4e23c5a5c847a5691fc7dda7eb4d2dec931e Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 19 Mar 2024 09:17:38 +0000 Subject: [PATCH 2/5] Test metric name Signed-off-by: Danny Kopping --- cli/cliui/provisionerjob_test.go | 3 ++- coderd/prometheusmetrics/prometheusmetrics_test.go | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/cliui/provisionerjob_test.go b/cli/cliui/provisionerjob_test.go index f3661ca8d1597..f75a8bc53f12a 100644 --- a/cli/cliui/provisionerjob_test.go +++ b/cli/cliui/provisionerjob_test.go @@ -11,9 +11,10 @@ import ( "testing" "time" - "github.com/coder/coder/v2/testutil" "github.com/stretchr/testify/assert" + "github.com/coder/coder/v2/testutil" + "github.com/coder/coder/v2/cli/cliui" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/codersdk" diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go index c962f03f7c2cb..8f8f8a2bb77fd 100644 --- a/coderd/prometheusmetrics/prometheusmetrics_test.go +++ b/coderd/prometheusmetrics/prometheusmetrics_test.go @@ -525,6 +525,8 @@ func TestExperimentsMetric(t *testing.T) { require.Lenf(t, out, 1, "unexpected number of registered metrics") for _, metric := range out[0].GetMetric() { + require.Equal(t, "coderd_experiments", out[0].GetName()) + labels := metric.GetLabel() require.Lenf(t, labels, 1, "unexpected number of labels") From 6f351d7ec2aa3b9e05958918f17572a2600be71a Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 19 Mar 2024 09:20:10 +0000 Subject: [PATCH 3/5] Ignore gocognit lint error for now Signed-off-by: Danny Kopping --- cli/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/server.go b/cli/server.go index 82ecb5edc2afa..43534d7bd7e5a 100644 --- a/cli/server.go +++ b/cli/server.go @@ -258,6 +258,7 @@ func enablePrometheus( ), nil } +//nolint:gocognit // TODO(dannyk): reduce complexity of this function func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *serpent.Command { if newAPI == nil { newAPI = func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) { From 50ccb7c970c9fa0a5afd470e18fc5b13a2edfd92 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 19 Mar 2024 11:25:54 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: Mathias Fredriksson --- coderd/prometheusmetrics/prometheusmetrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go index 147b2d95c50cc..d10c99ae6e5f5 100644 --- a/coderd/prometheusmetrics/prometheusmetrics.go +++ b/coderd/prometheusmetrics/prometheusmetrics.go @@ -532,6 +532,7 @@ func Experiments(_ slog.Logger, registerer prometheus.Registerer, exps []string, for _, enabled := range exps { if string(exp) == enabled { val = 1 + break } } From 11abaa6ecf9c6f09367748ae4119c2dfda30db63 Mon Sep 17 00:00:00 2001 From: Danny Kopping Date: Tue, 19 Mar 2024 11:18:11 +0000 Subject: [PATCH 5/5] Review comments, expanding tests Signed-off-by: Danny Kopping --- cli/server.go | 7 +- coderd/prometheusmetrics/prometheusmetrics.go | 8 +- .../prometheusmetrics_test.go | 95 ++++++++++++++----- 3 files changed, 80 insertions(+), 30 deletions(-) diff --git a/cli/server.go b/cli/server.go index 43534d7bd7e5a..78f7270074f81 100644 --- a/cli/server.go +++ b/cli/server.go @@ -894,7 +894,12 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. } defer closeAgentsFunc() - if err = prometheusmetrics.Experiments(logger, options.PrometheusRegistry, options.DeploymentValues.Experiments.Value(), codersdk.ExperimentsAll); err != nil { + var active codersdk.Experiments + for _, exp := range options.DeploymentValues.Experiments.Value() { + active = append(active, codersdk.Experiment(exp)) + } + + if err = prometheusmetrics.Experiments(options.PrometheusRegistry, active); err != nil { return xerrors.Errorf("register experiments metric: %w", err) } } diff --git a/coderd/prometheusmetrics/prometheusmetrics.go b/coderd/prometheusmetrics/prometheusmetrics.go index d10c99ae6e5f5..b2c4b46677eb6 100644 --- a/coderd/prometheusmetrics/prometheusmetrics.go +++ b/coderd/prometheusmetrics/prometheusmetrics.go @@ -517,7 +517,7 @@ func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.R } // Experiments registers a metric which indicates whether each experiment is enabled or not. -func Experiments(_ slog.Logger, registerer prometheus.Registerer, exps []string, all codersdk.Experiments) error { +func Experiments(registerer prometheus.Registerer, active codersdk.Experiments) error { experimentsGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: "coderd", Name: "experiments", @@ -527,10 +527,10 @@ func Experiments(_ slog.Logger, registerer prometheus.Registerer, exps []string, return err } - for _, exp := range all { + for _, exp := range codersdk.ExperimentsAll { var val float64 - for _, enabled := range exps { - if string(exp) == enabled { + for _, enabled := range active { + if exp == enabled { val = 1 break } diff --git a/coderd/prometheusmetrics/prometheusmetrics_test.go b/coderd/prometheusmetrics/prometheusmetrics_test.go index 8f8f8a2bb77fd..32e97f84c32b1 100644 --- a/coderd/prometheusmetrics/prometheusmetrics_test.go +++ b/coderd/prometheusmetrics/prometheusmetrics_test.go @@ -503,37 +503,82 @@ func TestAgentStats(t *testing.T) { func TestExperimentsMetric(t *testing.T) { t.Parallel() - log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) - reg := prometheus.NewRegistry() + tests := []struct { + name string + experiments codersdk.Experiments + expected map[codersdk.Experiment]float64 + }{ + { + name: "Enabled experiment is exported in metrics", + experiments: codersdk.Experiments{codersdk.ExperimentSharedPorts}, + expected: map[codersdk.Experiment]float64{ + codersdk.ExperimentSharedPorts: 1, + }, + }, + { + name: "Disabled experiment is exported in metrics", + experiments: codersdk.Experiments{}, + expected: map[codersdk.Experiment]float64{ + codersdk.ExperimentSharedPorts: 0, + }, + }, + { + name: "Unknown experiment is not exported in metrics", + experiments: codersdk.Experiments{codersdk.Experiment("bob")}, + expected: map[codersdk.Experiment]float64{}, + }, + } - const ( - a codersdk.Experiment = "a" - b codersdk.Experiment = "b" - c codersdk.Experiment = "c" - ) - allExps := codersdk.Experiments{a, b, c} - require.NoError(t, prometheusmetrics.Experiments(log, reg, []string{string(b), string(c)}, allExps)) + for _, tc := range tests { + tc := tc - expectation := map[codersdk.Experiment]float64{ - a: 0, - b: 1, - c: 1, - } + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + reg := prometheus.NewRegistry() - out, err := reg.Gather() - require.NoError(t, err) - require.Lenf(t, out, 1, "unexpected number of registered metrics") + require.NoError(t, prometheusmetrics.Experiments(reg, tc.experiments)) + + out, err := reg.Gather() + require.NoError(t, err) + require.Lenf(t, out, 1, "unexpected number of registered metrics") + + seen := make(map[codersdk.Experiment]float64) + + for _, metric := range out[0].GetMetric() { + require.Equal(t, "coderd_experiments", out[0].GetName()) + + labels := metric.GetLabel() + require.Lenf(t, labels, 1, "unexpected number of labels") - for _, metric := range out[0].GetMetric() { - require.Equal(t, "coderd_experiments", out[0].GetName()) + experiment := codersdk.Experiment(labels[0].GetValue()) + value := metric.GetGauge().GetValue() - labels := metric.GetLabel() - require.Lenf(t, labels, 1, "unexpected number of labels") + seen[experiment] = value - experiment := labels[0].GetValue() - expected, found := expectation[codersdk.Experiment(experiment)] - require.Truef(t, found, "did not find experiment %q in expectations", experiment) - require.EqualValues(t, expected, metric.GetGauge().GetValue()) + expectedValue := 0 + + // Find experiment we expect to be enabled. + for _, exp := range tc.experiments { + if experiment == exp { + expectedValue = 1 + break + } + } + + require.EqualValuesf(t, expectedValue, value, "expected %d value for experiment %q", expectedValue, experiment) + } + + // We don't want to define the state of all experiments because codersdk.ExperimentAll will change at some + // point and break these tests; so we only validate the experiments we know about. + for exp, val := range seen { + expectedVal, found := tc.expected[exp] + if !found { + t.Logf("ignoring experiment %q; it is not listed in expectations", exp) + continue + } + require.Equalf(t, expectedVal, val, "experiment %q did not match expected value %v", exp, expectedVal) + } + }) } }