feat(support): add client magicsock and agent prometheus metrics to support bundle (coder#12604)

johnstcn · mafredri · web-flow · commit b0c4e7504c5e · 2024-03-15T15:33:49.000Z
* feat(codersdk): add ability to fetch prometheus metrics directly from agent
* feat(support): add client magicsock and agent prometheus metrics to support bundle
* refactor(support): simplify AgentInfo control flow

Co-authored-by: Mathias Fredriksson &lt;mafredri@gmail.com&gt;
diff --git a/agent/agent.go b/agent/agent.go
@@ -25,6 +25,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/expfmt"
 	"github.com/spf13/afero"
 	"go.uber.org/atomic"
 	"golang.org/x/exp/slices"
@@ -34,6 +35,7 @@ import (
 	"tailscale.com/net/speedtest"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/netlogtype"
+	"tailscale.com/util/clientmetric"
 
 	"cdr.dev/slog"
 	"github.com/coder/retry"
@@ -1980,3 +1982,26 @@ func (a *apiConnRoutineManager) start(name string, b gracefulShutdownBehavior, f
 func (a *apiConnRoutineManager) wait() error {
 	return a.eg.Wait()
 }
+
+func PrometheusMetricsHandler(prometheusRegistry *prometheus.Registry, logger slog.Logger) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+
+		// Based on: https://github.com/tailscale/tailscale/blob/280255acae604796a1113861f5a84e6fa2dc6121/ipn/localapi/localapi.go#L489
+		clientmetric.WritePrometheusExpositionFormat(w)
+
+		metricFamilies, err := prometheusRegistry.Gather()
+		if err != nil {
+			logger.Error(context.Background(), "prometheus handler failed to gather metric families", slog.Error(err))
+			return
+		}
+
+		for _, metricFamily := range metricFamilies {
+			_, err = expfmt.MetricFamilyToText(w, metricFamily)
+			if err != nil {
+				logger.Error(context.Background(), "expfmt.MetricFamilyToText failed", slog.Error(err))
+				return
+			}
+		}
+	})
+}
diff --git a/agent/api.go b/agent/api.go
@@ -35,11 +35,13 @@ func (a *agent) apiHandler() http.Handler {
 		ignorePorts:   cpy,
 		cacheDuration: cacheDuration,
 	}
+	promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)
 	r.Get("/api/v0/listening-ports", lp.handler)
 	r.Get("/debug/logs", a.HandleHTTPDebugLogs)
 	r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)
 	r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)
 	r.Get("/debug/manifest", a.HandleHTTPDebugManifest)
+	r.Get("/debug/prometheus", promHandler.ServeHTTP)
 
 	return r
 }
diff --git a/cli/agent.go b/cli/agent.go
@@ -18,10 +18,8 @@ import (
 	"cloud.google.com/go/compute/metadata"
 	"golang.org/x/xerrors"
 	"gopkg.in/natefinch/lumberjack.v2"
-	"tailscale.com/util/clientmetric"
 
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/common/expfmt"
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
@@ -315,7 +313,8 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd {
 				ModifiedProcesses: nil,
 			})
 
-			prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus")
+			promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
+			prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
 			defer prometheusSrvClose()
 
 			debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
@@ -501,26 +500,3 @@ func urlPort(u string) (int, error) {
 	}
 	return -1, xerrors.Errorf("invalid port: %s", u)
 }
-
-func prometheusMetricsHandler(prometheusRegistry *prometheus.Registry, logger slog.Logger) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "text/plain")
-
-		// Based on: https://github.com/tailscale/tailscale/blob/280255acae604796a1113861f5a84e6fa2dc6121/ipn/localapi/localapi.go#L489
-		clientmetric.WritePrometheusExpositionFormat(w)
-
-		metricFamilies, err := prometheusRegistry.Gather()
-		if err != nil {
-			logger.Error(context.Background(), "Prometheus handler can't gather metric families", slog.Error(err))
-			return
-		}
-
-		for _, metricFamily := range metricFamilies {
-			_, err = expfmt.MetricFamilyToText(w, metricFamily)
-			if err != nil {
-				logger.Error(context.Background(), "expfmt.MetricFamilyToText failed", slog.Error(err))
-				return
-			}
-		}
-	})
-}
diff --git a/cli/support.go b/cli/support.go
@@ -176,8 +176,10 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 		"network/tailnet_debug.html":     src.Network.TailnetDebug,
 		"workspace/build_logs.txt":       humanizeBuildLogs(src.Workspace.BuildLogs),
 		"agent/logs.txt":                 string(src.Agent.Logs),
-		"agent/magicsock.html":           string(src.Agent.MagicsockHTML),
+		"agent/agent_magicsock.html":     string(src.Agent.AgentMagicsockHTML),
+		"agent/client_magicsock.html":    string(src.Agent.ClientMagicsockHTML),
 		"agent/startup_logs.txt":         humanizeAgentLogs(src.Agent.StartupLogs),
+		"agent/prometheus.txt":           string(src.Agent.Prometheus),
 		"workspace/template_file.zip":    string(templateVersionBytes),
 		"logs.txt":                       strings.Join(src.Logs, "\n"),
 	} {
diff --git a/cli/support_test.go b/cli/support_test.go
@@ -177,9 +177,12 @@ func assertBundleContents(t *testing.T, path string) {
 		case "agent/logs.txt":
 			bs := readBytesFromZip(t, f)
 			require.NotEmpty(t, bs, "logs should not be empty")
-		case "agent/magicsock.html":
+		case "agent/agent_magicsock.html":
 			bs := readBytesFromZip(t, f)
 			require.NotEmpty(t, bs, "agent magicsock should not be empty")
+		case "agent/client_magicsock.html":
+			bs := readBytesFromZip(t, f)
+			require.NotEmpty(t, bs, "client magicsock should not be empty")
 		case "agent/manifest.json":
 			var v agentsdk.Manifest
 			decodeJSONFromZip(t, f, &v)
@@ -192,6 +195,9 @@ func assertBundleContents(t *testing.T, path string) {
 			var v *ipnstate.PingResult
 			decodeJSONFromZip(t, f, &v)
 			require.NotEmpty(t, v, "ping result should not be empty")
+		case "agent/prometheus.txt":
+			bs := readBytesFromZip(t, f)
+			require.NotEmpty(t, bs, "agent prometheus metrics should not be empty")
 		case "agent/startup_logs.txt":
 			bs := readBytesFromZip(t, f)
 			require.Contains(t, string(bs), "started up")
diff --git a/codersdk/workspaceagentconn.go b/codersdk/workspaceagentconn.go
@@ -364,6 +364,9 @@ func (c *WorkspaceAgentConn) DebugMagicsock(ctx context.Context) ([]byte, error)
 	if err != nil {
 		return nil, xerrors.Errorf("do request: %w", err)
 	}
+	if res.StatusCode != http.StatusOK {
+		return nil, ReadBodyAsError(res)
+	}
 	defer res.Body.Close()
 	bs, err := io.ReadAll(res.Body)
 	if err != nil {
@@ -382,6 +385,9 @@ func (c *WorkspaceAgentConn) DebugManifest(ctx context.Context) ([]byte, error)
 		return nil, xerrors.Errorf("do request: %w", err)
 	}
 	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, ReadBodyAsError(res)
+	}
 	bs, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, xerrors.Errorf("read response body: %w", err)
@@ -398,6 +404,28 @@ func (c *WorkspaceAgentConn) DebugLogs(ctx context.Context) ([]byte, error) {
 		return nil, xerrors.Errorf("do request: %w", err)
 	}
 	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, ReadBodyAsError(res)
+	}
+	bs, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, xerrors.Errorf("read response body: %w", err)
+	}
+	return bs, nil
+}
+
+// PrometheusMetrics returns a response from the agent's prometheus metrics endpoint
+func (c *WorkspaceAgentConn) PrometheusMetrics(ctx context.Context) ([]byte, error) {
+	ctx, span := tracing.StartSpan(ctx)
+	defer span.End()
+	res, err := c.apiRequest(ctx, http.MethodGet, "/debug/prometheus", nil)
+	if err != nil {
+		return nil, xerrors.Errorf("do request: %w", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, ReadBodyAsError(res)
+	}
 	bs, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, xerrors.Errorf("read response body: %w", err)
diff --git a/support/support.go b/support/support.go
diff --git a/support/support_test.go b/support/support_test.go

Original file line number	Diff line number	Diff line change
`@@ -35,11 +35,13 @@ func (a *agent) apiHandler() http.Handler {`
`35`	`35`	`ignorePorts: cpy,`
`36`	`36`	`cacheDuration: cacheDuration,`
`37`	`37`	`}`
	`38`	`+ promHandler := PrometheusMetricsHandler(a.prometheusRegistry, a.logger)`
`38`	`39`	`r.Get("/api/v0/listening-ports", lp.handler)`
`39`	`40`	`r.Get("/debug/logs", a.HandleHTTPDebugLogs)`
`40`	`41`	`r.Get("/debug/magicsock", a.HandleHTTPDebugMagicsock)`
`41`	`42`	`r.Get("/debug/magicsock/debug-logging/{state}", a.HandleHTTPMagicsockDebugLoggingState)`
`42`	`43`	`r.Get("/debug/manifest", a.HandleHTTPDebugManifest)`
	`44`	`+ r.Get("/debug/prometheus", promHandler.ServeHTTP)`
`43`	`45`
`44`	`46`	`return r`
`45`	`47`	`}`