diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index ed7968577b455..734dac12f99c8 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -12388,6 +12388,9 @@ const docTemplate = `{ }, "websocket": { "$ref": "#/definitions/healthcheck.WebsocketReport" + }, + "workspace_proxy": { + "$ref": "#/definitions/healthcheck.WorkspaceProxyReport" } } }, @@ -12427,6 +12430,29 @@ const docTemplate = `{ } } }, + "healthcheck.WorkspaceProxyReport": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "severity": { + "$ref": "#/definitions/health.Severity" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "workspace_proxies": { + "$ref": "#/definitions/codersdk.RegionsResponse-codersdk_WorkspaceProxy" + } + } + }, "netcheck.Report": { "type": "object", "properties": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index f92e11b92609f..a9dd58f9c7a3e 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -11277,6 +11277,9 @@ }, "websocket": { "$ref": "#/definitions/healthcheck.WebsocketReport" + }, + "workspace_proxy": { + "$ref": "#/definitions/healthcheck.WorkspaceProxyReport" } } }, @@ -11312,6 +11315,29 @@ } } }, + "healthcheck.WorkspaceProxyReport": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "healthy": { + "type": "boolean" + }, + "severity": { + "$ref": "#/definitions/health.Severity" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "workspace_proxies": { + "$ref": "#/definitions/codersdk.RegionsResponse-codersdk_WorkspaceProxy" + } + } + }, "netcheck.Report": { "type": "object", "properties": { diff --git a/coderd/coderd.go b/coderd/coderd.go index 8b54b708cc85d..50cad7e0f8a2f 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -135,10 +135,12 @@ type Options struct { AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore] // AppSecurityKey is the crypto key used to sign and encrypt tokens related to // workspace applications. It consists of both a signing and encryption key. - AppSecurityKey workspaceapps.SecurityKey - HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report - HealthcheckTimeout time.Duration - HealthcheckRefresh time.Duration + AppSecurityKey workspaceapps.SecurityKey + + HealthcheckFunc func(ctx context.Context, apiKey string) *healthcheck.Report + HealthcheckTimeout time.Duration + HealthcheckRefresh time.Duration + WorkspaceProxiesFetchUpdater *atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater] // OAuthSigningKey is the crypto key used to sign and encrypt state strings // related to OAuth. This is a symmetric secret key using hmac to sign payloads. @@ -396,6 +398,13 @@ func New(options *Options) *API { *options.UpdateCheckOptions, ) } + + if options.WorkspaceProxiesFetchUpdater == nil { + options.WorkspaceProxiesFetchUpdater = &atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]{} + var wpfu healthcheck.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{} + options.WorkspaceProxiesFetchUpdater.Store(&wpfu) + } + if options.HealthcheckFunc == nil { options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report { return healthcheck.Run(ctx, &healthcheck.ReportOptions{ @@ -413,9 +422,14 @@ func New(options *Options) *API { DerpHealth: derphealth.ReportOptions{ DERPMap: api.DERPMap(), }, + WorkspaceProxy: healthcheck.WorkspaceProxyReportOptions{ + CurrentVersion: buildinfo.Version(), + WorkspaceProxiesFetchUpdater: *(options.WorkspaceProxiesFetchUpdater).Load(), + }, }) } } + if options.HealthcheckTimeout == 0 { options.HealthcheckTimeout = 30 * time.Second } diff --git a/coderd/healthcheck/healthcheck.go b/coderd/healthcheck/healthcheck.go index d5498482f977e..9233626419634 100644 --- a/coderd/healthcheck/healthcheck.go +++ b/coderd/healthcheck/healthcheck.go @@ -13,10 +13,11 @@ import ( ) const ( - SectionDERP string = "DERP" - SectionAccessURL string = "AccessURL" - SectionWebsocket string = "Websocket" - SectionDatabase string = "Database" + SectionDERP string = "DERP" + SectionAccessURL string = "AccessURL" + SectionWebsocket string = "Websocket" + SectionDatabase string = "Database" + SectionWorkspaceProxy string = "WorkspaceProxy" ) type Checker interface { @@ -24,6 +25,7 @@ type Checker interface { AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport + WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport } // @typescript-generate Report @@ -38,20 +40,22 @@ type Report struct { // FailingSections is a list of sections that have failed their healthcheck. FailingSections []string `json:"failing_sections"` - DERP derphealth.Report `json:"derp"` - AccessURL AccessURLReport `json:"access_url"` - Websocket WebsocketReport `json:"websocket"` - Database DatabaseReport `json:"database"` + DERP derphealth.Report `json:"derp"` + AccessURL AccessURLReport `json:"access_url"` + Websocket WebsocketReport `json:"websocket"` + Database DatabaseReport `json:"database"` + WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"` // The Coder version of the server that the report was generated on. CoderVersion string `json:"coder_version"` } type ReportOptions struct { - AccessURL AccessURLReportOptions - Database DatabaseReportOptions - DerpHealth derphealth.ReportOptions - Websocket WebsocketReportOptions + AccessURL AccessURLReportOptions + Database DatabaseReportOptions + DerpHealth derphealth.ReportOptions + Websocket WebsocketReportOptions + WorkspaceProxy WorkspaceProxyReportOptions Checker Checker } @@ -78,6 +82,11 @@ func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions) return report } +func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) { + report.Run(ctx, opts) + return report +} + func Run(ctx context.Context, opts *ReportOptions) *Report { var ( wg sync.WaitGroup @@ -136,6 +145,18 @@ func Run(ctx context.Context, opts *ReportOptions) *Report { report.Database = opts.Checker.Database(ctx, &opts.Database) }() + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if err := recover(); err != nil { + report.WorkspaceProxy.Error = ptr.Ref(fmt.Sprint(err)) + } + }() + + report.WorkspaceProxy = opts.Checker.WorkspaceProxy(ctx, &opts.WorkspaceProxy) + }() + report.CoderVersion = buildinfo.Version() wg.Wait() @@ -153,6 +174,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report { if !report.Database.Healthy { report.FailingSections = append(report.FailingSections, SectionDatabase) } + if !report.WorkspaceProxy.Healthy { + report.FailingSections = append(report.FailingSections, SectionWorkspaceProxy) + } report.Healthy = len(report.FailingSections) == 0 @@ -171,6 +195,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report { if report.Database.Severity.Value() > report.Severity.Value() { report.Severity = report.Database.Severity } + if report.WorkspaceProxy.Severity.Value() > report.Severity.Value() { + report.Severity = report.WorkspaceProxy.Severity + } return &report } diff --git a/coderd/healthcheck/healthcheck_test.go b/coderd/healthcheck/healthcheck_test.go index ddd268861c126..6ea8993209603 100644 --- a/coderd/healthcheck/healthcheck_test.go +++ b/coderd/healthcheck/healthcheck_test.go @@ -12,10 +12,11 @@ import ( ) type testChecker struct { - DERPReport derphealth.Report - AccessURLReport healthcheck.AccessURLReport - WebsocketReport healthcheck.WebsocketReport - DatabaseReport healthcheck.DatabaseReport + DERPReport derphealth.Report + AccessURLReport healthcheck.AccessURLReport + WebsocketReport healthcheck.WebsocketReport + DatabaseReport healthcheck.DatabaseReport + WorkspaceProxyReport healthcheck.WorkspaceProxyReport } func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report { @@ -34,6 +35,10 @@ func (c *testChecker) Database(context.Context, *healthcheck.DatabaseReportOptio return c.DatabaseReport } +func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProxyReportOptions) healthcheck.WorkspaceProxyReport { + return c.WorkspaceProxyReport +} + func TestHealthcheck(t *testing.T) { t.Parallel() @@ -62,6 +67,10 @@ func TestHealthcheck(t *testing.T) { Healthy: true, Severity: health.SeverityOK, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: true, severity: health.SeverityOK, @@ -85,6 +94,10 @@ func TestHealthcheck(t *testing.T) { Healthy: true, Severity: health.SeverityOK, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: false, severity: health.SeverityError, @@ -109,6 +122,10 @@ func TestHealthcheck(t *testing.T) { Healthy: true, Severity: health.SeverityOK, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: true, severity: health.SeverityWarning, @@ -132,6 +149,10 @@ func TestHealthcheck(t *testing.T) { Healthy: true, Severity: health.SeverityOK, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: false, severity: health.SeverityWarning, @@ -155,6 +176,10 @@ func TestHealthcheck(t *testing.T) { Healthy: true, Severity: health.SeverityOK, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: false, severity: health.SeverityError, @@ -178,12 +203,44 @@ func TestHealthcheck(t *testing.T) { Healthy: false, Severity: health.SeverityError, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: true, + Severity: health.SeverityOK, + }, }, healthy: false, severity: health.SeverityError, failingSections: []string{healthcheck.SectionDatabase}, }, { - name: "AllFail", + name: "ProxyFail", + checker: &testChecker{ + DERPReport: derphealth.Report{ + Healthy: true, + Severity: health.SeverityOK, + }, + AccessURLReport: healthcheck.AccessURLReport{ + Healthy: true, + Severity: health.SeverityOK, + }, + WebsocketReport: healthcheck.WebsocketReport{ + Healthy: true, + Severity: health.SeverityOK, + }, + DatabaseReport: healthcheck.DatabaseReport{ + Healthy: true, + Severity: health.SeverityOK, + }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: false, + Severity: health.SeverityError, + }, + }, + severity: health.SeverityError, + healthy: false, + failingSections: []string{healthcheck.SectionWorkspaceProxy}, + }, { + name: "AllFail", + healthy: false, checker: &testChecker{ DERPReport: derphealth.Report{ Healthy: false, @@ -201,14 +258,18 @@ func TestHealthcheck(t *testing.T) { Healthy: false, Severity: health.SeverityError, }, + WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{ + Healthy: false, + Severity: health.SeverityError, + }, }, - healthy: false, severity: health.SeverityError, failingSections: []string{ healthcheck.SectionDERP, healthcheck.SectionAccessURL, healthcheck.SectionWebsocket, healthcheck.SectionDatabase, + healthcheck.SectionWorkspaceProxy, }, }} { c := c @@ -228,6 +289,8 @@ func TestHealthcheck(t *testing.T) { assert.Equal(t, c.checker.AccessURLReport.Healthy, report.AccessURL.Healthy) assert.Equal(t, c.checker.AccessURLReport.Severity, report.AccessURL.Severity) assert.Equal(t, c.checker.WebsocketReport.Healthy, report.Websocket.Healthy) + assert.Equal(t, c.checker.WorkspaceProxyReport.Healthy, report.WorkspaceProxy.Healthy) + assert.Equal(t, c.checker.WorkspaceProxyReport.Warnings, report.WorkspaceProxy.Warnings) assert.Equal(t, c.checker.WebsocketReport.Severity, report.Websocket.Severity) assert.Equal(t, c.checker.DatabaseReport.Healthy, report.Database.Healthy) assert.Equal(t, c.checker.DatabaseReport.Severity, report.Database.Severity) diff --git a/coderd/healthcheck/workspaceproxy.go b/coderd/healthcheck/workspaceproxy.go new file mode 100644 index 0000000000000..3cfaf6e8c0158 --- /dev/null +++ b/coderd/healthcheck/workspaceproxy.go @@ -0,0 +1,146 @@ +package healthcheck + +import ( + "context" + "errors" + "sort" + + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/buildinfo" + "github.com/coder/coder/v2/coderd/healthcheck/health" + "github.com/coder/coder/v2/coderd/util/ptr" + "github.com/coder/coder/v2/codersdk" +) + +type WorkspaceProxyReportOptions struct { + // CurrentVersion is the current server version. + // We pass this in to make it easier to test. + CurrentVersion string + WorkspaceProxiesFetchUpdater WorkspaceProxiesFetchUpdater +} + +// @typescript-generate WorkspaceProxyReport +type WorkspaceProxyReport struct { + Healthy bool `json:"healthy"` + Severity health.Severity `json:"severity"` + Warnings []string `json:"warnings"` + Error *string `json:"error"` + + WorkspaceProxies codersdk.RegionsResponse[codersdk.WorkspaceProxy] `json:"workspace_proxies"` +} + +type WorkspaceProxiesFetchUpdater interface { + Fetch(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) + Update(context.Context) error +} + +// AGPLWorkspaceProxiesFetchUpdater implements WorkspaceProxiesFetchUpdater +// to the extent required by AGPL code. Which isn't that much. +type AGPLWorkspaceProxiesFetchUpdater struct{} + +func (*AGPLWorkspaceProxiesFetchUpdater) Fetch(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{}, nil +} + +func (*AGPLWorkspaceProxiesFetchUpdater) Update(context.Context) error { + return nil +} + +func (r *WorkspaceProxyReport) Run(ctx context.Context, opts *WorkspaceProxyReportOptions) { + r.Healthy = true + r.Severity = health.SeverityOK + r.Warnings = []string{} + + if opts.WorkspaceProxiesFetchUpdater == nil { + opts.WorkspaceProxiesFetchUpdater = &AGPLWorkspaceProxiesFetchUpdater{} + } + + // If this fails, just mark it as a warning. It is still updated in the background. + if err := opts.WorkspaceProxiesFetchUpdater.Update(ctx); err != nil { + r.Severity = health.SeverityWarning + r.Warnings = append(r.Warnings, xerrors.Errorf("update proxy health: %w", err).Error()) + return + } + + proxies, err := opts.WorkspaceProxiesFetchUpdater.Fetch(ctx) + if err != nil { + r.Healthy = false + r.Severity = health.SeverityError + r.Error = ptr.Ref(err.Error()) + return + } + + r.WorkspaceProxies = proxies + // Stable sort based on create timestamp. + sort.Slice(r.WorkspaceProxies.Regions, func(i int, j int) bool { + return r.WorkspaceProxies.Regions[i].CreatedAt.Before(r.WorkspaceProxies.Regions[j].CreatedAt) + }) + + var total, healthy int + for _, proxy := range r.WorkspaceProxies.Regions { + total++ + if proxy.Healthy { + healthy++ + } + + if len(proxy.Status.Report.Errors) > 0 { + for _, err := range proxy.Status.Report.Errors { + r.appendError(xerrors.New(err)) + } + } + } + + r.Severity = calculateSeverity(total, healthy) + r.Healthy = r.Severity.Value() < health.SeverityError.Value() + + // Versions _must_ match. Perform this check last. This will clobber any other severity. + for _, proxy := range r.WorkspaceProxies.Regions { + if vErr := checkVersion(proxy, opts.CurrentVersion); vErr != nil { + r.Healthy = false + r.Severity = health.SeverityError + r.appendError(vErr) + } + } +} + +// appendError appends errs onto r.Error. +// We only have one error, so multiple errors need to be squashed in there. +func (r *WorkspaceProxyReport) appendError(errs ...error) { + if len(errs) == 0 { + return + } + if r.Error != nil { + errs = append([]error{xerrors.New(*r.Error)}, errs...) + } + r.Error = ptr.Ref(errors.Join(errs...).Error()) +} + +func checkVersion(proxy codersdk.WorkspaceProxy, currentVersion string) error { + if proxy.Version == "" { + return nil // may have not connected yet, this is OK + } + if buildinfo.VersionsMatch(proxy.Version, currentVersion) { + return nil + } + + return xerrors.Errorf("proxy %q version %q does not match primary server version %q", + proxy.Name, + proxy.Version, + currentVersion, + ) +} + +// calculateSeverity returns: +// health.SeverityError if all proxies are unhealthy, +// health.SeverityOK if all proxies are healthy, +// health.SeverityWarning otherwise. +func calculateSeverity(total, healthy int) health.Severity { + if total == 0 || total == healthy { + return health.SeverityOK + } + if total-healthy == total { + return health.SeverityError + } + return health.SeverityWarning +} diff --git a/coderd/healthcheck/workspaceproxy_internal_test.go b/coderd/healthcheck/workspaceproxy_internal_test.go new file mode 100644 index 0000000000000..6b87b8de79eed --- /dev/null +++ b/coderd/healthcheck/workspaceproxy_internal_test.go @@ -0,0 +1,95 @@ +package healthcheck + +import ( + "fmt" + "testing" + + "github.com/coder/coder/v2/coderd/healthcheck/health" + "github.com/coder/coder/v2/coderd/util/ptr" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "golang.org/x/xerrors" +) + +func Test_WorkspaceProxyReport_appendErrors(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + expected string + prevErr string + errs []error + }{ + { + name: "nil", + errs: nil, + }, + { + name: "one error", + expected: assert.AnError.Error(), + errs: []error{assert.AnError}, + }, + { + name: "one error, one prev", + prevErr: "previous error", + expected: "previous error\n" + assert.AnError.Error(), + errs: []error{assert.AnError}, + }, + { + name: "two errors", + expected: assert.AnError.Error() + "\nanother error", + errs: []error{assert.AnError, xerrors.Errorf("another error")}, + }, + { + name: "two errors, one prev", + prevErr: "previous error", + expected: "previous error\n" + assert.AnError.Error() + "\nanother error", + errs: []error{assert.AnError, xerrors.Errorf("another error")}, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + var rpt WorkspaceProxyReport + if tt.prevErr != "" { + rpt.Error = ptr.Ref(tt.prevErr) + } + rpt.appendError(tt.errs...) + if tt.expected == "" { + require.Nil(t, rpt.Error) + } else { + require.NotNil(t, rpt.Error) + require.Equal(t, tt.expected, *rpt.Error) + } + }) + } +} + +func Test_calculateSeverity(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + total int + healthy int + expected health.Severity + }{ + {0, 0, health.SeverityOK}, + {1, 1, health.SeverityOK}, + {1, 0, health.SeverityError}, + {2, 2, health.SeverityOK}, + {2, 1, health.SeverityWarning}, + {2, 0, health.SeverityError}, + } { + tt := tt + name := fmt.Sprintf("%d total, %d healthy -> %s", tt.total, tt.healthy, tt.expected) + t.Run(name, func(t *testing.T) { + t.Parallel() + + actual := calculateSeverity(tt.total, tt.healthy) + assert.Equal(t, tt.expected, actual) + }) + } +} diff --git a/coderd/healthcheck/workspaceproxy_test.go b/coderd/healthcheck/workspaceproxy_test.go new file mode 100644 index 0000000000000..a96c13384fb5b --- /dev/null +++ b/coderd/healthcheck/workspaceproxy_test.go @@ -0,0 +1,238 @@ +package healthcheck_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/coder/coder/v2/coderd/healthcheck" + "github.com/coder/coder/v2/coderd/healthcheck/health" + "github.com/coder/coder/v2/codersdk" +) + +func TestWorkspaceProxies(t *testing.T) { + t.Parallel() + + var ( + newerPatchVersion = "v2.34.6" + currentVersion = "v2.34.5" + olderVersion = "v2.33.0" + ) + + for _, tt := range []struct { + name string + fetchWorkspaceProxies func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) + updateProxyHealth func(context.Context) error + expectedHealthy bool + expectedError string + expectedSeverity health.Severity + }{ + { + name: "NotEnabled", + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/NoProxies", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies(), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/OneHealthy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", true, currentVersion)), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/OneUnhealthy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", false, currentVersion)), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: false, + expectedSeverity: health.SeverityError, + }, + { + name: "Enabled/OneUnreachable", + fetchWorkspaceProxies: func(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{ + Regions: []codersdk.WorkspaceProxy{ + { + Region: codersdk.Region{ + Name: "gone", + Healthy: false, + }, + Version: currentVersion, + Status: codersdk.WorkspaceProxyStatus{ + Status: codersdk.ProxyUnreachable, + Report: codersdk.ProxyHealthReport{ + Errors: []string{ + "request to proxy failed: Get \"http://127.0.0.1:3001/healthz-report\": dial tcp 127.0.0.1:3001: connect: connection refused", + }, + }, + }, + }, + }, + }, nil + }, + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: false, + expectedSeverity: health.SeverityError, + expectedError: "connect: connection refused", + }, + { + name: "Enabled/AllHealthy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("alpha", true, currentVersion), + fakeWorkspaceProxy("beta", true, currentVersion), + ), + updateProxyHealth: func(ctx context.Context) error { + return nil + }, + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/OneHealthyOneUnhealthy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("alpha", false, currentVersion), + fakeWorkspaceProxy("beta", true, currentVersion), + ), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: true, + expectedSeverity: health.SeverityWarning, + }, + { + name: "Enabled/AllUnhealthy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("alpha", false, currentVersion), + fakeWorkspaceProxy("beta", false, currentVersion), + ), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: false, + expectedSeverity: health.SeverityError, + }, + { + name: "Enabled/OneOutOfDate", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("alpha", true, currentVersion), + fakeWorkspaceProxy("beta", true, olderVersion), + ), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: false, + expectedSeverity: health.SeverityError, + expectedError: `proxy "beta" version "v2.33.0" does not match primary server version "v2.34.5"`, + }, + { + name: "Enabled/OneSlightlyNewerButStillOK", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("alpha", true, currentVersion), + fakeWorkspaceProxy("beta", true, newerPatchVersion), + ), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/NotConnectedYet", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies( + fakeWorkspaceProxy("slowpoke", true, ""), + ), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: true, + expectedSeverity: health.SeverityOK, + }, + { + name: "Enabled/ErrFetchWorkspaceProxy", + fetchWorkspaceProxies: fakeFetchWorkspaceProxiesErr(assert.AnError), + updateProxyHealth: fakeUpdateProxyHealth(nil), + expectedHealthy: false, + expectedSeverity: health.SeverityError, + expectedError: assert.AnError.Error(), + }, + { + name: "Enabled/ErrUpdateProxyHealth", + fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", true, currentVersion)), + updateProxyHealth: fakeUpdateProxyHealth(assert.AnError), + expectedHealthy: true, + expectedSeverity: health.SeverityWarning, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + var rpt healthcheck.WorkspaceProxyReport + var opts healthcheck.WorkspaceProxyReportOptions + opts.CurrentVersion = currentVersion + if tt.fetchWorkspaceProxies != nil && tt.updateProxyHealth != nil { + opts.WorkspaceProxiesFetchUpdater = &fakeWorkspaceProxyFetchUpdater{ + fetchFunc: tt.fetchWorkspaceProxies, + updateFunc: tt.updateProxyHealth, + } + } else { + opts.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{} + } + + rpt.Run(context.Background(), &opts) + + assert.Equal(t, tt.expectedHealthy, rpt.Healthy) + assert.Equal(t, tt.expectedSeverity, rpt.Severity) + if tt.expectedError != "" { + assert.NotNil(t, rpt.Error) + assert.Contains(t, *rpt.Error, tt.expectedError) + } else { + if !assert.Nil(t, rpt.Error) { + assert.Empty(t, *rpt.Error) + } + } + }) + } +} + +// yet another implementation of the thing +type fakeWorkspaceProxyFetchUpdater struct { + fetchFunc func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) + updateFunc func(context.Context) error +} + +func (u *fakeWorkspaceProxyFetchUpdater) Fetch(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return u.fetchFunc(ctx) +} + +func (u *fakeWorkspaceProxyFetchUpdater) Update(ctx context.Context) error { + return u.updateFunc(ctx) +} + +func fakeWorkspaceProxy(name string, healthy bool, version string) codersdk.WorkspaceProxy { + return codersdk.WorkspaceProxy{ + Region: codersdk.Region{ + Name: name, + Healthy: healthy, + }, + Version: version, + } +} + +func fakeFetchWorkspaceProxies(ps ...codersdk.WorkspaceProxy) func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{ + Regions: ps, + }, nil + } +} + +func fakeFetchWorkspaceProxiesErr(err error) func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{ + Regions: []codersdk.WorkspaceProxy{}, + }, err + } +} + +func fakeUpdateProxyHealth(err error) func(context.Context) error { + return func(context.Context) error { + return err + } +} diff --git a/docs/api/debug.md b/docs/api/debug.md index c2a26321716d3..60576c1c0ac62 100644 --- a/docs/api/debug.md +++ b/docs/api/debug.md @@ -253,6 +253,39 @@ curl -X GET http://coder-server:8080/api/v2/debug/health \ "healthy": true, "severity": "ok", "warnings": ["string"] + }, + "workspace_proxy": { + "error": "string", + "healthy": true, + "severity": "ok", + "warnings": ["string"], + "workspace_proxies": { + "regions": [ + { + "created_at": "2019-08-24T14:15:22Z", + "deleted": true, + "derp_enabled": true, + "derp_only": true, + "display_name": "string", + "healthy": true, + "icon_url": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "path_app_url": "string", + "status": { + "checked_at": "2019-08-24T14:15:22Z", + "report": { + "errors": ["string"], + "warnings": ["string"] + }, + "status": "ok" + }, + "updated_at": "2019-08-24T14:15:22Z", + "version": "string", + "wildcard_hostname": "string" + } + ] + } } } ``` diff --git a/docs/api/schemas.md b/docs/api/schemas.md index c4dc42883987d..78992ff1a6c2e 100644 --- a/docs/api/schemas.md +++ b/docs/api/schemas.md @@ -7789,23 +7789,57 @@ If the schedule is empty, the user will be updated to use the default schedule.| "healthy": true, "severity": "ok", "warnings": ["string"] + }, + "workspace_proxy": { + "error": "string", + "healthy": true, + "severity": "ok", + "warnings": ["string"], + "workspace_proxies": { + "regions": [ + { + "created_at": "2019-08-24T14:15:22Z", + "deleted": true, + "derp_enabled": true, + "derp_only": true, + "display_name": "string", + "healthy": true, + "icon_url": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "path_app_url": "string", + "status": { + "checked_at": "2019-08-24T14:15:22Z", + "report": { + "errors": ["string"], + "warnings": ["string"] + }, + "status": "ok" + }, + "updated_at": "2019-08-24T14:15:22Z", + "version": "string", + "wildcard_hostname": "string" + } + ] + } } } ``` ### Properties -| Name | Type | Required | Restrictions | Description | -| ------------------ | ---------------------------------------------------------- | -------- | ------------ | ----------------------------------------------------------------------------------- | -| `access_url` | [healthcheck.AccessURLReport](#healthcheckaccessurlreport) | false | | | -| `coder_version` | string | false | | The Coder version of the server that the report was generated on. | -| `database` | [healthcheck.DatabaseReport](#healthcheckdatabasereport) | false | | | -| `derp` | [derphealth.Report](#derphealthreport) | false | | | -| `failing_sections` | array of string | false | | Failing sections is a list of sections that have failed their healthcheck. | -| `healthy` | boolean | false | | Healthy is true if the report returns no errors. Deprecated: use `Severity` instead | -| `severity` | [health.Severity](#healthseverity) | false | | Severity indicates the status of Coder health. | -| `time` | string | false | | Time is the time the report was generated at. | -| `websocket` | [healthcheck.WebsocketReport](#healthcheckwebsocketreport) | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------------ | -------------------------------------------------------------------- | -------- | ------------ | ----------------------------------------------------------------------------------- | +| `access_url` | [healthcheck.AccessURLReport](#healthcheckaccessurlreport) | false | | | +| `coder_version` | string | false | | The Coder version of the server that the report was generated on. | +| `database` | [healthcheck.DatabaseReport](#healthcheckdatabasereport) | false | | | +| `derp` | [derphealth.Report](#derphealthreport) | false | | | +| `failing_sections` | array of string | false | | Failing sections is a list of sections that have failed their healthcheck. | +| `healthy` | boolean | false | | Healthy is true if the report returns no errors. Deprecated: use `Severity` instead | +| `severity` | [health.Severity](#healthseverity) | false | | Severity indicates the status of Coder health. | +| `time` | string | false | | Time is the time the report was generated at. | +| `websocket` | [healthcheck.WebsocketReport](#healthcheckwebsocketreport) | false | | | +| `workspace_proxy` | [healthcheck.WorkspaceProxyReport](#healthcheckworkspaceproxyreport) | false | | | #### Enumerated Values @@ -7847,6 +7881,54 @@ If the schedule is empty, the user will be updated to use the default schedule.| | `severity` | `warning` | | `severity` | `error` | +## healthcheck.WorkspaceProxyReport + +```json +{ + "error": "string", + "healthy": true, + "severity": "ok", + "warnings": ["string"], + "workspace_proxies": { + "regions": [ + { + "created_at": "2019-08-24T14:15:22Z", + "deleted": true, + "derp_enabled": true, + "derp_only": true, + "display_name": "string", + "healthy": true, + "icon_url": "string", + "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08", + "name": "string", + "path_app_url": "string", + "status": { + "checked_at": "2019-08-24T14:15:22Z", + "report": { + "errors": ["string"], + "warnings": ["string"] + }, + "status": "ok" + }, + "updated_at": "2019-08-24T14:15:22Z", + "version": "string", + "wildcard_hostname": "string" + } + ] + } +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +| ------------------- | ---------------------------------------------------------------------------------------------------- | -------- | ------------ | ----------- | +| `error` | string | false | | | +| `healthy` | boolean | false | | | +| `severity` | [health.Severity](#healthseverity) | false | | | +| `warnings` | array of string | false | | | +| `workspace_proxies` | [codersdk.RegionsResponse-codersdk_WorkspaceProxy](#codersdkregionsresponse-codersdk_workspaceproxy) | false | | | + ## netcheck.Report ```json diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 1a34d594ce599..028ae5a6768c1 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -25,6 +25,7 @@ import ( "github.com/coder/coder/v2/coderd" agplaudit "github.com/coder/coder/v2/coderd/audit" agpldbauthz "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/healthcheck" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/rbac" @@ -374,6 +375,13 @@ func New(ctx context.Context, options *Options) (_ *API, err error) { // Use proxy health to return the healthy workspace proxy hostnames. f := api.ProxyHealth.ProxyHosts api.AGPL.WorkspaceProxyHostsFn.Store(&f) + + // Wire this up to healthcheck. + var fetchUpdater healthcheck.WorkspaceProxiesFetchUpdater = &workspaceProxiesFetchUpdater{ + fetchFunc: api.fetchWorkspaceProxies, + updateFunc: api.ProxyHealth.ForceUpdate, + } + api.AGPL.WorkspaceProxiesFetchUpdater.Store(&fetchUpdater) } err = api.PrometheusRegistry.Register(&api.licenseMetricsCollector) @@ -552,8 +560,8 @@ func (api *API) updateEntitlements(ctx context.Context) error { Log: api.Logger.Named("quota_committer"), Database: api.Database, } - ptr := proto.QuotaCommitter(&committer) - api.AGPL.QuotaCommitter.Store(&ptr) + qcPtr := proto.QuotaCommitter(&committer) + api.AGPL.QuotaCommitter.Store(&qcPtr) } else { api.AGPL.QuotaCommitter.Store(nil) } diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index eaef17542c6bd..fb01d4ab9a690 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -960,3 +960,20 @@ func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) cod }, } } + +// workspaceProxiesFetchUpdater implements healthcheck.WorkspaceProxyFetchUpdater +// in an actually useful and meaningful way. +type workspaceProxiesFetchUpdater struct { + fetchFunc func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) + updateFunc func(context.Context) error +} + +func (w *workspaceProxiesFetchUpdater) Fetch(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) { + //nolint:gocritic // Need perms to read all workspace proxies. + authCtx := dbauthz.AsSystemRestricted(ctx) + return w.fetchFunc(authCtx) +} + +func (w *workspaceProxiesFetchUpdater) Update(ctx context.Context) error { + return w.updateFunc(ctx) +} diff --git a/scripts/apitypings/main.go b/scripts/apitypings/main.go index 7db5565bf2bba..6a182d5c0d53b 100644 --- a/scripts/apitypings/main.go +++ b/scripts/apitypings/main.go @@ -867,6 +867,10 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { return TypescriptType{ValueType: "Record"}, nil case "github.com/coder/coder/v2/cli/clibase.URL": return TypescriptType{ValueType: "string"}, nil + // XXX: For some reason, the type generator generates this as `any` + // so explicitly specifying the correct generic TS type. + case "github.com/coder/coder/v2/codersdk.RegionsResponse[github.com/coder/coder/v2/codersdk.WorkspaceProxy]": + return TypescriptType{ValueType: "RegionsResponse"}, nil } // Some hard codes are a bit trickier. diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index ccf06da3f8b47..fe2ad416f7315 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2115,6 +2115,7 @@ export interface HealthcheckReport { readonly access_url: HealthcheckAccessURLReport; readonly websocket: HealthcheckWebsocketReport; readonly database: HealthcheckDatabaseReport; + readonly workspace_proxy: HealthcheckWorkspaceProxyReport; readonly coder_version: string; } @@ -2128,6 +2129,15 @@ export interface HealthcheckWebsocketReport { readonly error?: string; } +// From healthcheck/workspaceproxy.go +export interface HealthcheckWorkspaceProxyReport { + readonly healthy: boolean; + readonly severity: HealthSeverity; + readonly warnings: string[]; + readonly error?: string; + readonly workspace_proxies: RegionsResponse; +} + // The code below is generated from cli/clibase. // From clibase/clibase.go diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index b448e08ff1cda..076bdf2167e68 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -2827,6 +2827,14 @@ export const MockHealth: TypesGen.HealthcheckReport = { latency_ms: 92570, threshold_ms: 92570, }, + workspace_proxy: { + healthy: true, + severity: "ok", + warnings: [], + workspace_proxies: { + regions: [], + }, + }, coder_version: "v0.27.1-devel+c575292", }; @@ -2877,4 +2885,37 @@ export const DeploymentHealthUnhealthy: TypesGen.HealthcheckReport = { body: "", code: 0, }, + workspace_proxy: { + healthy: false, + error: "some error", + severity: "error", + warnings: [], + workspace_proxies: { + regions: [ + { + id: "df7e4b2b-2d40-47e5-a021-e5d08b219c77", + name: "unhealthy", + display_name: "unhealthy", + icon_url: "/emojis/1f5fa.png", + healthy: false, + path_app_url: "http://127.0.0.1:3001", + wildcard_hostname: "", + derp_enabled: true, + derp_only: false, + status: { + status: "unreachable", + report: { + errors: ["some error"], + warnings: [], + }, + checked_at: "2023-11-24T12:14:05.743303497Z", + }, + created_at: "2023-11-23T15:37:25.513213Z", + updated_at: "2023-11-23T18:09:19.734747Z", + deleted: false, + version: "v2.4.0-devel+89bae7eff", + }, + ], + }, + }, };