From c4c1a89bc895bcdab380d87ee1b5837ef75aa8f8 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 13:16:56 +0100 Subject: [PATCH 01/12] feat(coderd/healthcheck/derphealth): add some STUN-specific checks --- coderd/healthcheck/derphealth/derp.go | 18 +++- coderd/healthcheck/derphealth/derp_test.go | 110 ++++++++++++++++++++- coderd/healthcheck/health/model.go | 2 + 3 files changed, 126 insertions(+), 4 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index e72c527d700c4..82de78b9ccaf2 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -32,6 +32,8 @@ const ( warningNodeUsesWebsocket = `Node uses WebSockets because the "Upgrade: DERP" header may be blocked on the load balancer.` oneNodeUnhealthy = "Region is operational, but performance might be degraded as one node is unhealthy." missingNodeReport = "Missing node health report, probably a developer error." + noSTUN = "No nodes are capable of STUN, direct connections may not be possible." + stunMapVaryDest = "STUN detected variable mapping by destination IP, you may be behind a hard NAT." ) type ReportOptions struct { @@ -107,6 +109,9 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) { ncReport, netcheckErr := nc.GetReport(ctx, opts.DERPMap) r.Netcheck = ncReport r.NetcheckErr = convertError(netcheckErr) + if mapVaryDest, _ := r.Netcheck.MappingVariesByDestIP.Get(); mapVaryDest { + r.Warnings = append(r.Warnings, health.Messagef(health.CodeSTUNMapVaryDest, stunMapVaryDest)) + } wg.Wait() @@ -125,7 +130,10 @@ func (r *RegionReport) Run(ctx context.Context) { r.Warnings = []health.Message{} wg := &sync.WaitGroup{} - var unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. + var ( + unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. + stunCapableNodes int + ) wg.Add(len(r.Region.Nodes)) for _, node := range r.Region.Nodes { @@ -155,6 +163,9 @@ func (r *RegionReport) Run(ctx context.Context) { if nodeReport.Severity != health.SeverityOK { unhealthyNodes++ } + if nodeReport.STUN.Enabled && nodeReport.STUN.CanSTUN { + stunCapableNodes++ + } r.Warnings = append(r.Warnings, nodeReport.Warnings...) r.mu.Unlock() @@ -174,6 +185,11 @@ func (r *RegionReport) Run(ctx context.Context) { return } + if stunCapableNodes == 0 { + r.Severity = health.SeverityWarning + r.Warnings = append(r.Warnings, health.Messagef(health.CodeSTUNNoNodes, noSTUN)) + } + if len(r.Region.Nodes) == 1 { r.Healthy = r.NodeReports[0].Severity != health.SeverityError r.Severity = r.NodeReports[0].Severity diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index a7011f05e58ac..35901d22e118c 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -129,8 +129,67 @@ func TestDERP(t *testing.T) { assert.True(t, report.Healthy) assert.Equal(t, health.SeverityWarning, report.Severity) assert.True(t, report.Dismissed) - if assert.NotEmpty(t, report.Warnings) { - assert.Contains(t, report.Warnings[0].Code, health.CodeDERPOneNodeUnhealthy) + if assert.Len(t, report.Warnings, 2) { + assert.Contains(t, report.Warnings[0].Code, health.CodeSTUNNoNodes) + assert.Contains(t, report.Warnings[1].Code, health.CodeDERPOneNodeUnhealthy) + } + for _, region := range report.Regions { + assert.True(t, region.Healthy) + assert.True(t, region.NodeReports[0].Healthy) + assert.Empty(t, region.NodeReports[0].Warnings) + assert.NotNil(t, region.NodeReports[0].Warnings) + assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity) + assert.False(t, region.NodeReports[1].Healthy) + assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity) + assert.Len(t, region.Warnings, 2) + } + }) + + t.Run("HealthyWithNoSTUN", func(t *testing.T) { + t.Parallel() + + healthyDerpSrv := derp.NewServer(key.NewNode(), func(format string, args ...any) { t.Logf(format, args...) }) + defer healthyDerpSrv.Close() + healthySrv := httptest.NewServer(derphttp.Handler(healthyDerpSrv)) + defer healthySrv.Close() + + var ( + ctx = context.Background() + report = derphealth.Report{} + derpURL, _ = url.Parse(healthySrv.URL) + opts = &derphealth.ReportOptions{ + DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 999, + HostName: derpURL.Host, + IPv4: derpURL.Host, + STUNPort: -1, + InsecureForTests: true, + ForceHTTP: true, + }, { + Name: "badstun", + RegionID: 1000, + HostName: derpURL.Host, + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, + }}, + } + ) + + report.Run(ctx, opts) + + assert.True(t, report.Healthy) + assert.Equal(t, health.SeverityWarning, report.Severity) + if assert.Len(t, report.Warnings, 2) { + assert.Contains(t, report.Warnings[0].Code, health.CodeSTUNNoNodes) } for _, region := range report.Regions { assert.True(t, region.Healthy) @@ -140,7 +199,7 @@ func TestDERP(t *testing.T) { assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity) assert.False(t, region.NodeReports[1].Healthy) assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity) - assert.Len(t, region.Warnings, 1) + assert.Len(t, region.Warnings, 2) } }) @@ -291,8 +350,10 @@ func TestDERP(t *testing.T) { report.Run(ctx, opts) assert.True(t, report.Healthy) + assert.Equal(t, health.SeverityOK, report.Severity) for _, region := range report.Regions { assert.True(t, region.Healthy) + assert.Equal(t, health.SeverityOK, region.Severity) for _, node := range region.NodeReports { assert.True(t, node.Healthy) assert.False(t, node.CanExchangeMessages) @@ -304,6 +365,49 @@ func TestDERP(t *testing.T) { } } }) + + t.Run("STUNOnly/Error", func(t *testing.T) { + t.Parallel() + + var ( + ctx = context.Background() + report = derphealth.Report{} + opts = &derphealth.ReportOptions{ + DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "badstun", + RegionID: 999, + HostName: "badstun.example.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, + }, + }, + } + ) + + report.Run(ctx, opts) + assert.False(t, report.Healthy) + assert.Equal(t, health.SeverityError, report.Severity) + for _, region := range report.Regions { + assert.False(t, region.Healthy) + assert.Equal(t, health.SeverityError, region.Severity) + for _, node := range region.NodeReports { + assert.False(t, node.Healthy) + assert.False(t, node.CanExchangeMessages) + assert.Empty(t, node.ClientLogs) + assert.True(t, node.STUN.Enabled) + assert.False(t, node.STUN.CanSTUN) + assert.NotNil(t, node.STUN.Error) + } + } + }) } func tsDERPMap(ctx context.Context, t testing.TB) *tailcfg.DERPMap { diff --git a/coderd/healthcheck/health/model.go b/coderd/healthcheck/health/model.go index 9f853864fdf9f..33b5e9711b87b 100644 --- a/coderd/healthcheck/health/model.go +++ b/coderd/healthcheck/health/model.go @@ -36,6 +36,8 @@ const ( CodeDERPNodeUsesWebsocket Code = `EDERP01` CodeDERPOneNodeUnhealthy Code = `EDERP02` + CodeSTUNNoNodes = `ESTUN01` + CodeSTUNMapVaryDest = `ESTUN02` CodeProvisionerDaemonsNoProvisionerDaemons Code = `EPD01` CodeProvisionerDaemonVersionMismatch Code = `EPD02` From 33b6fb55269b929330b237d9a4f4953eaae069e6 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 13:58:13 +0100 Subject: [PATCH 02/12] fixup! feat(coderd/healthcheck/derphealth): add some STUN-specific checks --- coderd/healthcheck/derphealth/derp.go | 25 +++++--- coderd/healthcheck/derphealth/derp_test.go | 71 +++++++++++++++++++--- 2 files changed, 79 insertions(+), 17 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 82de78b9ccaf2..6347ea8299811 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -115,6 +115,20 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) { wg.Wait() + // Count the number of STUN-capable nodes. + var stunCapableNodes int + for _, region := range r.Regions { + for _, node := range region.NodeReports { + if node.STUN.CanSTUN { + stunCapableNodes++ + } + } + } + if stunCapableNodes == 0 { + r.Severity = health.SeverityWarning + r.Warnings = append(r.Warnings, health.Messagef(health.CodeSTUNNoNodes, noSTUN)) + } + // Review region reports and select the highest severity. for _, regionReport := range r.Regions { if regionReport.Severity.Value() > r.Severity.Value() { @@ -131,8 +145,7 @@ func (r *RegionReport) Run(ctx context.Context) { wg := &sync.WaitGroup{} var ( - unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. - stunCapableNodes int + unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. ) wg.Add(len(r.Region.Nodes)) @@ -163,9 +176,6 @@ func (r *RegionReport) Run(ctx context.Context) { if nodeReport.Severity != health.SeverityOK { unhealthyNodes++ } - if nodeReport.STUN.Enabled && nodeReport.STUN.CanSTUN { - stunCapableNodes++ - } r.Warnings = append(r.Warnings, nodeReport.Warnings...) r.mu.Unlock() @@ -185,11 +195,6 @@ func (r *RegionReport) Run(ctx context.Context) { return } - if stunCapableNodes == 0 { - r.Severity = health.SeverityWarning - r.Warnings = append(r.Warnings, health.Messagef(health.CodeSTUNNoNodes, noSTUN)) - } - if len(r.Region.Nodes) == 1 { r.Healthy = r.NodeReports[0].Severity != health.SeverityError r.Severity = r.NodeReports[0].Severity diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index 35901d22e118c..f2fb2840ebbc2 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -130,8 +130,8 @@ func TestDERP(t *testing.T) { assert.Equal(t, health.SeverityWarning, report.Severity) assert.True(t, report.Dismissed) if assert.Len(t, report.Warnings, 2) { - assert.Contains(t, report.Warnings[0].Code, health.CodeSTUNNoNodes) - assert.Contains(t, report.Warnings[1].Code, health.CodeDERPOneNodeUnhealthy) + assert.Contains(t, report.Warnings[0].Code, health.CodeDERPOneNodeUnhealthy) + assert.Contains(t, report.Warnings[1].Code, health.CodeSTUNNoNodes) } for _, region := range report.Regions { assert.True(t, region.Healthy) @@ -141,7 +141,7 @@ func TestDERP(t *testing.T) { assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity) assert.False(t, region.NodeReports[1].Healthy) assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity) - assert.Len(t, region.Warnings, 2) + assert.Len(t, region.Warnings, 1) } }) @@ -172,7 +172,7 @@ func TestDERP(t *testing.T) { ForceHTTP: true, }, { Name: "badstun", - RegionID: 1000, + RegionID: 999, HostName: derpURL.Host, STUNPort: 19302, STUNOnly: true, @@ -189,7 +189,8 @@ func TestDERP(t *testing.T) { assert.True(t, report.Healthy) assert.Equal(t, health.SeverityWarning, report.Severity) if assert.Len(t, report.Warnings, 2) { - assert.Contains(t, report.Warnings[0].Code, health.CodeSTUNNoNodes) + assert.EqualValues(t, report.Warnings[1].Code, health.CodeSTUNNoNodes) + assert.EqualValues(t, report.Warnings[0].Code, health.CodeDERPOneNodeUnhealthy) } for _, region := range report.Regions { assert.True(t, region.Healthy) @@ -199,7 +200,7 @@ func TestDERP(t *testing.T) { assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity) assert.False(t, region.NodeReports[1].Healthy) assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity) - assert.Len(t, region.Warnings, 2) + assert.Len(t, region.Warnings, 1) } }) @@ -366,7 +367,63 @@ func TestDERP(t *testing.T) { } }) - t.Run("STUNOnly/Error", func(t *testing.T) { + t.Run("STUNOnly/OneBadOneGood", func(t *testing.T) { + t.Parallel() + + var ( + ctx = context.Background() + report = derphealth.Report{} + opts = &derphealth.ReportOptions{ + DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "badstun", + RegionID: 999, + HostName: "badstun.example.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }, { + Name: "goodstun", + RegionID: 999, + HostName: "stun.l.google.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, + }, + }, + } + ) + + report.Run(ctx, opts) + assert.True(t, report.Healthy) + assert.Equal(t, health.SeverityWarning, report.Severity) + if assert.Len(t, report.Warnings, 1) { + assert.Equal(t, health.CodeDERPOneNodeUnhealthy, report.Warnings[0].Code) + } + for _, region := range report.Regions { + assert.True(t, region.Healthy) + assert.Equal(t, health.SeverityWarning, region.Severity) + // badstun + assert.False(t, region.NodeReports[0].Healthy) + assert.True(t, region.NodeReports[0].STUN.Enabled) + assert.False(t, region.NodeReports[0].STUN.CanSTUN) + assert.NotNil(t, region.NodeReports[0].STUN.Error) + // goodstun + assert.True(t, region.NodeReports[1].Healthy) + assert.True(t, region.NodeReports[1].STUN.Enabled) + assert.True(t, region.NodeReports[1].STUN.CanSTUN) + assert.Nil(t, region.NodeReports[1].STUN.Error) + } + }) + + t.Run("STUNOnly/NoStun", func(t *testing.T) { t.Parallel() var ( From ca366234f86f986dbb8e71753026146b0ff90a31 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 16:41:33 +0100 Subject: [PATCH 03/12] fmt --- coderd/healthcheck/derphealth/derp.go | 4 +- coderd/healthcheck/derphealth/derp_test.go | 74 +++++++++++----------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 6347ea8299811..d6c1ac9cb5132 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -144,9 +144,7 @@ func (r *RegionReport) Run(ctx context.Context) { r.Warnings = []health.Message{} wg := &sync.WaitGroup{} - var ( - unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. - ) + var unhealthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. wg.Add(len(r.Region.Nodes)) for _, node := range r.Region.Nodes { diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index f2fb2840ebbc2..b68f63955c537 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -374,30 +374,31 @@ func TestDERP(t *testing.T) { ctx = context.Background() report = derphealth.Report{} opts = &derphealth.ReportOptions{ - DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ - 1: { - EmbeddedRelay: true, - RegionID: 999, - Nodes: []*tailcfg.DERPNode{{ - Name: "badstun", - RegionID: 999, - HostName: "badstun.example.com", - STUNPort: 19302, - STUNOnly: true, - InsecureForTests: true, - ForceHTTP: true, - }, { - Name: "goodstun", - RegionID: 999, - HostName: "stun.l.google.com", - STUNPort: 19302, - STUNOnly: true, - InsecureForTests: true, - ForceHTTP: true, - }}, + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "badstun", + RegionID: 999, + HostName: "badstun.example.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }, { + Name: "goodstun", + RegionID: 999, + HostName: "stun.l.google.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, }, }, - }, } ) @@ -430,22 +431,23 @@ func TestDERP(t *testing.T) { ctx = context.Background() report = derphealth.Report{} opts = &derphealth.ReportOptions{ - DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ - 1: { - EmbeddedRelay: true, - RegionID: 999, - Nodes: []*tailcfg.DERPNode{{ - Name: "badstun", - RegionID: 999, - HostName: "badstun.example.com", - STUNPort: 19302, - STUNOnly: true, - InsecureForTests: true, - ForceHTTP: true, - }}, + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "badstun", + RegionID: 999, + HostName: "badstun.example.com", + STUNPort: 19302, + STUNOnly: true, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, }, }, - }, } ) From 38c6470fbb2746c16f0e5d166672be87a7f8b0b0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 16:42:15 +0100 Subject: [PATCH 04/12] feat(support): add messages from healthcheck to support bundle output --- cli/support.go | 13 +++++++-- support/support.go | 59 +++++++++++++++++++++++++++++++++++++++++ support/support_test.go | 41 ++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 2 deletions(-) diff --git a/cli/support.go b/cli/support.go index f2f962a358f1a..8f7ff67cbba30 100644 --- a/cli/support.go +++ b/cli/support.go @@ -101,7 +101,7 @@ func (r *RootCmd) supportBundle() *serpent.Command { // Check if we're running inside a workspace if val, found := os.LookupEnv("CODER"); found && val == "true" { - _, _ = fmt.Fprintln(inv.Stderr, "Running inside Coder workspace; this can affect results!") + cliui.Warn(inv.Stderr, "Running inside Coder workspace; this can affect results!") cliLog.Debug(inv.Context(), "running inside coder workspace") } @@ -122,7 +122,7 @@ func (r *RootCmd) supportBundle() *serpent.Command { if len(inv.Args) == 0 { cliLog.Warn(inv.Context(), "no workspace specified") - _, _ = fmt.Fprintln(inv.Stderr, "Warning: no workspace specified. This will result in incomplete information.") + cliui.Warn(inv.Stderr, "No workspace specified. This will result in incomplete information.") } else { ws, err := namedWorkspace(inv.Context(), client, inv.Args[0]) if err != nil { @@ -184,6 +184,14 @@ func (r *RootCmd) supportBundle() *serpent.Command { _ = os.Remove(outputPath) // best effort return xerrors.Errorf("create support bundle: %w", err) } + msgs := support.Summarize(bun) + if len(msgs) != 0 { + cliui.Warn(inv.Stderr, "Potential issues detected:\n", msgs...) + cliLog.Warn(inv.Context(), "auto-detected issues") + for _, msg := range msgs { + cliLog.Warn(inv.Context(), msg) + } + } bun.CLILogs = cliLogBuf.Bytes() if err := writeBundle(bun, zwr); err != nil { @@ -191,6 +199,7 @@ func (r *RootCmd) supportBundle() *serpent.Command { return xerrors.Errorf("write support bundle to %s: %w", outputPath, err) } _, _ = fmt.Fprintln(inv.Stderr, "Wrote support bundle to "+outputPath) + return nil }, } diff --git a/support/support.go b/support/support.go index 341e01e1862bb..e5db0ccba16c2 100644 --- a/support/support.go +++ b/support/support.go @@ -520,3 +520,62 @@ func sanitizeEnv(kvs map[string]string) { } } } + +func Summarize(b *Bundle) (msgs []string) { + if b == nil { + return []string{} + } + if b.Network.Netcheck == nil { + msgs = append(msgs, "Netcheck missing from bundle!") + } else { + if b.Network.Netcheck.Error != nil { + msgs = append(msgs, "Client netcheck: "+*b.Network.Netcheck.Error) + } + for _, warn := range b.Network.Netcheck.Warnings { + msgs = append(msgs, "Client netcheck: "+warn.String()) + } + } + + if b.Deployment.HealthReport == nil { + msgs = append(msgs, "Deployment health check missing from bundle!") + } else { + if b.Deployment.HealthReport.AccessURL.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.AccessURL.Error) + } + for _, warn := range b.Deployment.HealthReport.AccessURL.Warnings { + msgs = append(msgs, "Deployment health: "+warn.String()) + } + if b.Deployment.HealthReport.DERP.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.DERP.Error) + } + for _, warn := range b.Deployment.HealthReport.DERP.Warnings { + msgs = append(msgs, "Deployment health: "+warn.String()) + } + if b.Deployment.HealthReport.Database.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.Database.Error) + } + for _, warn := range b.Deployment.HealthReport.Database.Warnings { + msgs = append(msgs, "Deployment health: "+warn.String()) + } + if b.Deployment.HealthReport.ProvisionerDaemons.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.ProvisionerDaemons.Error) + } + for _, warn := range *&b.Deployment.HealthReport.ProvisionerDaemons.Warnings { + msgs = append(msgs, "Deployment health: "+warn.String()) + } + if b.Deployment.HealthReport.Websocket.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.Websocket.Error) + } + for _, warn := range *&b.Deployment.HealthReport.Websocket.Warnings { + msgs = append(msgs, "Deployment health: "+warn) + } + if b.Deployment.HealthReport.WorkspaceProxy.Error != nil { + msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.WorkspaceProxy.Error) + } + for _, warn := range *&b.Deployment.HealthReport.WorkspaceProxy.Warnings { + msgs = append(msgs, "Deployment health: "+warn.String()) + } + } + + return msgs +} diff --git a/support/support_test.go b/support/support_test.go index 55eb6a1f23bd9..38267ebae46f8 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -23,6 +23,8 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbfake" "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/healthcheck/derphealth" + "github.com/coder/coder/v2/coderd/healthcheck/health" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/support" @@ -245,3 +247,42 @@ func assertNotNilNotEmpty[T any](t *testing.T, v T, msg string) { assert.NotEmpty(t, v, msg+" but was empty") } } + +func Test_Summarize(t *testing.T) { + for _, tt := range []struct { + name string + in support.Bundle + expected []string + }{ + { + name: "empty", + in: support.Bundle{}, + expected: []string{"Netcheck missing from bundle!"}, + }, + { + name: "network health report", + in: support.Bundle{ + Network: support.Network{ + Netcheck: &derphealth.Report{ + Warnings: []health.Message{ + {Code: "TEST", Message: "test"}, + }, + }, + }, + }, + expected: []string{"TEST: test"}, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + actual := support.Summarize(&tt.in) + if len(tt.expected) == 0 { + assert.Empty(t, actual) + } else { + for _, exp := range tt.expected { + assert.Contains(t, actual, exp) + } + } + }) + } +} From 50fa1517d204f1c93385b6733e0c0c04c1aa13e9 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 17:51:30 +0100 Subject: [PATCH 05/12] lint --- support/support.go | 6 +++--- support/support_test.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/support/support.go b/support/support.go index e5db0ccba16c2..56910a04b444f 100644 --- a/support/support.go +++ b/support/support.go @@ -560,19 +560,19 @@ func Summarize(b *Bundle) (msgs []string) { if b.Deployment.HealthReport.ProvisionerDaemons.Error != nil { msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.ProvisionerDaemons.Error) } - for _, warn := range *&b.Deployment.HealthReport.ProvisionerDaemons.Warnings { + for _, warn := range b.Deployment.HealthReport.ProvisionerDaemons.Warnings { msgs = append(msgs, "Deployment health: "+warn.String()) } if b.Deployment.HealthReport.Websocket.Error != nil { msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.Websocket.Error) } - for _, warn := range *&b.Deployment.HealthReport.Websocket.Warnings { + for _, warn := range b.Deployment.HealthReport.Websocket.Warnings { msgs = append(msgs, "Deployment health: "+warn) } if b.Deployment.HealthReport.WorkspaceProxy.Error != nil { msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.WorkspaceProxy.Error) } - for _, warn := range *&b.Deployment.HealthReport.WorkspaceProxy.Warnings { + for _, warn := range b.Deployment.HealthReport.WorkspaceProxy.Warnings { msgs = append(msgs, "Deployment health: "+warn.String()) } } diff --git a/support/support_test.go b/support/support_test.go index 38267ebae46f8..7b0bbf69754a5 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -249,6 +249,7 @@ func assertNotNilNotEmpty[T any](t *testing.T, v T, msg string) { } func Test_Summarize(t *testing.T) { + t.Parallel() for _, tt := range []struct { name string in support.Bundle @@ -275,6 +276,7 @@ func Test_Summarize(t *testing.T) { } { tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() actual := support.Summarize(&tt.in) if len(tt.expected) == 0 { assert.Empty(t, actual) From 6f04178acec2785945e43e02728aee7ae5328a7b Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 19:49:56 +0100 Subject: [PATCH 06/12] fix tests --- support/support_test.go | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/support/support_test.go b/support/support_test.go index 7b0bbf69754a5..6189c3695d01e 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -27,6 +27,7 @@ import ( "github.com/coder/coder/v2/coderd/healthcheck/health" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/codersdk/healthsdk" "github.com/coder/coder/v2/support" "github.com/coder/coder/v2/testutil" ) @@ -261,7 +262,7 @@ func Test_Summarize(t *testing.T) { expected: []string{"Netcheck missing from bundle!"}, }, { - name: "network health report", + name: "warnings", in: support.Bundle{ Network: support.Network{ Netcheck: &derphealth.Report{ @@ -270,8 +271,41 @@ func Test_Summarize(t *testing.T) { }, }, }, + Deployment: support.Deployment{ + HealthReport: &healthsdk.HealthcheckReport{ + AccessURL: healthsdk.AccessURLReport{ + Warnings: []health.Message{ + {Code: "TEST", Message: "test"}, + }, + }, + }, + }, + }, + expected: []string{ + "Client netcheck: TEST: test", + "Deployment health: TEST: test", + }, + }, + { + name: "errors", + in: support.Bundle{ + Network: support.Network{ + Netcheck: &derphealth.Report{ + Error: ptr.Ref("yikes"), + }, + }, + Deployment: support.Deployment{ + HealthReport: &healthsdk.HealthcheckReport{ + AccessURL: healthsdk.AccessURLReport{ + Error: ptr.Ref("zoinks"), + }, + }, + }, + }, + expected: []string{ + "Client netcheck: yikes", + "Deployment health: zoinks", }, - expected: []string{"TEST: test"}, }, } { tt := tt From 56f81fc6b5b5c3774f72b8953c22a01836a4dbfe Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 12 Apr 2024 20:18:35 +0100 Subject: [PATCH 07/12] document STUN warnings --- docs/admin/healthcheck.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/admin/healthcheck.md b/docs/admin/healthcheck.md index 8712dae2a5aea..1dd05431302d1 100644 --- a/docs/admin/healthcheck.md +++ b/docs/admin/healthcheck.md @@ -170,6 +170,31 @@ curl -v "https://coder.company.com/derp" # DERP requires connection upgrade ``` +### ESTUN01 + +_No STUN nodes available._ + +**Problem:** This is shown if no STUN-capable nodes are available. Coder will +use STUN to establish [direct connections](../networking/stun.md). Without at +least one working STUN server, direct connections may not be possible. + +**Solution:** Ensure that the +[configured STUN severs](../cli/server.md#derp-server-stun-addresses) are +reachable from Coder and that UDP traffic can be sent/received on the configured +port. + +### ESTUN02 + +_Mapping varies based on destination IP. You may be behind a hard NAT._ + +**Problem:** This is a warning shown when multiple attempts to determine our +public IP address/port via STUN resulted in different `ip:port` combinations. +This is a sign that you are behind a "hard NAT", and may result in difficulty +establishing direct connections. However, it does not mean that direct +connections are impossible. + +**Solution:** Engage with your network administrator. + ## Websocket Coder makes heavy use of [WebSockets](https://datatracker.ietf.org/doc/rfc6455/) From f3f5284a0138aa07f5c513bb96e7ddc9b2ea7cfc Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 15 Apr 2024 09:55:48 +0100 Subject: [PATCH 08/12] rephrase STUN warnings --- coderd/healthcheck/derphealth/derp.go | 4 ++-- docs/admin/healthcheck.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index d6c1ac9cb5132..3b9323fcafac1 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -32,8 +32,8 @@ const ( warningNodeUsesWebsocket = `Node uses WebSockets because the "Upgrade: DERP" header may be blocked on the load balancer.` oneNodeUnhealthy = "Region is operational, but performance might be degraded as one node is unhealthy." missingNodeReport = "Missing node health report, probably a developer error." - noSTUN = "No nodes are capable of STUN, direct connections may not be possible." - stunMapVaryDest = "STUN detected variable mapping by destination IP, you may be behind a hard NAT." + noSTUN = "No STUN servers are available." + stunMapVaryDest = "STUN returned different addresses; you may be behind a hard NAT." ) type ReportOptions struct { diff --git a/docs/admin/healthcheck.md b/docs/admin/healthcheck.md index 1dd05431302d1..1b3918a3bb253 100644 --- a/docs/admin/healthcheck.md +++ b/docs/admin/healthcheck.md @@ -172,11 +172,11 @@ curl -v "https://coder.company.com/derp" ### ESTUN01 -_No STUN nodes available._ +_No STUN servers available._ -**Problem:** This is shown if no STUN-capable nodes are available. Coder will -use STUN to establish [direct connections](../networking/stun.md). Without at -least one working STUN server, direct connections may not be possible. +**Problem:** This is shown if no STUN servers are available. Coder will use STUN +to establish [direct connections](../networking/stun.md). Without at least one +working STUN server, direct connections may not be possible. **Solution:** Ensure that the [configured STUN severs](../cli/server.md#derp-server-stun-addresses) are @@ -185,7 +185,7 @@ port. ### ESTUN02 -_Mapping varies based on destination IP. You may be behind a hard NAT._ +_STUN returned different addresses; you may be behind a hard NAT._ **Problem:** This is a warning shown when multiple attempts to determine our public IP address/port via STUN resulted in different `ip:port` combinations. From e5da288eee10a79b73af9b3ac351b7e593bc3476 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 15 Apr 2024 11:34:52 +0100 Subject: [PATCH 09/12] only warn about STUN if at least one STUN node is configured --- coderd/healthcheck/derphealth/derp.go | 6 +++++- coderd/healthcheck/derphealth/derp_test.go | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 3b9323fcafac1..9e79b5e116d77 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -117,14 +117,18 @@ func (r *Report) Run(ctx context.Context, opts *ReportOptions) { // Count the number of STUN-capable nodes. var stunCapableNodes int + var stunTotalNodes int for _, region := range r.Regions { for _, node := range region.NodeReports { + if node.STUN.Enabled { + stunTotalNodes++ + } if node.STUN.CanSTUN { stunCapableNodes++ } } } - if stunCapableNodes == 0 { + if stunCapableNodes == 0 && stunTotalNodes > 0 { r.Severity = health.SeverityWarning r.Warnings = append(r.Warnings, health.Messagef(health.CodeSTUNNoNodes, noSTUN)) } diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index b68f63955c537..90e5db63c9763 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -129,15 +129,13 @@ func TestDERP(t *testing.T) { assert.True(t, report.Healthy) assert.Equal(t, health.SeverityWarning, report.Severity) assert.True(t, report.Dismissed) - if assert.Len(t, report.Warnings, 2) { + if assert.Len(t, report.Warnings, 1) { assert.Contains(t, report.Warnings[0].Code, health.CodeDERPOneNodeUnhealthy) - assert.Contains(t, report.Warnings[1].Code, health.CodeSTUNNoNodes) } for _, region := range report.Regions { assert.True(t, region.Healthy) assert.True(t, region.NodeReports[0].Healthy) assert.Empty(t, region.NodeReports[0].Warnings) - assert.NotNil(t, region.NodeReports[0].Warnings) assert.Equal(t, health.SeverityOK, region.NodeReports[0].Severity) assert.False(t, region.NodeReports[1].Healthy) assert.Equal(t, health.SeverityError, region.NodeReports[1].Severity) From f6af0e016d750361fc9a544ec200d9e6785060e4 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 15 Apr 2024 12:40:41 +0100 Subject: [PATCH 10/12] remove auto-summarization for now --- cli/support.go | 8 ------- support/support.go | 59 ---------------------------------------------- 2 files changed, 67 deletions(-) diff --git a/cli/support.go b/cli/support.go index 8f7ff67cbba30..5d375c66d88f3 100644 --- a/cli/support.go +++ b/cli/support.go @@ -184,14 +184,6 @@ func (r *RootCmd) supportBundle() *serpent.Command { _ = os.Remove(outputPath) // best effort return xerrors.Errorf("create support bundle: %w", err) } - msgs := support.Summarize(bun) - if len(msgs) != 0 { - cliui.Warn(inv.Stderr, "Potential issues detected:\n", msgs...) - cliLog.Warn(inv.Context(), "auto-detected issues") - for _, msg := range msgs { - cliLog.Warn(inv.Context(), msg) - } - } bun.CLILogs = cliLogBuf.Bytes() if err := writeBundle(bun, zwr); err != nil { diff --git a/support/support.go b/support/support.go index 56910a04b444f..341e01e1862bb 100644 --- a/support/support.go +++ b/support/support.go @@ -520,62 +520,3 @@ func sanitizeEnv(kvs map[string]string) { } } } - -func Summarize(b *Bundle) (msgs []string) { - if b == nil { - return []string{} - } - if b.Network.Netcheck == nil { - msgs = append(msgs, "Netcheck missing from bundle!") - } else { - if b.Network.Netcheck.Error != nil { - msgs = append(msgs, "Client netcheck: "+*b.Network.Netcheck.Error) - } - for _, warn := range b.Network.Netcheck.Warnings { - msgs = append(msgs, "Client netcheck: "+warn.String()) - } - } - - if b.Deployment.HealthReport == nil { - msgs = append(msgs, "Deployment health check missing from bundle!") - } else { - if b.Deployment.HealthReport.AccessURL.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.AccessURL.Error) - } - for _, warn := range b.Deployment.HealthReport.AccessURL.Warnings { - msgs = append(msgs, "Deployment health: "+warn.String()) - } - if b.Deployment.HealthReport.DERP.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.DERP.Error) - } - for _, warn := range b.Deployment.HealthReport.DERP.Warnings { - msgs = append(msgs, "Deployment health: "+warn.String()) - } - if b.Deployment.HealthReport.Database.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.Database.Error) - } - for _, warn := range b.Deployment.HealthReport.Database.Warnings { - msgs = append(msgs, "Deployment health: "+warn.String()) - } - if b.Deployment.HealthReport.ProvisionerDaemons.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.ProvisionerDaemons.Error) - } - for _, warn := range b.Deployment.HealthReport.ProvisionerDaemons.Warnings { - msgs = append(msgs, "Deployment health: "+warn.String()) - } - if b.Deployment.HealthReport.Websocket.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.Websocket.Error) - } - for _, warn := range b.Deployment.HealthReport.Websocket.Warnings { - msgs = append(msgs, "Deployment health: "+warn) - } - if b.Deployment.HealthReport.WorkspaceProxy.Error != nil { - msgs = append(msgs, "Deployment health: "+*b.Deployment.HealthReport.WorkspaceProxy.Error) - } - for _, warn := range b.Deployment.HealthReport.WorkspaceProxy.Warnings { - msgs = append(msgs, "Deployment health: "+warn.String()) - } - } - - return msgs -} From 0c3a1e6818e9ee05dfacbefc3a68caf3b16502fd Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 15 Apr 2024 12:41:15 +0100 Subject: [PATCH 11/12] fixup! remove auto-summarization for now --- support/support_test.go | 74 ----------------------------------------- 1 file changed, 74 deletions(-) diff --git a/support/support_test.go b/support/support_test.go index 6189c3695d01e..b0244e3bb1a51 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -248,77 +248,3 @@ func assertNotNilNotEmpty[T any](t *testing.T, v T, msg string) { assert.NotEmpty(t, v, msg+" but was empty") } } - -func Test_Summarize(t *testing.T) { - t.Parallel() - for _, tt := range []struct { - name string - in support.Bundle - expected []string - }{ - { - name: "empty", - in: support.Bundle{}, - expected: []string{"Netcheck missing from bundle!"}, - }, - { - name: "warnings", - in: support.Bundle{ - Network: support.Network{ - Netcheck: &derphealth.Report{ - Warnings: []health.Message{ - {Code: "TEST", Message: "test"}, - }, - }, - }, - Deployment: support.Deployment{ - HealthReport: &healthsdk.HealthcheckReport{ - AccessURL: healthsdk.AccessURLReport{ - Warnings: []health.Message{ - {Code: "TEST", Message: "test"}, - }, - }, - }, - }, - }, - expected: []string{ - "Client netcheck: TEST: test", - "Deployment health: TEST: test", - }, - }, - { - name: "errors", - in: support.Bundle{ - Network: support.Network{ - Netcheck: &derphealth.Report{ - Error: ptr.Ref("yikes"), - }, - }, - Deployment: support.Deployment{ - HealthReport: &healthsdk.HealthcheckReport{ - AccessURL: healthsdk.AccessURLReport{ - Error: ptr.Ref("zoinks"), - }, - }, - }, - }, - expected: []string{ - "Client netcheck: yikes", - "Deployment health: zoinks", - }, - }, - } { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - actual := support.Summarize(&tt.in) - if len(tt.expected) == 0 { - assert.Empty(t, actual) - } else { - for _, exp := range tt.expected { - assert.Contains(t, actual, exp) - } - } - }) - } -} From a96ab2d5f740de54688688aee35d5d3cf0c53550 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 15 Apr 2024 12:46:49 +0100 Subject: [PATCH 12/12] fixup! fixup! remove auto-summarization for now --- support/support_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/support/support_test.go b/support/support_test.go index b0244e3bb1a51..55eb6a1f23bd9 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -23,11 +23,8 @@ import ( "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbfake" "github.com/coder/coder/v2/coderd/database/dbtime" - "github.com/coder/coder/v2/coderd/healthcheck/derphealth" - "github.com/coder/coder/v2/coderd/healthcheck/health" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" - "github.com/coder/coder/v2/codersdk/healthsdk" "github.com/coder/coder/v2/support" "github.com/coder/coder/v2/testutil" )