From bd056bbe8eb904e100f4951134731dd72bb18659 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 21 Nov 2023 11:15:54 +0100 Subject: [PATCH 1/4] one unhealthy node is fine --- coderd/healthcheck/derphealth/derp.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 83dfcbf4a15a2..0ea124192f4c9 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -146,6 +146,7 @@ func (r *RegionReport) Run(ctx context.Context) { r.NodeReports = []*NodeReport{} wg := &sync.WaitGroup{} + var healthyNodes int // atomic.Int64 is not mandatory as we depend on RegionReport mutex. wg.Add(len(r.Region.Nodes)) for _, node := range r.Region.Nodes { @@ -169,8 +170,8 @@ func (r *RegionReport) Run(ctx context.Context) { r.mu.Lock() r.NodeReports = append(r.NodeReports, &nodeReport) - if !nodeReport.Healthy { - r.Healthy = false + if nodeReport.Healthy { + healthyNodes++ } for _, w := range nodeReport.Warnings { @@ -179,8 +180,13 @@ func (r *RegionReport) Run(ctx context.Context) { r.mu.Unlock() }() } - wg.Wait() + + // Coder allows for 1 unhealthy node in the region, unless there is only 1 node. + if len(r.Region.Nodes) == 1 && healthyNodes == 0 || + healthyNodes+1 >= len(r.Region.Nodes) { + r.Healthy = false + } } func (r *NodeReport) derpURL() *url.URL { From bb21b03a8b9107dd5d1b2d14b2166c9231340bbd Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 21 Nov 2023 11:34:37 +0100 Subject: [PATCH 2/4] derp test --- coderd/healthcheck/derphealth/derp_test.go | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index c9ed3d591557e..69dc5653425c3 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -81,6 +81,57 @@ func TestDERP(t *testing.T) { } }) + t.Run("HealthyWithNodeDegraded", func(t *testing.T) { + t.Parallel() + + healthyDerpSrv := derp.NewServer(key.NewNode(), func(format string, args ...any) { t.Logf(format, args...) }) + defer healthyDerpSrv.Close() + healthySrv := httptest.NewServer(derphttp.Handler(healthyDerpSrv)) + defer healthySrv.Close() + + var ( + ctx = context.Background() + report = derphealth.Report{} + derpURL, _ = url.Parse(healthySrv.URL) + opts = &derphealth.ReportOptions{ + DERPMap: &tailcfg.DERPMap{Regions: map[int]*tailcfg.DERPRegion{ + 1: { + EmbeddedRelay: true, + RegionID: 999, + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 999, + HostName: derpURL.Host, + IPv4: derpURL.Host, + STUNPort: -1, + InsecureForTests: true, + ForceHTTP: true, + }, { + Name: "1b", + RegionID: 999, + HostName: "derp.is.dead.tld", + IPv4: "derp.is.dead.tld", + STUNPort: -1, + InsecureForTests: true, + ForceHTTP: true, + }}, + }, + }}, + } + ) + + report.Run(ctx, opts) + + assert.True(t, report.Healthy) + for _, region := range report.Regions { + assert.True(t, region.Healthy) + assert.True(t, region.NodeReports[0].Healthy) + assert.False(t, region.NodeReports[1].Healthy) + assert.Len(t, region.NodeReports[1].Warnings, 1) + assert.Len(t, region.Warnings, 1) + } + }) + t.Run("Tailscale/Dallas/OK", func(t *testing.T) { t.Parallel() From 478f171068c48941774dd8beae58b4bd2af4db75 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 21 Nov 2023 11:55:34 +0100 Subject: [PATCH 3/4] fix --- coderd/healthcheck/derphealth/derp.go | 6 ++++-- coderd/healthcheck/derphealth/derp_test.go | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 0ea124192f4c9..5441b0764126b 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -26,6 +26,7 @@ import ( const ( warningNodeUsesWebsocket = `Node uses WebSockets because the "Upgrade: DERP" header may be blocked on the load balancer.` + oneNodeUnhealthy = "Region is operational, but performance might be degraded as one node is unhealthy." ) // @typescript-generate Report @@ -183,9 +184,10 @@ func (r *RegionReport) Run(ctx context.Context) { wg.Wait() // Coder allows for 1 unhealthy node in the region, unless there is only 1 node. - if len(r.Region.Nodes) == 1 && healthyNodes == 0 || - healthyNodes+1 >= len(r.Region.Nodes) { + if len(r.Region.Nodes) == 1 && healthyNodes == 0 || healthyNodes+1 < len(r.Region.Nodes) { r.Healthy = false + } else if healthyNodes+1 == len(r.Region.Nodes) { + r.Warnings = append(r.Warnings, oneNodeUnhealthy) } } diff --git a/coderd/healthcheck/derphealth/derp_test.go b/coderd/healthcheck/derphealth/derp_test.go index 69dc5653425c3..a877e03fd2d57 100644 --- a/coderd/healthcheck/derphealth/derp_test.go +++ b/coderd/healthcheck/derphealth/derp_test.go @@ -127,7 +127,6 @@ func TestDERP(t *testing.T) { assert.True(t, region.Healthy) assert.True(t, region.NodeReports[0].Healthy) assert.False(t, region.NodeReports[1].Healthy) - assert.Len(t, region.NodeReports[1].Warnings, 1) assert.Len(t, region.Warnings, 1) } }) From 26278549542f573f2e9daa97adbc33d4a238933f Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Tue, 21 Nov 2023 12:33:57 +0100 Subject: [PATCH 4/4] Update coderd/healthcheck/derphealth/derp.go Co-authored-by: Cian Johnston --- coderd/healthcheck/derphealth/derp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/healthcheck/derphealth/derp.go b/coderd/healthcheck/derphealth/derp.go index 5441b0764126b..8f99ca5994d25 100644 --- a/coderd/healthcheck/derphealth/derp.go +++ b/coderd/healthcheck/derphealth/derp.go @@ -184,9 +184,9 @@ func (r *RegionReport) Run(ctx context.Context) { wg.Wait() // Coder allows for 1 unhealthy node in the region, unless there is only 1 node. - if len(r.Region.Nodes) == 1 && healthyNodes == 0 || healthyNodes+1 < len(r.Region.Nodes) { - r.Healthy = false - } else if healthyNodes+1 == len(r.Region.Nodes) { + if len(r.Region.Nodes) == 1 { + r.Healthy = healthyNodes == len(r.Region.Nodes) + } else if healthyNodes < len(r.Region.Nodes) { r.Warnings = append(r.Warnings, oneNodeUnhealthy) } }