fix: Leaking yamux session after HTTP handler is closed (#329)

kylecarbs · web-flow · commit 65de96c8b4e1 · 2022-02-18T22:06:56.000-06:00
* fix: Leaking yamux session after HTTP handler is closed Closes #317. The httptest server cancels the context after the connection is closed, but if a connection takes a long time to close, the request would never end. This applies a context to the entire listener that cancels on test cleanup. After discussion with @bryphe-coder, reducing the parallel limit on Windows is likely to reduce failures as well. * Switch to windows-2022 to improve decompression * Invalidate cache on matrix OS
diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml
@@ -122,7 +122,7 @@ jobs:
         os:
           - ubuntu-latest
           - macos-latest
-          - windows-latest
+          - windows-2022
     steps:
       - uses: actions/checkout@v2
 
@@ -138,9 +138,9 @@ jobs:
             ~/.cache/go-build
             ~/Library/Caches/go-build
             %LocalAppData%\go-build
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          key: ${{ matrix.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
-            ${{ runner.os }}-go-
+            ${{ matrix.os }}-go-
 
       - run: go install gotest.tools/gotestsum@latest
 
@@ -150,9 +150,13 @@ jobs:
           terraform_wrapper: false
 
       - name: Test with Mock Database
+        shell: bash
+        env:
+          GOCOUNT: ${{ runner.os == 'Windows' && 3 || 5 }}
+          GOMAXPROCS: ${{ runner.os == 'Windows' && 1 || 2 }}
         run: gotestsum --junitfile="gotests.xml" --packages="./..." --
           -covermode=atomic -coverprofile="gotests.coverage"
-          -timeout=3m -count=5 -race -short -parallel=2
+          -timeout=3m -count=$GOCOUNT -race -short -failfast
 
       - name: Upload DataDog Trace
         if: (success() || failure()) && github.actor != 'dependabot[bot]'
@@ -166,10 +170,10 @@ jobs:
         if: runner.os == 'Linux'
         run: DB=true gotestsum --junitfile="gotests.xml" --packages="./..." --
           -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m
-          -count=1 -race -parallel=2
+          -count=1 -race -parallel=2 -failfast
 
       - name: Upload DataDog Trace
-        if: (success() || failure()) && github.actor != 'dependabot[bot]'
+        if: (success() || failure()) && github.actor != 'dependabot[bot]' && runner.os == 'Linux'
         env:
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
           DD_DATABASE: postgresql
diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"io"
+	"net"
 	"net/http/httptest"
 	"net/url"
 	"os"
@@ -59,7 +60,13 @@ func New(t *testing.T) *codersdk.Client {
 		Database: db,
 		Pubsub:   pubsub,
 	})
-	srv := httptest.NewServer(handler)
+	srv := httptest.NewUnstartedServer(handler)
+	srv.Config.BaseContext = func(_ net.Listener) context.Context {
+		ctx, cancelFunc := context.WithCancel(context.Background())
+		t.Cleanup(cancelFunc)
+		return ctx
+	}
+	srv.Start()
 	serverURL, err := url.Parse(srv.URL)
 	require.NoError(t, err)
 	t.Cleanup(srv.Close)
diff --git a/peer/conn.go b/peer/conn.go
@@ -183,12 +183,15 @@ func (c *Conn) init() error {
 		}
 	})
 	c.rtc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) {
-		if c.isClosed() {
-			// Make sure we don't log after Close() has been called.
-			return
-		}
-		c.opts.Logger.Debug(context.Background(), "rtc connection updated",
-			slog.F("state", peerConnectionState))
+		go func() {
+			c.closeMutex.Lock()
+			defer c.closeMutex.Unlock()
+			if c.isClosed() {
+				return
+			}
+			c.opts.Logger.Debug(context.Background(), "rtc connection updated",
+				slog.F("state", peerConnectionState))
+		}()
 
 		switch peerConnectionState {
 		case webrtc.PeerConnectionStateDisconnected:
diff --git a/provisionerd/provisionerd.go b/provisionerd/provisionerd.go
@@ -110,10 +110,13 @@ func (p *provisionerDaemon) connect(ctx context.Context) {
 			if errors.Is(err, context.Canceled) {
 				return
 			}
+			p.closeMutex.Lock()
 			if p.isClosed() {
+				p.closeMutex.Unlock()
 				return
 			}
 			p.opts.Logger.Warn(context.Background(), "failed to dial", slog.Error(err))
+			p.closeMutex.Unlock()
 			continue
 		}
 		p.opts.Logger.Debug(context.Background(), "connected")
diff --git a/pty/pty_other.go b/pty/pty_other.go
@@ -6,6 +6,7 @@ package pty
 import (
 	"io"
 	"os"
+	"sync"
 
 	"github.com/creack/pty"
 )
@@ -23,6 +24,7 @@ func newPty() (PTY, error) {
 }
 
 type otherPty struct {
+	mutex    sync.Mutex
 	pty, tty *os.File
 }
 
@@ -41,13 +43,18 @@ func (p *otherPty) Output() io.ReadWriter {
 }
 
 func (p *otherPty) Resize(cols uint16, rows uint16) error {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
 	return pty.Setsize(p.tty, &pty.Winsize{
 		Rows: rows,
 		Cols: cols,
 	})
 }
 
 func (p *otherPty) Close() error {
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+
 	err := p.pty.Close()
 	if err != nil {
 		return err
diff --git a/pty/start_other.go b/pty/start_other.go
@@ -8,13 +8,17 @@ import (
 	"syscall"
 
 	"github.com/creack/pty"
+	"golang.org/x/xerrors"
 )
 
 func startPty(cmd *exec.Cmd) (PTY, error) {
 	ptty, tty, err := pty.Open()
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("open: %w", err)
 	}
+	defer func() {
+		_ = tty.Close()
+	}()
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setsid:  true,
 		Setctty: true,
@@ -25,7 +29,7 @@ func startPty(cmd *exec.Cmd) (PTY, error) {
 	err = cmd.Start()
 	if err != nil {
 		_ = ptty.Close()
-		return nil, err
+		return nil, xerrors.Errorf("start: %w", err)
 	}
 	return &otherPty{
 		pty: ptty,

Original file line number	Diff line number	Diff line change
`@@ -110,10 +110,13 @@ func (p *provisionerDaemon) connect(ctx context.Context) {`
`110`	`110`	`if errors.Is(err, context.Canceled) {`
`111`	`111`	`return`
`112`	`112`	`}`
	`113`	`+ p.closeMutex.Lock()`
`113`	`114`	`if p.isClosed() {`
	`115`	`+ p.closeMutex.Unlock()`
`114`	`116`	`return`
`115`	`117`	`}`
`116`	`118`	`p.opts.Logger.Warn(context.Background(), "failed to dial", slog.Error(err))`
	`119`	`+ p.closeMutex.Unlock()`
`117`	`120`	`continue`
`118`	`121`	`}`
`119`	`122`	`p.opts.Logger.Debug(context.Background(), "connected")`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ package pty`
`6`	`6`	`import (`
`7`	`7`	`"io"`
`8`	`8`	`"os"`
	`9`	`+ "sync"`
`9`	`10`
`10`	`11`	`"github.com/creack/pty"`
`11`	`12`	`)`
`@@ -23,6 +24,7 @@ func newPty() (PTY, error) {`
`23`	`24`	`}`
`24`	`25`
`25`	`26`	`type otherPty struct {`
	`27`	`+ mutex sync.Mutex`
`26`	`28`	`pty, tty *os.File`
`27`	`29`	`}`
`28`	`30`
`@@ -41,13 +43,18 @@ func (p *otherPty) Output() io.ReadWriter {`
`41`	`43`	`}`
`42`	`44`
`43`	`45`	`func (p *otherPty) Resize(cols uint16, rows uint16) error {`
	`46`	`+ p.mutex.Lock()`
	`47`	`+ defer p.mutex.Unlock()`
`44`	`48`	`return pty.Setsize(p.tty, &pty.Winsize{`
`45`	`49`	`Rows: rows,`
`46`	`50`	`Cols: cols,`
`47`	`51`	`})`
`48`	`52`	`}`
`49`	`53`
`50`	`54`	`func (p *otherPty) Close() error {`
	`55`	`+ p.mutex.Lock()`
	`56`	`+ defer p.mutex.Unlock()`
	`57`	`+`
`51`	`58`	`err := p.pty.Close()`
`52`	`59`	`if err != nil {`
`53`	`60`	`return err`