From 748ebf89ec8390e35883ca25b83573a8d35b7328 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 14:19:09 +0000 Subject: [PATCH 1/8] fix: TCP connections leaking after RTC disconnects --- wsnet/dial_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ wsnet/listen.go | 22 +++++++++++++++++----- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/wsnet/dial_test.go b/wsnet/dial_test.go index 5d75cfd4..da016bb4 100644 --- a/wsnet/dial_test.go +++ b/wsnet/dial_test.go @@ -266,6 +266,46 @@ func TestDial(t *testing.T) { _ = conn.Close() assert.Equal(t, 1, dialer.activeConnections()) }) + + t.Run("Close Listeners on Disconnect", func(t *testing.T) { + t.Parallel() + + tcpListener, err := net.Listen("tcp", "0.0.0.0:0") + require.NoError(t, err) + go func() { + _, _ = tcpListener.Accept() + }() + + connectAddr, listenAddr := createDumbBroker(t) + l, err := Listen(context.Background(), slogtest.Make(t, nil), listenAddr, "") + require.NoError(t, err) + + turnAddr, closeTurn := createTURNServer(t, ice.SchemeTypeTURN) + dialer, err := DialWebsocket(context.Background(), connectAddr, &DialOptions{ + ICEServers: []webrtc.ICEServer{{ + URLs: []string{fmt.Sprintf("turn:%s", turnAddr)}, + Username: "example", + Credential: testPass, + CredentialType: webrtc.ICECredentialTypePassword, + }}, + }, nil) + require.NoError(t, err) + + _, err = dialer.DialContext(context.Background(), "tcp", tcpListener.Addr().String()) + require.NoError(t, err) + + closeTurn() + + for i := 0; i < 15; i++ { + time.Sleep(time.Second) + + if len(l.(*listener).connClosers) == 0 { + break + } + } + + assert.Equal(t, 0, len(l.(*listener).connClosers)) + }) } func BenchmarkThroughput(b *testing.B) { diff --git a/wsnet/listen.go b/wsnet/listen.go index 02f13f41..42a448ef 100644 --- a/wsnet/listen.go +++ b/wsnet/listen.go @@ -255,17 +255,26 @@ func (l *listener) negotiate(ctx context.Context, conn net.Conn) { return } rtc.OnConnectionStateChange(func(pcs webrtc.PeerConnectionState) { - l.log.Debug(ctx, "connection state change", slog.F("state", pcs.String())) + l.log.Info(ctx, "connection state change", slog.F("state", pcs.String())) if pcs == webrtc.PeerConnectionStateConnecting { return } - _ = conn.Close() + if pcs == webrtc.PeerConnectionStateConnected { + // Safe to close the negotiating WebSocket. + _ = conn.Close() + return + } + + // Close connections opened when RTC was alive. + l.connClosersMut.Lock() + defer l.connClosersMut.Unlock() + for _, connCloser := range l.connClosers { + _ = connCloser.Close() + } + l.connClosers = make([]io.Closer, 0) }) flushCandidates := proxyICECandidates(rtc, conn) - l.connClosersMut.Lock() - l.connClosers = append(l.connClosers, rtc) - l.connClosersMut.Unlock() rtc.OnDataChannel(l.handle(ctx, msg)) l.log.Debug(ctx, "set remote description", slog.F("offer", *msg.Offer)) @@ -420,6 +429,9 @@ func (l *listener) handle(ctx context.Context, msg BrokerMessage) func(dc *webrt dc: dc, rw: rw, } + l.connClosersMut.Lock() + l.connClosers = append(l.connClosers, co) + l.connClosersMut.Unlock() co.init() defer nc.Close() defer co.Close() From c89026e52ea01f23d665695f99fe09d4e8a9bd9e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 14:24:30 +0000 Subject: [PATCH 2/8] Fix race condition --- wsnet/dial_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/wsnet/dial_test.go b/wsnet/dial_test.go index da016bb4..1c3ee421 100644 --- a/wsnet/dial_test.go +++ b/wsnet/dial_test.go @@ -296,15 +296,22 @@ func TestDial(t *testing.T) { closeTurn() + list := l.(*listener) + for i := 0; i < 15; i++ { time.Sleep(time.Second) - if len(l.(*listener).connClosers) == 0 { + list.connClosersMut.Lock() + size := len(list.connClosers) + list.connClosersMut.Unlock() + if size == 0 { break } } - assert.Equal(t, 0, len(l.(*listener).connClosers)) + list.connClosersMut.Lock() + assert.Equal(t, 0, len(list.connClosers)) + list.connClosersMut.Unlock() }) } From c8ec3e171235e514324e034ad89eb49c6b486a5b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 14:27:53 +0000 Subject: [PATCH 3/8] Add switch --- wsnet/dial_test.go | 18 ++++-------------- wsnet/listen.go | 6 +++--- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/wsnet/dial_test.go b/wsnet/dial_test.go index 1c3ee421..c138a958 100644 --- a/wsnet/dial_test.go +++ b/wsnet/dial_test.go @@ -297,21 +297,11 @@ func TestDial(t *testing.T) { closeTurn() list := l.(*listener) - - for i := 0; i < 15; i++ { - time.Sleep(time.Second) - + assert.Eventually(t, func() bool { list.connClosersMut.Lock() - size := len(list.connClosers) - list.connClosersMut.Unlock() - if size == 0 { - break - } - } - - list.connClosersMut.Lock() - assert.Equal(t, 0, len(list.connClosers)) - list.connClosersMut.Unlock() + defer list.connClosersMut.Unlock() + return len(list.connClosers) == 0 + }, time.Second*15, time.Second) }) } diff --git a/wsnet/listen.go b/wsnet/listen.go index 42a448ef..1964b647 100644 --- a/wsnet/listen.go +++ b/wsnet/listen.go @@ -256,10 +256,10 @@ func (l *listener) negotiate(ctx context.Context, conn net.Conn) { } rtc.OnConnectionStateChange(func(pcs webrtc.PeerConnectionState) { l.log.Info(ctx, "connection state change", slog.F("state", pcs.String())) - if pcs == webrtc.PeerConnectionStateConnecting { + switch pcs { + case webrtc.PeerConnectionStateConnected: return - } - if pcs == webrtc.PeerConnectionStateConnected { + case webrtc.PeerConnectionStateConnecting: // Safe to close the negotiating WebSocket. _ = conn.Close() return From 6d888d2f9987ee57e28a58df15d00d9433019d61 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 14:28:14 +0000 Subject: [PATCH 4/8] Lower tick time --- wsnet/dial_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wsnet/dial_test.go b/wsnet/dial_test.go index c138a958..79e7ae39 100644 --- a/wsnet/dial_test.go +++ b/wsnet/dial_test.go @@ -301,7 +301,7 @@ func TestDial(t *testing.T) { list.connClosersMut.Lock() defer list.connClosersMut.Unlock() return len(list.connClosers) == 0 - }, time.Second*15, time.Second) + }, time.Second*15, time.Millisecond*100) }) } From 76378039dc8ef3147d71b3345cc2e1e490a943c9 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 14:34:23 +0000 Subject: [PATCH 5/8] Fix data race --- wsnet/conn.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wsnet/conn.go b/wsnet/conn.go index de67c3c4..1b0bea42 100644 --- a/wsnet/conn.go +++ b/wsnet/conn.go @@ -169,7 +169,9 @@ func (c *dataChannelConn) Write(b []byte) (n int, err error) { func (c *dataChannelConn) Close() error { c.closedMutex.Lock() + c.writeMutex.Lock() defer c.closedMutex.Unlock() + defer c.writeMutex.Unlock() if !c.closed { c.closed = true close(c.sendMore) From 2adfb26227eff61b5db8e8edb55732a75bd24172 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 19:41:58 +0000 Subject: [PATCH 6/8] Update Go version --- ci/image/Dockerfile | 2 +- wsnet/conn.go | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/image/Dockerfile b/ci/image/Dockerfile index 08dde84b..b77bf8aa 100644 --- a/ci/image/Dockerfile +++ b/ci/image/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.16.3 +FROM golang:1.16.5 ENV GOFLAGS="-mod=readonly" ENV CI=true diff --git a/wsnet/conn.go b/wsnet/conn.go index 1b0bea42..de67c3c4 100644 --- a/wsnet/conn.go +++ b/wsnet/conn.go @@ -169,9 +169,7 @@ func (c *dataChannelConn) Write(b []byte) (n int, err error) { func (c *dataChannelConn) Close() error { c.closedMutex.Lock() - c.writeMutex.Lock() defer c.closedMutex.Unlock() - defer c.writeMutex.Unlock() if !c.closed { c.closed = true close(c.sendMore) From 6e59ce25ce016cd279cf7c47a00e8f9a2d5022e5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 22 Jul 2021 19:50:47 +0000 Subject: [PATCH 7/8] Fix log err --- wsnet/listen.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wsnet/listen.go b/wsnet/listen.go index 1964b647..803e140b 100644 --- a/wsnet/listen.go +++ b/wsnet/listen.go @@ -159,9 +159,11 @@ func (l *listener) dial(ctx context.Context) (<-chan error, error) { // so the cognitive overload linter has been disabled. // nolint:gocognit,nestif func (l *listener) negotiate(ctx context.Context, conn net.Conn) { + id := atomic.AddInt64(&l.nextConnNumber, 1) + ctx = slog.With(ctx, slog.F("conn_id", id)) + var ( err error - id = atomic.AddInt64(&l.nextConnNumber, 1) decoder = json.NewDecoder(conn) rtc *webrtc.PeerConnection // If candidates are sent before an offer, we place them here. @@ -171,7 +173,7 @@ func (l *listener) negotiate(ctx context.Context, conn net.Conn) { // Sends the error provided then closes the connection. // If RTC isn't connected, we'll close it. closeError = func(err error) { - l.log.Warn(ctx, "negotiation error, closing connection", slog.Error(err)) + // l.log.Warn(ctx, "negotiation error, closing connection", slog.Error(err)) d, _ := json.Marshal(&BrokerMessage{ Error: err.Error(), @@ -187,7 +189,6 @@ func (l *listener) negotiate(ctx context.Context, conn net.Conn) { } ) - ctx = slog.With(ctx, slog.F("conn_id", id)) l.log.Info(ctx, "accepted new session from broker connection, negotiating") for { From cd3e6a09c53f0163cd0f29e2edc97c5cf0792a8e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 26 Jul 2021 16:12:35 +0000 Subject: [PATCH 8/8] Fix race --- wsnet/conn.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wsnet/conn.go b/wsnet/conn.go index de67c3c4..40fa50ae 100644 --- a/wsnet/conn.go +++ b/wsnet/conn.go @@ -129,6 +129,8 @@ type dataChannelConn struct { } func (c *dataChannelConn) init() { + c.closedMutex.Lock() + defer c.closedMutex.Unlock() c.sendMore = make(chan struct{}, 1) c.dc.SetBufferedAmountLowThreshold(bufferedAmountLowThreshold) c.dc.OnBufferedAmountLow(func() {