From bf2f59fdae7747598dfbbb745abf0da04cf455c5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 02:36:53 +0000 Subject: [PATCH 01/22] fix: Synchronize peer logging with a channel We were depending on the close mutex to properly report connection state. This ensures the RTC connection is properly closed before returning. --- peer/conn.go | 110 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 38 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 8ee3e9dd490c9..b4ae3866f98e9 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -3,6 +3,8 @@ package peer import ( "bytes" "context" + "fmt" + "crypto/rand" "crypto/sha256" "io" @@ -69,6 +71,8 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp rtc: rtc, offerrer: client, closed: make(chan struct{}), + closedRTC: make(chan struct{}), + closedICE: make(chan struct{}), dcOpenChannel: make(chan *webrtc.DataChannel), dcDisconnectChannel: make(chan struct{}), dcFailedChannel: make(chan struct{}), @@ -76,8 +80,8 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // of this will cause a connection failure. localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), pendingRemoteCandidates: make([]webrtc.ICECandidateInit, 0), - localSessionDescriptionChannel: make(chan webrtc.SessionDescription), - remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription), + localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), + remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } if client { // If we're the client, we want to flip the echo and @@ -109,6 +113,8 @@ type Conn struct { offerrer bool closed chan struct{} + closedRTC chan struct{} + closedICE chan struct{} closeMutex sync.Mutex closeError error @@ -142,26 +148,22 @@ type Conn struct { func (c *Conn) init() error { c.rtc.OnNegotiationNeeded(c.negotiate) c.rtc.OnICEConnectionStateChange(func(iceConnectionState webrtc.ICEConnectionState) { - // Close must be locked here otherwise log output can appear - // after the connection has been closed. - c.closeMutex.Lock() - defer c.closeMutex.Unlock() - if c.isClosed() { - return - } - c.opts.Logger.Debug(context.Background(), "ice connection state updated", slog.F("state", iceConnectionState)) + + if iceConnectionState == webrtc.ICEConnectionStateClosed { + select { + case <-c.closedICE: + default: + close(c.closedICE) + } + } + }) + c.rtc.OnSignalingStateChange(func(signalState webrtc.SignalingState) { + c.opts.Logger.Debug(context.Background(), "signal state updated", + slog.F("state", signalState)) }) c.rtc.OnICEGatheringStateChange(func(iceGatherState webrtc.ICEGathererState) { - // Close can't be locked here, because this is triggered - // when close is called. It doesn't appear this get's - // executed after close though, so it shouldn't cause - // problems. - if c.isClosed() { - return - } - c.opts.Logger.Debug(context.Background(), "ice gathering state updated", slog.F("state", iceGatherState)) }) @@ -171,7 +173,7 @@ func (c *Conn) init() error { } json := iceCandidate.ToJSON() c.opts.Logger.Debug(context.Background(), "writing candidate to channel", - slog.F("hash", sha256.Sum224([]byte(json.Candidate))), + slog.F("hash", c.hashCandidate(json)), slog.F("length", len(json.Candidate)), ) select { @@ -188,19 +190,11 @@ func (c *Conn) init() error { default: } }) - c.rtc.OnConnectionStateChange(func(pcs webrtc.PeerConnectionState) { - // Close must be locked here otherwise log output can appear - // after the connection has been closed. - c.closeMutex.Lock() - defer c.closeMutex.Unlock() - if c.isClosed() { - return - } - + c.rtc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) { c.opts.Logger.Debug(context.Background(), "rtc connection updated", - slog.F("state", pcs)) + slog.F("state", peerConnectionState)) - switch pcs { + switch peerConnectionState { case webrtc.PeerConnectionStateDisconnected: for i := 0; i < int(c.dcDisconnectListeners.Load()); i++ { select { @@ -216,6 +210,20 @@ func (c *Conn) init() error { } } } + + if peerConnectionState == webrtc.PeerConnectionStateClosed { + // Pion executes event handlers after close is called + // on the RTC connection. This ensures our Close() + // handler properly cleans up before returning. + // + // Pion can execute this multiple times, so we check + // if it's open before closing. + select { + case <-c.closedRTC: + default: + close(c.closedRTC) + } + } }) _, err := c.pingChannel() if err != nil { @@ -275,17 +283,30 @@ func (c *Conn) pingEchoChannel() (*Channel, error) { return c.pingEchoChan, c.pingEchoError } +// Computes a hash of the ICE candidate. Used for debug logging. +func (*Conn) hashCandidate(iceCandidate webrtc.ICECandidateInit) string { + hash := sha256.Sum224([]byte(iceCandidate.Candidate)) + return fmt.Sprintf("%x", hash[:8]) +} + func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") if c.offerrer { + c.closeMutex.Lock() offer, err := c.rtc.CreateOffer(&webrtc.OfferOptions{}) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("create offer: %w", err)) return } - c.opts.Logger.Debug(context.Background(), "setting local description") + c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) + if c.isClosed() { + return + } + c.closeMutex.Lock() err = c.rtc.SetLocalDescription(offer) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return @@ -305,7 +326,9 @@ func (c *Conn) negotiate() { } c.opts.Logger.Debug(context.Background(), "setting remote description") + c.closeMutex.Lock() err := c.rtc.SetRemoteDescription(remoteDescription) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) return @@ -317,15 +340,17 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("create answer: %w", err)) return } - c.opts.Logger.Debug(context.Background(), "setting local description") + c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) + if c.isClosed() { + return + } + c.closeMutex.Lock() err = c.rtc.SetLocalDescription(answer) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return } - if c.isClosed() { - return - } select { case <-c.closed: return @@ -339,9 +364,8 @@ func (c *Conn) negotiate() { c.pendingCandidatesMutex.Lock() defer c.pendingCandidatesMutex.Unlock() for _, pendingCandidate := range c.pendingRemoteCandidates { - hash := sha256.Sum224([]byte(pendingCandidate.Candidate)) c.opts.Logger.Debug(context.Background(), "flushing buffered remote candidate", - slog.F("hash", hash), + slog.F("hash", c.hashCandidate(pendingCandidate)), slog.F("length", len(pendingCandidate.Candidate)), ) err := c.rtc.AddICECandidate(pendingCandidate) @@ -368,7 +392,7 @@ func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { c.pendingCandidatesMutex.Lock() defer c.pendingCandidatesMutex.Unlock() fields := []slog.Field{ - slog.F("hash", sha256.Sum224([]byte(i.Candidate))), + slog.F("hash", c.hashCandidate(i)), slog.F("length", len(i.Candidate)), } if !c.pendingCandidatesFlushed { @@ -542,5 +566,15 @@ func (c *Conn) CloseWithError(err error) error { // All logging, goroutines, and async functionality is cleaned up after this. c.dcClosedWaitGroup.Wait() + if c.rtc.ConnectionState() != webrtc.PeerConnectionStateNew { + c.opts.Logger.Debug(context.Background(), "waiting for rtc connection close...") + <-c.closedRTC + } + + if c.rtc.ICEConnectionState() != webrtc.ICEConnectionStateNew { + c.opts.Logger.Debug(context.Background(), "waiting for ice connection close...") + <-c.closedICE + } + return err } From aad88e057bb17a1518f68646ca1780c6c3e20b33 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 05:42:37 +0000 Subject: [PATCH 02/22] Disable pion logging --- peer/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peer/conn.go b/peer/conn.go index 850da40b71029..b4ae3866f98e9 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -55,7 +55,7 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp opts.SettingEngine.SetSrflxAcceptanceMinWait(0) opts.SettingEngine.DetachDataChannels() lf := logging.NewDefaultLoggerFactory() - lf.DefaultLogLevel = logging.LogLevelTrace + lf.DefaultLogLevel = logging.LogLevelDisabled opts.SettingEngine.LoggerFactory = lf api := webrtc.NewAPI(webrtc.WithSettingEngine(opts.SettingEngine)) rtc, err := api.NewPeerConnection(webrtc.Configuration{ From 59751f4ee83c5760f2528d2db99086956ffa3a00 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 06:02:15 +0000 Subject: [PATCH 03/22] Remove buffer --- peer/conn.go | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index b4ae3866f98e9..6d154faf20a16 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -79,7 +79,6 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), - pendingRemoteCandidates: make([]webrtc.ICECandidateInit, 0), localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } @@ -129,10 +128,6 @@ type Conn struct { localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription - pendingRemoteCandidates []webrtc.ICECandidateInit - pendingCandidatesMutex sync.Mutex - pendingCandidatesFlushed bool - pingChannelID uint16 pingEchoChannelID uint16 @@ -357,28 +352,6 @@ func (c *Conn) negotiate() { case c.localSessionDescriptionChannel <- answer: } } - - // The ICE transport resets when the remote description is updated. - // Adding ICE candidates before this point causes a failed connection, - // because the candidate would be lost. - c.pendingCandidatesMutex.Lock() - defer c.pendingCandidatesMutex.Unlock() - for _, pendingCandidate := range c.pendingRemoteCandidates { - c.opts.Logger.Debug(context.Background(), "flushing buffered remote candidate", - slog.F("hash", c.hashCandidate(pendingCandidate)), - slog.F("length", len(pendingCandidate.Candidate)), - ) - err := c.rtc.AddICECandidate(pendingCandidate) - if err != nil { - _ = c.CloseWithError(xerrors.Errorf("flush pending remote candidate: %w", err)) - return - } - } - c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", - slog.F("count", len(c.pendingRemoteCandidates)), - ) - c.pendingCandidatesFlushed = true - c.pendingRemoteCandidates = make([]webrtc.ICECandidateInit, 0) } // LocalCandidate returns a channel that emits when a local candidate @@ -389,18 +362,10 @@ func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { // AddRemoteCandidate adds a remote candidate to the RTC connection. func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { - c.pendingCandidatesMutex.Lock() - defer c.pendingCandidatesMutex.Unlock() - fields := []slog.Field{ + c.opts.Logger.Debug(context.Background(), "adding remote candidate", slog.F("hash", c.hashCandidate(i)), slog.F("length", len(i.Candidate)), - } - if !c.pendingCandidatesFlushed { - c.opts.Logger.Debug(context.Background(), "bufferring remote candidate", fields...) - c.pendingRemoteCandidates = append(c.pendingRemoteCandidates, i) - return nil - } - c.opts.Logger.Debug(context.Background(), "adding remote candidate", fields...) + ) return c.rtc.AddICECandidate(i) } From 107eb968a7de74b76096d1815afc0179ce04c01e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 06:22:49 +0000 Subject: [PATCH 04/22] Try ICE servers --- peer/conn.go | 39 +++++++++++++++++++++++++++++++++++++-- peer/conn_test.go | 8 ++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 6d154faf20a16..2dcc3671fdb21 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -79,6 +79,7 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), + pendingRemoteCandidates: make([]webrtc.ICECandidateInit, 0), localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } @@ -128,6 +129,10 @@ type Conn struct { localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription + pendingRemoteCandidates []webrtc.ICECandidateInit + pendingCandidatesMutex sync.Mutex + pendingCandidatesFlushed bool + pingChannelID uint16 pingEchoChannelID uint16 @@ -329,6 +334,28 @@ func (c *Conn) negotiate() { return } + // The RemoteDescription must be set before ICE candidates can be + // added to a WebRTC connection. + c.pendingCandidatesMutex.Lock() + for _, pendingCandidate := range c.pendingRemoteCandidates { + hash := sha256.Sum224([]byte(pendingCandidate.Candidate)) + c.opts.Logger.Debug(context.Background(), "flushing buffered remote candidate", + slog.F("hash", hash), + slog.F("length", len(pendingCandidate.Candidate)), + ) + err := c.rtc.AddICECandidate(pendingCandidate) + if err != nil { + _ = c.CloseWithError(xerrors.Errorf("flush pending remote candidate: %w", err)) + return + } + } + c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", + slog.F("count", len(c.pendingRemoteCandidates)), + ) + c.pendingCandidatesFlushed = true + c.pendingRemoteCandidates = make([]webrtc.ICECandidateInit, 0) + c.pendingCandidatesMutex.Unlock() + if !c.offerrer { answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) if err != nil { @@ -362,10 +389,18 @@ func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { // AddRemoteCandidate adds a remote candidate to the RTC connection. func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { - c.opts.Logger.Debug(context.Background(), "adding remote candidate", + c.pendingCandidatesMutex.Lock() + defer c.pendingCandidatesMutex.Unlock() + fields := []slog.Field{ slog.F("hash", c.hashCandidate(i)), slog.F("length", len(i.Candidate)), - ) + } + if !c.pendingCandidatesFlushed { + c.opts.Logger.Debug(context.Background(), "bufferring remote candidate", fields...) + c.pendingRemoteCandidates = append(c.pendingRemoteCandidates, i) + return nil + } + c.opts.Logger.Debug(context.Background(), "adding remote candidate", fields...) return c.rtc.AddICECandidate(i) } diff --git a/peer/conn_test.go b/peer/conn_test.go index cc59e0cfbc93d..48be373432307 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -271,7 +271,9 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R c1SettingEngine.SetVNet(c1Net) c1SettingEngine.SetPrflxAcceptanceMinWait(0) c1SettingEngine.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval) - channel1, err := peer.Client([]webrtc.ICEServer{}, &peer.ConnOptions{ + channel1, err := peer.Client([]webrtc.ICEServer{{ + URLs: []string{"stun:stun.l.google.com:19302"}, + }}, &peer.ConnOptions{ SettingEngine: c1SettingEngine, Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), }) @@ -283,7 +285,9 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R c2SettingEngine.SetVNet(c2Net) c2SettingEngine.SetPrflxAcceptanceMinWait(0) c2SettingEngine.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval) - channel2, err := peer.Server([]webrtc.ICEServer{}, &peer.ConnOptions{ + channel2, err := peer.Server([]webrtc.ICEServer{{ + URLs: []string{"stun:stun.l.google.com:19302"}, + }}, &peer.ConnOptions{ SettingEngine: c2SettingEngine, Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), }) From 86cb3d4c427543514b703b27d3736767dd561552 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 13:21:57 +0000 Subject: [PATCH 05/22] Remove flushed --- peer/conn.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 2dcc3671fdb21..8992a74bb9360 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -129,9 +129,8 @@ type Conn struct { localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription - pendingRemoteCandidates []webrtc.ICECandidateInit - pendingCandidatesMutex sync.Mutex - pendingCandidatesFlushed bool + pendingRemoteCandidates []webrtc.ICECandidateInit + pendingCandidatesMutex sync.Mutex pingChannelID uint16 pingEchoChannelID uint16 @@ -352,7 +351,6 @@ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", slog.F("count", len(c.pendingRemoteCandidates)), ) - c.pendingCandidatesFlushed = true c.pendingRemoteCandidates = make([]webrtc.ICECandidateInit, 0) c.pendingCandidatesMutex.Unlock() @@ -395,7 +393,7 @@ func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { slog.F("hash", c.hashCandidate(i)), slog.F("length", len(i.Candidate)), } - if !c.pendingCandidatesFlushed { + if c.rtc.RemoteDescription() == nil { c.opts.Logger.Debug(context.Background(), "bufferring remote candidate", fields...) c.pendingRemoteCandidates = append(c.pendingRemoteCandidates, i) return nil From 90d26f2a22d338b74a0ab590725c04baf429bba3 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 14:33:17 +0000 Subject: [PATCH 06/22] Add diagram explaining handshake --- peer/conn.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/peer/conn.go b/peer/conn.go index 8992a74bb9360..5d9cfdcb3b045 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -288,6 +288,22 @@ func (*Conn) hashCandidate(iceCandidate webrtc.ICECandidateInit) string { return fmt.Sprintf("%x", hash[:8]) } +// Negotiate exchanges ICECandidate pairs over the exposed channels. +// The diagram below shows the expected handshake. pion/webrtc v3 +// uses trickle ICE by default. See: https://webrtchacks.com/trickle-ice/ +// ┌────────┐ ┌────────┐ +// │offerrer│ │answerer│ +// │(client)│ │(server)│ +// └─┬────┬─┘ └─┬──────┘ +// │ │ offer │ +// ┌──────────▼┐ ├──────────────►├──►┌───────────┐ +// │STUN Server│ │ │ │STUN Server│ +// │(optional) ├──►│ candidate │◄──┤(optional) │ +// └───────────┘ │ (async) │ └───────────┘ +// │◄─────────────►│ +// │ │ +// │ answer │ +// └◄──────────────┘ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") From a5bd7e42539410f4a9e17caec627d208267bd908 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 14:46:41 +0000 Subject: [PATCH 07/22] Fix candidate accept ordering --- peer/conn.go | 47 +++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 5d9cfdcb3b045..1057a212fa391 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -79,7 +79,7 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), - pendingRemoteCandidates: make([]webrtc.ICECandidateInit, 0), + pendingCandidates: make([]webrtc.ICECandidateInit, 0), localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } @@ -129,8 +129,8 @@ type Conn struct { localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription - pendingRemoteCandidates []webrtc.ICECandidateInit - pendingCandidatesMutex sync.Mutex + pendingCandidates []webrtc.ICECandidateInit + pendingCandidatesMutex sync.Mutex pingChannelID uint16 pingEchoChannelID uint16 @@ -170,11 +170,19 @@ func (c *Conn) init() error { if iceCandidate == nil { return } + c.pendingCandidatesMutex.Lock() + defer c.pendingCandidatesMutex.Unlock() json := iceCandidate.ToJSON() - c.opts.Logger.Debug(context.Background(), "writing candidate to channel", + fields := []slog.Field{ slog.F("hash", c.hashCandidate(json)), slog.F("length", len(json.Candidate)), - ) + } + if c.rtc.RemoteDescription() == nil { + c.pendingCandidates = append(c.pendingCandidates, json) + c.opts.Logger.Debug(context.Background(), "buffering candidate", fields...) + return + } + c.opts.Logger.Debug(context.Background(), "sending candidate directly", fields...) select { case <-c.closed: break @@ -352,22 +360,21 @@ func (c *Conn) negotiate() { // The RemoteDescription must be set before ICE candidates can be // added to a WebRTC connection. c.pendingCandidatesMutex.Lock() - for _, pendingCandidate := range c.pendingRemoteCandidates { - hash := sha256.Sum224([]byte(pendingCandidate.Candidate)) - c.opts.Logger.Debug(context.Background(), "flushing buffered remote candidate", - slog.F("hash", hash), + for _, pendingCandidate := range c.pendingCandidates { + c.opts.Logger.Debug(context.Background(), "sending buffered remote candidate", + slog.F("hash", c.hashCandidate(pendingCandidate)), slog.F("length", len(pendingCandidate.Candidate)), ) - err := c.rtc.AddICECandidate(pendingCandidate) - if err != nil { - _ = c.CloseWithError(xerrors.Errorf("flush pending remote candidate: %w", err)) + select { + case <-c.closed: return + case c.localCandidateChannel <- pendingCandidate: } } c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", - slog.F("count", len(c.pendingRemoteCandidates)), + slog.F("count", len(c.pendingCandidates)), ) - c.pendingRemoteCandidates = make([]webrtc.ICECandidateInit, 0) + c.pendingCandidates = make([]webrtc.ICECandidateInit, 0) c.pendingCandidatesMutex.Unlock() if !c.offerrer { @@ -403,18 +410,10 @@ func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { // AddRemoteCandidate adds a remote candidate to the RTC connection. func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { - c.pendingCandidatesMutex.Lock() - defer c.pendingCandidatesMutex.Unlock() - fields := []slog.Field{ + c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("hash", c.hashCandidate(i)), slog.F("length", len(i.Candidate)), - } - if c.rtc.RemoteDescription() == nil { - c.opts.Logger.Debug(context.Background(), "bufferring remote candidate", fields...) - c.pendingRemoteCandidates = append(c.pendingRemoteCandidates, i) - return nil - } - c.opts.Logger.Debug(context.Background(), "adding remote candidate", fields...) + ) return c.rtc.AddICECandidate(i) } From 6528cdff34617d9c284a0c455fae6f8ba99a5fb0 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 15:03:47 +0000 Subject: [PATCH 08/22] Add debug logging to peerbroker --- peerbroker/dial_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/peerbroker/dial_test.go b/peerbroker/dial_test.go index 3d6fe807e0dab..30066b8d82397 100644 --- a/peerbroker/dial_test.go +++ b/peerbroker/dial_test.go @@ -9,6 +9,10 @@ import ( "go.uber.org/goleak" "storj.io/drpc/drpcconn" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" + + "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker" "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" @@ -28,7 +32,9 @@ func TestDial(t *testing.T) { defer client.Close() defer server.Close() - listener, err := peerbroker.Listen(server, nil) + listener, err := peerbroker.Listen(server, &peer.ConnOptions{ + Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), + }) require.NoError(t, err) api := proto.NewDRPCPeerBrokerClient(drpcconn.New(client)) @@ -36,7 +42,9 @@ func TestDial(t *testing.T) { require.NoError(t, err) clientConn, err := peerbroker.Dial(stream, []webrtc.ICEServer{{ URLs: []string{"stun:stun.l.google.com:19302"}, - }}, nil) + }}, &peer.ConnOptions{ + Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), + }) require.NoError(t, err) defer clientConn.Close() From b6003c78c1b6c2bedb933e0507952dac7d3438d9 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 15:12:17 +0000 Subject: [PATCH 09/22] Fix send ordering --- peer/conn.go | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 1057a212fa391..6a0b134752c02 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -357,25 +357,9 @@ func (c *Conn) negotiate() { return } - // The RemoteDescription must be set before ICE candidates can be - // added to a WebRTC connection. - c.pendingCandidatesMutex.Lock() - for _, pendingCandidate := range c.pendingCandidates { - c.opts.Logger.Debug(context.Background(), "sending buffered remote candidate", - slog.F("hash", c.hashCandidate(pendingCandidate)), - slog.F("length", len(pendingCandidate.Candidate)), - ) - select { - case <-c.closed: - return - case c.localCandidateChannel <- pendingCandidate: - } + if c.offerrer { + c.flushPendingCandidates() } - c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", - slog.F("count", len(c.pendingCandidates)), - ) - c.pendingCandidates = make([]webrtc.ICECandidateInit, 0) - c.pendingCandidatesMutex.Unlock() if !c.offerrer { answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) @@ -394,12 +378,35 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return } + select { case <-c.closed: return case c.localSessionDescriptionChannel <- answer: } + + c.flushPendingCandidates() + } +} + +func (c *Conn) flushPendingCandidates() { + c.pendingCandidatesMutex.Lock() + defer c.pendingCandidatesMutex.Unlock() + for _, pendingCandidate := range c.pendingCandidates { + c.opts.Logger.Debug(context.Background(), "sending buffered remote candidate", + slog.F("hash", c.hashCandidate(pendingCandidate)), + slog.F("length", len(pendingCandidate.Candidate)), + ) + select { + case <-c.closed: + return + case c.localCandidateChannel <- pendingCandidate: + } } + c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", + slog.F("count", len(c.pendingCandidates)), + ) + c.pendingCandidates = make([]webrtc.ICECandidateInit, 0) } // LocalCandidate returns a channel that emits when a local candidate From 7445f12b54510af27b845bd46995f3a6405ec892 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 15:21:46 +0000 Subject: [PATCH 10/22] Lock adding ICE candidate --- peer/conn.go | 80 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 6a0b134752c02..596940cdc5c94 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -79,7 +79,8 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), - pendingCandidates: make([]webrtc.ICECandidateInit, 0), + pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), + pendingCandidatesToAccept: make([]webrtc.ICECandidateInit, 0), localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } @@ -129,8 +130,11 @@ type Conn struct { localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription - pendingCandidates []webrtc.ICECandidateInit - pendingCandidatesMutex sync.Mutex + pendingCandidatesToSend []webrtc.ICECandidateInit + pendingCandidatesToSendMutex sync.Mutex + + pendingCandidatesToAccept []webrtc.ICECandidateInit + pendingCandidatesToAcceptMutex sync.Mutex pingChannelID uint16 pingEchoChannelID uint16 @@ -170,19 +174,19 @@ func (c *Conn) init() error { if iceCandidate == nil { return } - c.pendingCandidatesMutex.Lock() - defer c.pendingCandidatesMutex.Unlock() + c.pendingCandidatesToSendMutex.Lock() + defer c.pendingCandidatesToSendMutex.Unlock() json := iceCandidate.ToJSON() fields := []slog.Field{ slog.F("hash", c.hashCandidate(json)), slog.F("length", len(json.Candidate)), } if c.rtc.RemoteDescription() == nil { - c.pendingCandidates = append(c.pendingCandidates, json) - c.opts.Logger.Debug(context.Background(), "buffering candidate", fields...) + c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, json) + c.opts.Logger.Debug(context.Background(), "buffering local candidate to send", fields...) return } - c.opts.Logger.Debug(context.Background(), "sending candidate directly", fields...) + c.opts.Logger.Debug(context.Background(), "sending local candidate directly", fields...) select { case <-c.closed: break @@ -357,10 +361,6 @@ func (c *Conn) negotiate() { return } - if c.offerrer { - c.flushPendingCandidates() - } - if !c.offerrer { answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) if err != nil { @@ -384,16 +384,11 @@ func (c *Conn) negotiate() { return case c.localSessionDescriptionChannel <- answer: } - - c.flushPendingCandidates() } -} -func (c *Conn) flushPendingCandidates() { - c.pendingCandidatesMutex.Lock() - defer c.pendingCandidatesMutex.Unlock() - for _, pendingCandidate := range c.pendingCandidates { - c.opts.Logger.Debug(context.Background(), "sending buffered remote candidate", + c.pendingCandidatesToSendMutex.Lock() + for _, pendingCandidate := range c.pendingCandidatesToSend { + c.opts.Logger.Debug(context.Background(), "sending buffered local candidate", slog.F("hash", c.hashCandidate(pendingCandidate)), slog.F("length", len(pendingCandidate.Candidate)), ) @@ -403,10 +398,29 @@ func (c *Conn) flushPendingCandidates() { case c.localCandidateChannel <- pendingCandidate: } } + c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", + slog.F("count", len(c.pendingCandidatesToSend)), + ) + c.pendingCandidatesToSend = make([]webrtc.ICECandidateInit, 0) + c.pendingCandidatesToSendMutex.Unlock() + + c.pendingCandidatesToAcceptMutex.Lock() + defer c.pendingCandidatesToAcceptMutex.Unlock() + for _, pendingCandidate := range c.pendingCandidatesToAccept { + c.opts.Logger.Debug(context.Background(), "adding buffered remote candidate", + slog.F("hash", c.hashCandidate(pendingCandidate)), + slog.F("length", len(pendingCandidate.Candidate)), + ) + err = c.rtc.AddICECandidate(pendingCandidate) + if err != nil { + _ = c.CloseWithError(xerrors.Errorf("accept buffered remote candidate: %w", err)) + return + } + } c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", - slog.F("count", len(c.pendingCandidates)), + slog.F("count", len(c.pendingCandidatesToAccept)), ) - c.pendingCandidates = make([]webrtc.ICECandidateInit, 0) + c.pendingCandidatesToAccept = make([]webrtc.ICECandidateInit, 0) } // LocalCandidate returns a channel that emits when a local candidate @@ -416,12 +430,22 @@ func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { } // AddRemoteCandidate adds a remote candidate to the RTC connection. -func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { - c.opts.Logger.Debug(context.Background(), "accepting candidate", - slog.F("hash", c.hashCandidate(i)), - slog.F("length", len(i.Candidate)), - ) - return c.rtc.AddICECandidate(i) +func (c *Conn) AddRemoteCandidate(iceCandidate webrtc.ICECandidateInit) error { + c.pendingCandidatesToAcceptMutex.Lock() + defer c.pendingCandidatesToAcceptMutex.Unlock() + fields := []slog.Field{ + slog.F("hash", c.hashCandidate(iceCandidate)), + slog.F("length", len(iceCandidate.Candidate)), + } + // The consumer doesn't need to set the session description before + // adding remote candidates. This buffers it so an error doesn't occur. + if c.rtc.RemoteDescription() == nil { + c.opts.Logger.Debug(context.Background(), "buffering remote candidate to accept", fields...) + c.pendingCandidatesToAccept = append(c.pendingCandidatesToAccept, iceCandidate) + return nil + } + c.opts.Logger.Debug(context.Background(), "adding remote candidate", fields...) + return c.rtc.AddICECandidate(iceCandidate) } // LocalSessionDescription returns a channel that emits a session description From e155e31bacf1de6ce30561eb3d53ad3c52e34777 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 16:02:07 +0000 Subject: [PATCH 11/22] Add test for negotiating out of order --- peer/conn_test.go | 52 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/peer/conn_test.go b/peer/conn_test.go index 48be373432307..504cdc0144b1d 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -64,6 +64,7 @@ func TestConn(t *testing.T) { t.Run("Ping", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) _, err := client.Ping() require.NoError(t, err) _, err = server.Ping() @@ -72,7 +73,8 @@ func TestConn(t *testing.T) { t.Run("PingNetworkOffline", func(t *testing.T) { t.Parallel() - _, server, wan := createPair(t) + client, server, wan := createPair(t) + exchange(client, server) _, err := server.Ping() require.NoError(t, err) err = wan.Stop() @@ -83,7 +85,8 @@ func TestConn(t *testing.T) { t.Run("PingReconnect", func(t *testing.T) { t.Parallel() - _, server, wan := createPair(t) + client, server, wan := createPair(t) + exchange(client, server) _, err := server.Ping() require.NoError(t, err) // Create a channel that closes on disconnect. @@ -104,6 +107,7 @@ func TestConn(t *testing.T) { t.Run("Accept", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) @@ -119,6 +123,7 @@ func TestConn(t *testing.T) { t.Run("AcceptNetworkOffline", func(t *testing.T) { t.Parallel() client, server, wan := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) @@ -135,6 +140,7 @@ func TestConn(t *testing.T) { t.Run("Buffering", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) @@ -159,6 +165,7 @@ func TestConn(t *testing.T) { t.Run("NetConn", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) srv, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer srv.Close() @@ -211,6 +218,7 @@ func TestConn(t *testing.T) { t.Run("CloseBeforeNegotiate", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) err := client.Close() require.NoError(t, err) err = server.Close() @@ -230,6 +238,7 @@ func TestConn(t *testing.T) { t.Run("PingConcurrent", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) var wg sync.WaitGroup wg.Add(2) go func() { @@ -244,6 +253,19 @@ func TestConn(t *testing.T) { }() wg.Wait() }) + + t.Run("NegotiateOutOfOrder", func(t *testing.T) { + t.Parallel() + client, server, _ := createPair(t) + server.SetRemoteSessionDescription(<-client.LocalSessionDescription()) + err := client.AddRemoteCandidate(<-server.LocalCandidate()) + require.NoError(t, err) + client.SetRemoteSessionDescription(<-server.LocalSessionDescription()) + err = server.AddRemoteCandidate(<-client.LocalCandidate()) + require.NoError(t, err) + _, err = client.Ping() + require.NoError(t, err) + }) } func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.Router) { @@ -302,14 +324,18 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R _ = wan.Stop() }) + return channel1, channel2, wan +} + +func exchange(client *peer.Conn, server *peer.Conn) { go func() { for { select { - case c := <-channel2.LocalCandidate(): - _ = channel1.AddRemoteCandidate(c) - case c := <-channel2.LocalSessionDescription(): - channel1.SetRemoteSessionDescription(c) - case <-channel2.Closed(): + case c := <-server.LocalCandidate(): + _ = client.AddRemoteCandidate(c) + case c := <-server.LocalSessionDescription(): + client.SetRemoteSessionDescription(c) + case <-server.Closed(): return } } @@ -318,15 +344,13 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R go func() { for { select { - case c := <-channel1.LocalCandidate(): - _ = channel2.AddRemoteCandidate(c) - case c := <-channel1.LocalSessionDescription(): - channel2.SetRemoteSessionDescription(c) - case <-channel1.Closed(): + case c := <-client.LocalCandidate(): + _ = server.AddRemoteCandidate(c) + case c := <-client.LocalSessionDescription(): + server.SetRemoteSessionDescription(c) + case <-client.Closed(): return } } }() - - return channel1, channel2, wan } From 65f378cba4b676066866ad9191babb8d90f99be5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 16:53:12 +0000 Subject: [PATCH 12/22] Reduce connection to a single negotiation channel --- peer/conn.go | 132 ++++++--------- peer/conn_test.go | 56 ++---- peerbroker/dial.go | 80 +++++---- peerbroker/listen.go | 38 +---- peerbroker/proto/peerbroker.pb.go | 271 ++++++++++++++++-------------- peerbroker/proto/peerbroker.proto | 13 +- 6 files changed, 272 insertions(+), 318 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 596940cdc5c94..8638df549e616 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -78,11 +78,9 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp dcFailedChannel: make(chan struct{}), // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. - localCandidateChannel: make(chan webrtc.ICECandidateInit, 16), - pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), - pendingCandidatesToAccept: make([]webrtc.ICECandidateInit, 0), - localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), - remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), + localNegotiator: make(chan Negotiation, 8), + remoteSessionDescription: make(chan webrtc.SessionDescription, 1), + pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), } if client { // If we're the client, we want to flip the echo and @@ -126,15 +124,12 @@ type Conn struct { dcFailedListeners atomic.Uint32 dcClosedWaitGroup sync.WaitGroup - localCandidateChannel chan webrtc.ICECandidateInit - localSessionDescriptionChannel chan webrtc.SessionDescription - remoteSessionDescriptionChannel chan webrtc.SessionDescription + localNegotiator chan Negotiation + remoteSessionDescription chan webrtc.SessionDescription pendingCandidatesToSend []webrtc.ICECandidateInit pendingCandidatesToSendMutex sync.Mutex - - pendingCandidatesToAccept []webrtc.ICECandidateInit - pendingCandidatesToAcceptMutex sync.Mutex + pendingCandidatesFlushed bool pingChannelID uint16 pingEchoChannelID uint16 @@ -148,6 +143,12 @@ type Conn struct { pingError error } +// Negotiation represents a handshake message between peer connections. +type Negotiation struct { + SessionDescription *webrtc.SessionDescription + ICECandidates []webrtc.ICECandidateInit +} + func (c *Conn) init() error { c.rtc.OnNegotiationNeeded(c.negotiate) c.rtc.OnICEConnectionStateChange(func(iceConnectionState webrtc.ICEConnectionState) { @@ -181,7 +182,7 @@ func (c *Conn) init() error { slog.F("hash", c.hashCandidate(json)), slog.F("length", len(json.Candidate)), } - if c.rtc.RemoteDescription() == nil { + if !c.pendingCandidatesFlushed { c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, json) c.opts.Logger.Debug(context.Background(), "buffering local candidate to send", fields...) return @@ -190,7 +191,7 @@ func (c *Conn) init() error { select { case <-c.closed: break - case c.localCandidateChannel <- json: + case c.localNegotiator <- Negotiation{nil, []webrtc.ICECandidateInit{json}}: } }) c.rtc.OnDataChannel(func(dc *webrtc.DataChannel) { @@ -341,20 +342,20 @@ func (c *Conn) negotiate() { select { case <-c.closed: return - case c.localSessionDescriptionChannel <- offer: + case c.localNegotiator <- Negotiation{&offer, nil}: } } - var remoteDescription webrtc.SessionDescription + var sessionDescription webrtc.SessionDescription select { case <-c.closed: return - case remoteDescription = <-c.remoteSessionDescriptionChannel: + case sessionDescription = <-c.remoteSessionDescription: } c.opts.Logger.Debug(context.Background(), "setting remote description") c.closeMutex.Lock() - err := c.rtc.SetRemoteDescription(remoteDescription) + err := c.rtc.SetRemoteDescription(sessionDescription) c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) @@ -378,80 +379,58 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return } - + c.opts.Logger.Debug(context.Background(), "sending answer") select { case <-c.closed: return - case c.localSessionDescriptionChannel <- answer: + case c.localNegotiator <- Negotiation{&answer, nil}: } } c.pendingCandidatesToSendMutex.Lock() - for _, pendingCandidate := range c.pendingCandidatesToSend { - c.opts.Logger.Debug(context.Background(), "sending buffered local candidate", - slog.F("hash", c.hashCandidate(pendingCandidate)), - slog.F("length", len(pendingCandidate.Candidate)), - ) + defer c.pendingCandidatesToSendMutex.Unlock() + if len(c.pendingCandidatesToSend) > 0 { select { case <-c.closed: return - case c.localCandidateChannel <- pendingCandidate: + case c.localNegotiator <- Negotiation{nil, c.pendingCandidatesToSend}: } } c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", slog.F("count", len(c.pendingCandidatesToSend)), ) c.pendingCandidatesToSend = make([]webrtc.ICECandidateInit, 0) - c.pendingCandidatesToSendMutex.Unlock() - - c.pendingCandidatesToAcceptMutex.Lock() - defer c.pendingCandidatesToAcceptMutex.Unlock() - for _, pendingCandidate := range c.pendingCandidatesToAccept { - c.opts.Logger.Debug(context.Background(), "adding buffered remote candidate", - slog.F("hash", c.hashCandidate(pendingCandidate)), - slog.F("length", len(pendingCandidate.Candidate)), - ) - err = c.rtc.AddICECandidate(pendingCandidate) - if err != nil { - _ = c.CloseWithError(xerrors.Errorf("accept buffered remote candidate: %w", err)) - return - } - } - c.opts.Logger.Debug(context.Background(), "flushed buffered remote candidates", - slog.F("count", len(c.pendingCandidatesToAccept)), - ) - c.pendingCandidatesToAccept = make([]webrtc.ICECandidateInit, 0) + c.pendingCandidatesFlushed = true } -// LocalCandidate returns a channel that emits when a local candidate -// needs to be exchanged with a remote connection. -func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { - return c.localCandidateChannel +func (c *Conn) LocalNegotiation() <-chan Negotiation { + return c.localNegotiator } -// AddRemoteCandidate adds a remote candidate to the RTC connection. -func (c *Conn) AddRemoteCandidate(iceCandidate webrtc.ICECandidateInit) error { - c.pendingCandidatesToAcceptMutex.Lock() - defer c.pendingCandidatesToAcceptMutex.Unlock() - fields := []slog.Field{ - slog.F("hash", c.hashCandidate(iceCandidate)), - slog.F("length", len(iceCandidate.Candidate)), - } - // The consumer doesn't need to set the session description before - // adding remote candidates. This buffers it so an error doesn't occur. - if c.rtc.RemoteDescription() == nil { - c.opts.Logger.Debug(context.Background(), "buffering remote candidate to accept", fields...) - c.pendingCandidatesToAccept = append(c.pendingCandidatesToAccept, iceCandidate) - return nil - } - c.opts.Logger.Debug(context.Background(), "adding remote candidate", fields...) - return c.rtc.AddICECandidate(iceCandidate) -} +func (c *Conn) AddRemoteNegotiation(negotiation Negotiation) error { + if negotiation.SessionDescription != nil { + c.opts.Logger.Debug(context.Background(), "adding remote negotiation with session description") + select { + case <-c.closed: + return nil + case c.remoteSessionDescription <- *negotiation.SessionDescription: + } + } + + if len(negotiation.ICECandidates) > 0 { + c.opts.Logger.Debug(context.Background(), "adding remote negotiation with ice candidates", + slog.F("count", len(negotiation.ICECandidates))) + c.closeMutex.Lock() + defer c.closeMutex.Unlock() + for _, iceCandidate := range negotiation.ICECandidates { + err := c.rtc.AddICECandidate(iceCandidate) + if err != nil { + return err + } + } + } -// LocalSessionDescription returns a channel that emits a session description -// when one is required to be exchanged. -func (c *Conn) LocalSessionDescription() <-chan webrtc.SessionDescription { - return c.localSessionDescriptionChannel + return nil } // SetConfiguration applies options to the WebRTC connection. @@ -460,19 +439,6 @@ func (c *Conn) SetConfiguration(configuration webrtc.Configuration) error { return c.rtc.SetConfiguration(configuration) } -// SetRemoteSessionDescription sets the remote description for the WebRTC connection. -func (c *Conn) SetRemoteSessionDescription(sessionDescription webrtc.SessionDescription) { - if c.isClosed() { - return - } - c.closeMutex.Lock() - defer c.closeMutex.Unlock() - select { - case <-c.closed: - case c.remoteSessionDescriptionChannel <- sessionDescription: - } -} - // Accept blocks waiting for a channel to be opened. func (c *Conn) Accept(ctx context.Context) (*Channel, error) { var dataChannel *webrtc.DataChannel diff --git a/peer/conn_test.go b/peer/conn_test.go index 504cdc0144b1d..2bdf450cae144 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -35,7 +35,7 @@ var ( // In CI resources are frequently contended, so increasing this value // results in less flakes. if os.Getenv("CI") == "true" { - return 3 * time.Second + return time.Second } return 100 * time.Millisecond }() @@ -64,7 +64,6 @@ func TestConn(t *testing.T) { t.Run("Ping", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) _, err := client.Ping() require.NoError(t, err) _, err = server.Ping() @@ -73,8 +72,7 @@ func TestConn(t *testing.T) { t.Run("PingNetworkOffline", func(t *testing.T) { t.Parallel() - client, server, wan := createPair(t) - exchange(client, server) + _, server, wan := createPair(t) _, err := server.Ping() require.NoError(t, err) err = wan.Stop() @@ -85,8 +83,7 @@ func TestConn(t *testing.T) { t.Run("PingReconnect", func(t *testing.T) { t.Parallel() - client, server, wan := createPair(t) - exchange(client, server) + _, server, wan := createPair(t) _, err := server.Ping() require.NoError(t, err) // Create a channel that closes on disconnect. @@ -107,7 +104,6 @@ func TestConn(t *testing.T) { t.Run("Accept", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) @@ -123,7 +119,6 @@ func TestConn(t *testing.T) { t.Run("AcceptNetworkOffline", func(t *testing.T) { t.Parallel() client, server, wan := createPair(t) - exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) @@ -140,21 +135,22 @@ func TestConn(t *testing.T) { t.Run("Buffering", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) require.NoError(t, err) defer sch.Close() go func() { + bytes := make([]byte, 4096) for i := 0; i < 1024; i++ { - _, err := cch.Write(make([]byte, 4096)) + _, err := cch.Write(bytes) require.NoError(t, err) } _ = cch.Close() }() + bytes := make([]byte, 4096) for { - _, err = sch.Read(make([]byte, 4096)) + _, err = sch.Read(bytes) if err != nil { require.ErrorIs(t, err, peer.ErrClosed) break @@ -165,7 +161,6 @@ func TestConn(t *testing.T) { t.Run("NetConn", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) srv, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer srv.Close() @@ -218,7 +213,6 @@ func TestConn(t *testing.T) { t.Run("CloseBeforeNegotiate", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) err := client.Close() require.NoError(t, err) err = server.Close() @@ -238,7 +232,6 @@ func TestConn(t *testing.T) { t.Run("PingConcurrent", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) - exchange(client, server) var wg sync.WaitGroup wg.Add(2) go func() { @@ -253,19 +246,6 @@ func TestConn(t *testing.T) { }() wg.Wait() }) - - t.Run("NegotiateOutOfOrder", func(t *testing.T) { - t.Parallel() - client, server, _ := createPair(t) - server.SetRemoteSessionDescription(<-client.LocalSessionDescription()) - err := client.AddRemoteCandidate(<-server.LocalCandidate()) - require.NoError(t, err) - client.SetRemoteSessionDescription(<-server.LocalSessionDescription()) - err = server.AddRemoteCandidate(<-client.LocalCandidate()) - require.NoError(t, err) - _, err = client.Ping() - require.NoError(t, err) - }) } func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.Router) { @@ -324,18 +304,12 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R _ = wan.Stop() }) - return channel1, channel2, wan -} - -func exchange(client *peer.Conn, server *peer.Conn) { go func() { for { select { - case c := <-server.LocalCandidate(): - _ = client.AddRemoteCandidate(c) - case c := <-server.LocalSessionDescription(): - client.SetRemoteSessionDescription(c) - case <-server.Closed(): + case c := <-channel2.LocalNegotiation(): + _ = channel1.AddRemoteNegotiation(c) + case <-channel2.Closed(): return } } @@ -344,13 +318,13 @@ func exchange(client *peer.Conn, server *peer.Conn) { go func() { for { select { - case c := <-client.LocalCandidate(): - _ = server.AddRemoteCandidate(c) - case c := <-client.LocalSessionDescription(): - server.SetRemoteSessionDescription(c) - case <-client.Closed(): + case c := <-channel1.LocalNegotiation(): + _ = channel2.AddRemoteNegotiation(c) + case <-channel1.Closed(): return } } }() + + return channel1, channel2, wan } diff --git a/peerbroker/dial.go b/peerbroker/dial.go index 9af9c663bec6f..8b3363b69f8ba 100644 --- a/peerbroker/dial.go +++ b/peerbroker/dial.go @@ -53,29 +53,16 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we select { case <-peerConn.Closed(): return - case sessionDescription := <-peerConn.LocalSessionDescription(): + case localNegotiation := <-peerConn.LocalNegotiation(): err = stream.Send(&proto.NegotiateConnection_ClientToServer{ - Message: &proto.NegotiateConnection_ClientToServer_Offer{ - Offer: &proto.WebRTCSessionDescription{ - SdpType: int32(sessionDescription.Type), - Sdp: sessionDescription.SDP, - }, + Message: &proto.NegotiateConnection_ClientToServer_Negotiation{ + Negotiation: convertLocalNegotiation(localNegotiation), }, }) if err != nil { _ = peerConn.CloseWithError(xerrors.Errorf("send local session description: %w", err)) return } - case iceCandidate := <-peerConn.LocalCandidate(): - err = stream.Send(&proto.NegotiateConnection_ClientToServer{ - Message: &proto.NegotiateConnection_ClientToServer_IceCandidate{ - IceCandidate: iceCandidate.Candidate, - }, - }) - if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("send local candidate: %w", err)) - return - } } } }() @@ -87,27 +74,56 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we _ = peerConn.CloseWithError(xerrors.Errorf("recv: %w", err)) return } - - switch { - case serverToClientMessage.GetAnswer() != nil: - peerConn.SetRemoteSessionDescription(webrtc.SessionDescription{ - Type: webrtc.SDPType(serverToClientMessage.GetAnswer().SdpType), - SDP: serverToClientMessage.GetAnswer().Sdp, - }) - case serverToClientMessage.GetIceCandidate() != "": - err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ - Candidate: serverToClientMessage.GetIceCandidate(), - }) - if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) - return - } - default: + if serverToClientMessage.GetNegotiation() == nil { _ = peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(serverToClientMessage).String())) return } + + err = peerConn.AddRemoteNegotiation(convertProtoNegotiation(serverToClientMessage.Negotiation)) + if err != nil { + _ = peerConn.CloseWithError(xerrors.Errorf("add remote negotiation: %w", err)) + return + } } }() return peerConn, nil } + +func convertLocalNegotiation(localNegotiation peer.Negotiation) *proto.Negotiation { + protoNegotation := &proto.Negotiation{} + if localNegotiation.SessionDescription != nil { + protoNegotation.SessionDescription = &proto.WebRTCSessionDescription{ + SdpType: int32(localNegotiation.SessionDescription.Type), + Sdp: localNegotiation.SessionDescription.SDP, + } + } + if len(localNegotiation.ICECandidates) > 0 { + iceCandidates := make([]string, 0, len(localNegotiation.ICECandidates)) + for _, iceCandidate := range localNegotiation.ICECandidates { + iceCandidates = append(iceCandidates, iceCandidate.Candidate) + } + protoNegotation.IceCandidates = iceCandidates + } + return protoNegotation +} + +func convertProtoNegotiation(protoNegotiation *proto.Negotiation) peer.Negotiation { + localNegotiation := peer.Negotiation{} + if protoNegotiation.SessionDescription != nil { + localNegotiation.SessionDescription = &webrtc.SessionDescription{ + Type: webrtc.SDPType(protoNegotiation.SessionDescription.SdpType), + SDP: protoNegotiation.SessionDescription.Sdp, + } + } + if len(protoNegotiation.IceCandidates) > 0 { + candidates := make([]webrtc.ICECandidateInit, 0, len(protoNegotiation.IceCandidates)) + for _, iceCandidate := range protoNegotiation.IceCandidates { + candidates = append(candidates, webrtc.ICECandidateInit{ + Candidate: iceCandidate, + }) + } + localNegotiation.ICECandidates = candidates + } + return localNegotiation +} diff --git a/peerbroker/listen.go b/peerbroker/listen.go index db218a209fc8a..51c64cd2c83cd 100644 --- a/peerbroker/listen.go +++ b/peerbroker/listen.go @@ -120,27 +120,12 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego select { case <-peerConn.Closed(): return - case sessionDescription := <-peerConn.LocalSessionDescription(): + case localNegotiation := <-peerConn.LocalNegotiation(): err = stream.Send(&proto.NegotiateConnection_ServerToClient{ - Message: &proto.NegotiateConnection_ServerToClient_Answer{ - Answer: &proto.WebRTCSessionDescription{ - SdpType: int32(sessionDescription.Type), - Sdp: sessionDescription.SDP, - }, - }, + Negotiation: convertLocalNegotiation(localNegotiation), }) if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("send local session description: %w", err)) - return - } - case iceCandidate := <-peerConn.LocalCandidate(): - err = stream.Send(&proto.NegotiateConnection_ServerToClient{ - Message: &proto.NegotiateConnection_ServerToClient_IceCandidate{ - IceCandidate: iceCandidate.Candidate, - }, - }) - if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("send local candidate: %w", err)) + _ = peerConn.CloseWithError(xerrors.Errorf("send local negotiation: %w", err)) return } } @@ -156,11 +141,11 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego } switch { - case clientToServerMessage.GetOffer() != nil: - peerConn.SetRemoteSessionDescription(webrtc.SessionDescription{ - Type: webrtc.SDPType(clientToServerMessage.GetOffer().SdpType), - SDP: clientToServerMessage.GetOffer().Sdp, - }) + case clientToServerMessage.GetNegotiation() != nil: + err = peerConn.AddRemoteNegotiation(convertProtoNegotiation(clientToServerMessage.GetNegotiation())) + if err != nil { + return peerConn.CloseWithError(xerrors.Errorf("add remote negotiation: %w", err)) + } case clientToServerMessage.GetServers() != nil: // Convert protobuf ICE servers to the WebRTC type. iceServers := make([]webrtc.ICEServer, 0, len(clientToServerMessage.GetServers().Servers)) @@ -178,13 +163,6 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego if err != nil { return peerConn.CloseWithError(xerrors.Errorf("set ice configuration: %w", err)) } - case clientToServerMessage.GetIceCandidate() != "": - err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ - Candidate: clientToServerMessage.GetIceCandidate(), - }) - if err != nil { - return peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) - } default: return peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(clientToServerMessage).String())) } diff --git a/peerbroker/proto/peerbroker.pb.go b/peerbroker/proto/peerbroker.pb.go index dc89ea8cb57eb..825f747a704f8 100644 --- a/peerbroker/proto/peerbroker.pb.go +++ b/peerbroker/proto/peerbroker.pb.go @@ -193,6 +193,61 @@ func (x *WebRTCICEServers) GetServers() []*WebRTCICEServer { return nil } +type Negotiation struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SessionDescription *WebRTCSessionDescription `protobuf:"bytes,1,opt,name=session_description,json=sessionDescription,proto3" json:"session_description,omitempty"` + IceCandidates []string `protobuf:"bytes,2,rep,name=ice_candidates,json=iceCandidates,proto3" json:"ice_candidates,omitempty"` +} + +func (x *Negotiation) Reset() { + *x = Negotiation{} + if protoimpl.UnsafeEnabled { + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Negotiation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Negotiation) ProtoMessage() {} + +func (x *Negotiation) ProtoReflect() protoreflect.Message { + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Negotiation.ProtoReflect.Descriptor instead. +func (*Negotiation) Descriptor() ([]byte, []int) { + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3} +} + +func (x *Negotiation) GetSessionDescription() *WebRTCSessionDescription { + if x != nil { + return x.SessionDescription + } + return nil +} + +func (x *Negotiation) GetIceCandidates() []string { + if x != nil { + return x.IceCandidates + } + return nil +} + type NegotiateConnection struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -202,7 +257,7 @@ type NegotiateConnection struct { func (x *NegotiateConnection) Reset() { *x = NegotiateConnection{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -215,7 +270,7 @@ func (x *NegotiateConnection) String() string { func (*NegotiateConnection) ProtoMessage() {} func (x *NegotiateConnection) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -228,7 +283,7 @@ func (x *NegotiateConnection) ProtoReflect() protoreflect.Message { // Deprecated: Use NegotiateConnection.ProtoReflect.Descriptor instead. func (*NegotiateConnection) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4} } type NegotiateConnection_ClientToServer struct { @@ -238,15 +293,14 @@ type NegotiateConnection_ClientToServer struct { // Types that are assignable to Message: // *NegotiateConnection_ClientToServer_Servers - // *NegotiateConnection_ClientToServer_Offer - // *NegotiateConnection_ClientToServer_IceCandidate + // *NegotiateConnection_ClientToServer_Negotiation Message isNegotiateConnection_ClientToServer_Message `protobuf_oneof:"message"` } func (x *NegotiateConnection_ClientToServer) Reset() { *x = NegotiateConnection_ClientToServer{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -259,7 +313,7 @@ func (x *NegotiateConnection_ClientToServer) String() string { func (*NegotiateConnection_ClientToServer) ProtoMessage() {} func (x *NegotiateConnection_ClientToServer) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -272,7 +326,7 @@ func (x *NegotiateConnection_ClientToServer) ProtoReflect() protoreflect.Message // Deprecated: Use NegotiateConnection_ClientToServer.ProtoReflect.Descriptor instead. func (*NegotiateConnection_ClientToServer) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3, 0} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4, 0} } func (m *NegotiateConnection_ClientToServer) GetMessage() isNegotiateConnection_ClientToServer_Message { @@ -289,20 +343,13 @@ func (x *NegotiateConnection_ClientToServer) GetServers() *WebRTCICEServers { return nil } -func (x *NegotiateConnection_ClientToServer) GetOffer() *WebRTCSessionDescription { - if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_Offer); ok { - return x.Offer +func (x *NegotiateConnection_ClientToServer) GetNegotiation() *Negotiation { + if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_Negotiation); ok { + return x.Negotiation } return nil } -func (x *NegotiateConnection_ClientToServer) GetIceCandidate() string { - if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_IceCandidate); ok { - return x.IceCandidate - } - return "" -} - type isNegotiateConnection_ClientToServer_Message interface { isNegotiateConnection_ClientToServer_Message() } @@ -311,19 +358,13 @@ type NegotiateConnection_ClientToServer_Servers struct { Servers *WebRTCICEServers `protobuf:"bytes,1,opt,name=servers,proto3,oneof"` } -type NegotiateConnection_ClientToServer_Offer struct { - Offer *WebRTCSessionDescription `protobuf:"bytes,2,opt,name=offer,proto3,oneof"` -} - -type NegotiateConnection_ClientToServer_IceCandidate struct { - IceCandidate string `protobuf:"bytes,3,opt,name=ice_candidate,json=iceCandidate,proto3,oneof"` +type NegotiateConnection_ClientToServer_Negotiation struct { + Negotiation *Negotiation `protobuf:"bytes,2,opt,name=negotiation,proto3,oneof"` } func (*NegotiateConnection_ClientToServer_Servers) isNegotiateConnection_ClientToServer_Message() {} -func (*NegotiateConnection_ClientToServer_Offer) isNegotiateConnection_ClientToServer_Message() {} - -func (*NegotiateConnection_ClientToServer_IceCandidate) isNegotiateConnection_ClientToServer_Message() { +func (*NegotiateConnection_ClientToServer_Negotiation) isNegotiateConnection_ClientToServer_Message() { } type NegotiateConnection_ServerToClient struct { @@ -331,16 +372,13 @@ type NegotiateConnection_ServerToClient struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Types that are assignable to Message: - // *NegotiateConnection_ServerToClient_Answer - // *NegotiateConnection_ServerToClient_IceCandidate - Message isNegotiateConnection_ServerToClient_Message `protobuf_oneof:"message"` + Negotiation *Negotiation `protobuf:"bytes,1,opt,name=negotiation,proto3" json:"negotiation,omitempty"` } func (x *NegotiateConnection_ServerToClient) Reset() { *x = NegotiateConnection_ServerToClient{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -353,7 +391,7 @@ func (x *NegotiateConnection_ServerToClient) String() string { func (*NegotiateConnection_ServerToClient) ProtoMessage() {} func (x *NegotiateConnection_ServerToClient) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -366,47 +404,16 @@ func (x *NegotiateConnection_ServerToClient) ProtoReflect() protoreflect.Message // Deprecated: Use NegotiateConnection_ServerToClient.ProtoReflect.Descriptor instead. func (*NegotiateConnection_ServerToClient) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3, 1} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4, 1} } -func (m *NegotiateConnection_ServerToClient) GetMessage() isNegotiateConnection_ServerToClient_Message { - if m != nil { - return m.Message - } - return nil -} - -func (x *NegotiateConnection_ServerToClient) GetAnswer() *WebRTCSessionDescription { - if x, ok := x.GetMessage().(*NegotiateConnection_ServerToClient_Answer); ok { - return x.Answer +func (x *NegotiateConnection_ServerToClient) GetNegotiation() *Negotiation { + if x != nil { + return x.Negotiation } return nil } -func (x *NegotiateConnection_ServerToClient) GetIceCandidate() string { - if x, ok := x.GetMessage().(*NegotiateConnection_ServerToClient_IceCandidate); ok { - return x.IceCandidate - } - return "" -} - -type isNegotiateConnection_ServerToClient_Message interface { - isNegotiateConnection_ServerToClient_Message() -} - -type NegotiateConnection_ServerToClient_Answer struct { - Answer *WebRTCSessionDescription `protobuf:"bytes,1,opt,name=answer,proto3,oneof"` -} - -type NegotiateConnection_ServerToClient_IceCandidate struct { - IceCandidate string `protobuf:"bytes,2,opt,name=ice_candidate,json=iceCandidate,proto3,oneof"` -} - -func (*NegotiateConnection_ServerToClient_Answer) isNegotiateConnection_ServerToClient_Message() {} - -func (*NegotiateConnection_ServerToClient_IceCandidate) isNegotiateConnection_ServerToClient_Message() { -} - var File_peerbroker_proto_peerbroker_proto protoreflect.FileDescriptor var file_peerbroker_proto_peerbroker_proto_rawDesc = []byte{ @@ -431,40 +438,43 @@ var file_peerbroker_proto_peerbroker_proto_rawDesc = []byte{ 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x49, 0x43, 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x22, 0xd7, 0x02, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xba, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x07, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, - 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, - 0x49, 0x43, 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x48, 0x00, 0x52, 0x07, 0x73, 0x65, - 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x3c, 0x0a, 0x05, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, - 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x66, - 0x66, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0d, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x61, 0x6e, 0x64, 0x69, - 0x64, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x63, - 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x82, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x06, 0x61, 0x6e, 0x73, 0x77, - 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, - 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x53, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, - 0x52, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0d, 0x69, 0x63, 0x65, 0x5f, - 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x0c, 0x69, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x42, - 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x87, 0x01, 0x0a, 0x0a, 0x50, - 0x65, 0x65, 0x72, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x79, 0x0a, 0x13, 0x4e, 0x65, 0x67, - 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, - 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, - 0x1a, 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, - 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, - 0x28, 0x01, 0x30, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, - 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0x8b, 0x01, 0x0a, 0x0b, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x55, 0x0a, 0x13, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, + 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, + 0x43, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x12, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x63, 0x65, 0x5f, 0x63, + 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x0d, 0x69, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xf7, + 0x01, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x92, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x07, 0x73, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x65, 0x65, + 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x49, 0x43, + 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x48, 0x00, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x12, 0x3b, 0x0a, 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, + 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x48, 0x00, 0x52, 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x4b, 0x0a, 0x0e, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x39, 0x0a, + 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, + 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x6e, 0x65, 0x67, + 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x87, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, + 0x72, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x79, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, + 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, + 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x1a, 0x2e, + 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, + 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x28, 0x01, + 0x30, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, 0x65, 0x65, + 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -479,27 +489,29 @@ func file_peerbroker_proto_peerbroker_proto_rawDescGZIP() []byte { return file_peerbroker_proto_peerbroker_proto_rawDescData } -var file_peerbroker_proto_peerbroker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_peerbroker_proto_peerbroker_proto_msgTypes = make([]protoimpl.MessageInfo, 7) var file_peerbroker_proto_peerbroker_proto_goTypes = []interface{}{ (*WebRTCSessionDescription)(nil), // 0: peerbroker.WebRTCSessionDescription (*WebRTCICEServer)(nil), // 1: peerbroker.WebRTCICEServer (*WebRTCICEServers)(nil), // 2: peerbroker.WebRTCICEServers - (*NegotiateConnection)(nil), // 3: peerbroker.NegotiateConnection - (*NegotiateConnection_ClientToServer)(nil), // 4: peerbroker.NegotiateConnection.ClientToServer - (*NegotiateConnection_ServerToClient)(nil), // 5: peerbroker.NegotiateConnection.ServerToClient + (*Negotiation)(nil), // 3: peerbroker.Negotiation + (*NegotiateConnection)(nil), // 4: peerbroker.NegotiateConnection + (*NegotiateConnection_ClientToServer)(nil), // 5: peerbroker.NegotiateConnection.ClientToServer + (*NegotiateConnection_ServerToClient)(nil), // 6: peerbroker.NegotiateConnection.ServerToClient } var file_peerbroker_proto_peerbroker_proto_depIdxs = []int32{ 1, // 0: peerbroker.WebRTCICEServers.servers:type_name -> peerbroker.WebRTCICEServer - 2, // 1: peerbroker.NegotiateConnection.ClientToServer.servers:type_name -> peerbroker.WebRTCICEServers - 0, // 2: peerbroker.NegotiateConnection.ClientToServer.offer:type_name -> peerbroker.WebRTCSessionDescription - 0, // 3: peerbroker.NegotiateConnection.ServerToClient.answer:type_name -> peerbroker.WebRTCSessionDescription - 4, // 4: peerbroker.PeerBroker.NegotiateConnection:input_type -> peerbroker.NegotiateConnection.ClientToServer - 5, // 5: peerbroker.PeerBroker.NegotiateConnection:output_type -> peerbroker.NegotiateConnection.ServerToClient - 5, // [5:6] is the sub-list for method output_type - 4, // [4:5] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 0, // 1: peerbroker.Negotiation.session_description:type_name -> peerbroker.WebRTCSessionDescription + 2, // 2: peerbroker.NegotiateConnection.ClientToServer.servers:type_name -> peerbroker.WebRTCICEServers + 3, // 3: peerbroker.NegotiateConnection.ClientToServer.negotiation:type_name -> peerbroker.Negotiation + 3, // 4: peerbroker.NegotiateConnection.ServerToClient.negotiation:type_name -> peerbroker.Negotiation + 5, // 5: peerbroker.PeerBroker.NegotiateConnection:input_type -> peerbroker.NegotiateConnection.ClientToServer + 6, // 6: peerbroker.PeerBroker.NegotiateConnection:output_type -> peerbroker.NegotiateConnection.ServerToClient + 6, // [6:7] is the sub-list for method output_type + 5, // [5:6] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_peerbroker_proto_peerbroker_proto_init() } @@ -545,7 +557,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } file_peerbroker_proto_peerbroker_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NegotiateConnection); i { + switch v := v.(*Negotiation); i { case 0: return &v.state case 1: @@ -557,7 +569,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } file_peerbroker_proto_peerbroker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*NegotiateConnection_ClientToServer); i { + switch v := v.(*NegotiateConnection); i { case 0: return &v.state case 1: @@ -569,6 +581,18 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } file_peerbroker_proto_peerbroker_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NegotiateConnection_ClientToServer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_peerbroker_proto_peerbroker_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NegotiateConnection_ServerToClient); i { case 0: return &v.state @@ -581,14 +605,9 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } } - file_peerbroker_proto_peerbroker_proto_msgTypes[4].OneofWrappers = []interface{}{ - (*NegotiateConnection_ClientToServer_Servers)(nil), - (*NegotiateConnection_ClientToServer_Offer)(nil), - (*NegotiateConnection_ClientToServer_IceCandidate)(nil), - } file_peerbroker_proto_peerbroker_proto_msgTypes[5].OneofWrappers = []interface{}{ - (*NegotiateConnection_ServerToClient_Answer)(nil), - (*NegotiateConnection_ServerToClient_IceCandidate)(nil), + (*NegotiateConnection_ClientToServer_Servers)(nil), + (*NegotiateConnection_ClientToServer_Negotiation)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -596,7 +615,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_peerbroker_proto_peerbroker_proto_rawDesc, NumEnums: 0, - NumMessages: 6, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/peerbroker/proto/peerbroker.proto b/peerbroker/proto/peerbroker.proto index c4f660231c5a4..e35d40f828086 100644 --- a/peerbroker/proto/peerbroker.proto +++ b/peerbroker/proto/peerbroker.proto @@ -20,19 +20,20 @@ message WebRTCICEServers { repeated WebRTCICEServer servers = 1; } +message Negotiation { + WebRTCSessionDescription session_description = 1; + repeated string ice_candidates = 2; +} + message NegotiateConnection { message ClientToServer { oneof message { WebRTCICEServers servers = 1; - WebRTCSessionDescription offer = 2; - string ice_candidate = 3; + Negotiation negotiation = 2; } } message ServerToClient { - oneof message { - WebRTCSessionDescription answer = 1; - string ice_candidate = 2; - } + Negotiation negotiation = 1; } } From 700afe4c3d0c629fff98793a7bdf0cc848a02e3e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 17:39:30 +0000 Subject: [PATCH 13/22] Improve test times by pre-installing Terraform --- .github/workflows/coder.yaml | 4 ++++ coderd/userpassword/userpassword_test.go | 4 ++++ go.mod | 2 -- go.sum | 3 --- provisioner/terraform/provision_test.go | 12 ------------ 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 7e0b890edad9b..99549972fd807 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -148,6 +148,10 @@ jobs: - run: go install gotest.tools/gotestsum@latest + - uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 1.1.2 + - name: Test with Mock Database run: gotestsum --jsonfile="gotests.json" --packages="./..." -- diff --git a/coderd/userpassword/userpassword_test.go b/coderd/userpassword/userpassword_test.go index 0546163d24902..a07d2c07edd5c 100644 --- a/coderd/userpassword/userpassword_test.go +++ b/coderd/userpassword/userpassword_test.go @@ -1,3 +1,7 @@ +// This test runs slowly on MacOS instance, and really +// only needs to run on Linux anyways. +//go:build linux + package userpassword_test import ( diff --git a/go.mod b/go.mod index 7505a34a7167c..8a204cc59a88e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/golang-migrate/migrate/v4 v4.15.1 github.com/google/uuid v1.3.0 github.com/hashicorp/go-version v1.4.0 - github.com/hashicorp/hc-install v0.3.1 github.com/hashicorp/terraform-config-inspect v0.0.0-20211115214459-90acf1ca460f github.com/hashicorp/terraform-exec v0.15.0 github.com/justinas/nosurf v1.1.1 @@ -64,7 +63,6 @@ require ( github.com/google/go-cmp v0.5.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl/v2 v2.11.1 // indirect diff --git a/go.sum b/go.sum index 564c6aaae7be9..85350bdbd968b 100644 --- a/go.sum +++ b/go.sum @@ -693,8 +693,6 @@ github.com/hashicorp/go-version v1.4.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hc-install v0.3.1 h1:VIjllE6KyAI1A244G8kTaHXy+TL5/XYzvrtFi8po/Yk= -github.com/hashicorp/hc-install v0.3.1/go.mod h1:3LCdWcCDS1gaHC9mhHCGbkYfoY6vdsKohGjugbZdZak= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -1309,7 +1307,6 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/provisioner/terraform/provision_test.go b/provisioner/terraform/provision_test.go index b596c85d0bf15..71b7864bbe5af 100644 --- a/provisioner/terraform/provision_test.go +++ b/provisioner/terraform/provision_test.go @@ -9,27 +9,16 @@ import ( "path/filepath" "testing" - "github.com/hashicorp/go-version" "github.com/stretchr/testify/require" "storj.io/drpc/drpcconn" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk/proto" - - "github.com/hashicorp/hc-install/product" - "github.com/hashicorp/hc-install/releases" ) func TestProvision(t *testing.T) { t.Parallel() - installer := &releases.ExactVersion{ - Product: product.Terraform, - Version: version.Must(version.NewVersion("1.1.2")), - } - execPath, err := installer.Install(context.Background()) - require.NoError(t, err) - client, server := provisionersdk.TransportPipe() ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(func() { @@ -42,7 +31,6 @@ func TestProvision(t *testing.T) { ServeOptions: &provisionersdk.ServeOptions{ Transport: server, }, - BinaryPath: execPath, }) require.NoError(t, err) }() From 7eccb81c6304b36168b3b78979bcbab8e73cc6d5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 17:51:06 +0000 Subject: [PATCH 14/22] Lock remote session description being applied --- .github/workflows/coder.yaml | 2 ++ peer/conn.go | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 99549972fd807..d40fd3ba5eba0 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -149,8 +149,10 @@ jobs: - run: go install gotest.tools/gotestsum@latest - uses: hashicorp/setup-terraform@v1 + if: runner.os == 'Linux' with: terraform_version: 1.1.2 + terraform_wrapper: false - name: Test with Mock Database run: diff --git a/peer/conn.go b/peer/conn.go index 8638df549e616..58f19a2287239 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -79,7 +79,7 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp // This channel needs to be bufferred otherwise slow consumers // of this will cause a connection failure. localNegotiator: make(chan Negotiation, 8), - remoteSessionDescription: make(chan webrtc.SessionDescription, 1), + remoteSessionDescription: make(chan webrtc.SessionDescription), pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), } if client { @@ -124,8 +124,9 @@ type Conn struct { dcFailedListeners atomic.Uint32 dcClosedWaitGroup sync.WaitGroup - localNegotiator chan Negotiation - remoteSessionDescription chan webrtc.SessionDescription + localNegotiator chan Negotiation + remoteSessionDescription chan webrtc.SessionDescription + remoteSessionDescriptionMutex sync.Mutex pendingCandidatesToSend []webrtc.ICECandidateInit pendingCandidatesToSendMutex sync.Mutex @@ -329,9 +330,6 @@ func (c *Conn) negotiate() { return } c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) - if c.isClosed() { - return - } c.closeMutex.Lock() err = c.rtc.SetLocalDescription(offer) c.closeMutex.Unlock() @@ -353,6 +351,9 @@ func (c *Conn) negotiate() { case sessionDescription = <-c.remoteSessionDescription: } + // This prevents candidates from being added while + // the remote description is being set. + c.remoteSessionDescriptionMutex.Lock() c.opts.Logger.Debug(context.Background(), "setting remote description") c.closeMutex.Lock() err := c.rtc.SetRemoteDescription(sessionDescription) @@ -361,17 +362,17 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) return } + c.remoteSessionDescriptionMutex.Unlock() if !c.offerrer { + c.closeMutex.Lock() answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("create answer: %w", err)) return } c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) - if c.isClosed() { - return - } c.closeMutex.Lock() err = c.rtc.SetLocalDescription(answer) c.closeMutex.Unlock() @@ -418,10 +419,10 @@ func (c *Conn) AddRemoteNegotiation(negotiation Negotiation) error { } if len(negotiation.ICECandidates) > 0 { + c.remoteSessionDescriptionMutex.Lock() + defer c.remoteSessionDescriptionMutex.Unlock() c.opts.Logger.Debug(context.Background(), "adding remote negotiation with ice candidates", slog.F("count", len(negotiation.ICECandidates))) - c.closeMutex.Lock() - defer c.closeMutex.Unlock() for _, iceCandidate := range negotiation.ICECandidates { err := c.rtc.AddICECandidate(iceCandidate) if err != nil { From 679e4aa798ea168a04aa68a9355a2bc68b1d3e53 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 18:07:46 +0000 Subject: [PATCH 15/22] Organize conn --- peer/conn.go | 139 ++++++++++++++++++++------------------------------- 1 file changed, 55 insertions(+), 84 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 58f19a2287239..2dc0d61a9f5af 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -3,10 +3,8 @@ package peer import ( "bytes" "context" - "fmt" "crypto/rand" - "crypto/sha256" "io" "sync" "time" @@ -124,7 +122,8 @@ type Conn struct { dcFailedListeners atomic.Uint32 dcClosedWaitGroup sync.WaitGroup - localNegotiator chan Negotiation + localNegotiator chan Negotiation + remoteSessionDescription chan webrtc.SessionDescription remoteSessionDescriptionMutex sync.Mutex @@ -179,16 +178,12 @@ func (c *Conn) init() error { c.pendingCandidatesToSendMutex.Lock() defer c.pendingCandidatesToSendMutex.Unlock() json := iceCandidate.ToJSON() - fields := []slog.Field{ - slog.F("hash", c.hashCandidate(json)), - slog.F("length", len(json.Candidate)), - } if !c.pendingCandidatesFlushed { c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, json) - c.opts.Logger.Debug(context.Background(), "buffering local candidate to send", fields...) + c.opts.Logger.Debug(context.Background(), "buffering local candidate") return } - c.opts.Logger.Debug(context.Background(), "sending local candidate directly", fields...) + c.opts.Logger.Debug(context.Background(), "sending local candidate") select { case <-c.closed: break @@ -250,93 +245,24 @@ func (c *Conn) init() error { return nil } -func (c *Conn) pingChannel() (*Channel, error) { - c.pingOnce.Do(func() { - c.pingChan, c.pingError = c.dialChannel(context.Background(), "ping", &ChannelOptions{ - ID: c.pingChannelID, - Negotiated: true, - OpenOnDisconnect: true, - }) - if c.pingError != nil { - return - } - }) - return c.pingChan, c.pingError -} - -func (c *Conn) pingEchoChannel() (*Channel, error) { - c.pingEchoOnce.Do(func() { - c.pingEchoChan, c.pingEchoError = c.dialChannel(context.Background(), "echo", &ChannelOptions{ - ID: c.pingEchoChannelID, - Negotiated: true, - OpenOnDisconnect: true, - }) - if c.pingEchoError != nil { - return - } - go func() { - for { - data := make([]byte, pingDataLength) - bytesRead, err := c.pingEchoChan.Read(data) - if err != nil { - if c.isClosed() { - return - } - _ = c.CloseWithError(xerrors.Errorf("read ping echo channel: %w", err)) - return - } - _, err = c.pingEchoChan.Write(data[:bytesRead]) - if err != nil { - _ = c.CloseWithError(xerrors.Errorf("write ping echo channel: %w", err)) - return - } - } - }() - }) - return c.pingEchoChan, c.pingEchoError -} - -// Computes a hash of the ICE candidate. Used for debug logging. -func (*Conn) hashCandidate(iceCandidate webrtc.ICECandidateInit) string { - hash := sha256.Sum224([]byte(iceCandidate.Candidate)) - return fmt.Sprintf("%x", hash[:8]) -} - // Negotiate exchanges ICECandidate pairs over the exposed channels. // The diagram below shows the expected handshake. pion/webrtc v3 // uses trickle ICE by default. See: https://webrtchacks.com/trickle-ice/ -// ┌────────┐ ┌────────┐ -// │offerrer│ │answerer│ -// │(client)│ │(server)│ -// └─┬────┬─┘ └─┬──────┘ -// │ │ offer │ -// ┌──────────▼┐ ├──────────────►├──►┌───────────┐ -// │STUN Server│ │ │ │STUN Server│ -// │(optional) ├──►│ candidate │◄──┤(optional) │ -// └───────────┘ │ (async) │ └───────────┘ -// │◄─────────────►│ -// │ │ -// │ answer │ -// └◄──────────────┘ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") if c.offerrer { - c.closeMutex.Lock() offer, err := c.rtc.CreateOffer(&webrtc.OfferOptions{}) - c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("create offer: %w", err)) return } - c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) - c.closeMutex.Lock() err = c.rtc.SetLocalDescription(offer) - c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return } + c.opts.Logger.Debug(context.Background(), "sending offer") select { case <-c.closed: return @@ -355,9 +281,7 @@ func (c *Conn) negotiate() { // the remote description is being set. c.remoteSessionDescriptionMutex.Lock() c.opts.Logger.Debug(context.Background(), "setting remote description") - c.closeMutex.Lock() err := c.rtc.SetRemoteDescription(sessionDescription) - c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) return @@ -365,15 +289,13 @@ func (c *Conn) negotiate() { c.remoteSessionDescriptionMutex.Unlock() if !c.offerrer { - c.closeMutex.Lock() answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) - c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("create answer: %w", err)) return } - c.opts.Logger.Debug(context.Background(), "setting local description", slog.F("closed", c.isClosed())) c.closeMutex.Lock() + // pion doesn't handle a close properly if it occurs during this function. err = c.rtc.SetLocalDescription(answer) c.closeMutex.Unlock() if err != nil { @@ -404,10 +326,13 @@ func (c *Conn) negotiate() { c.pendingCandidatesFlushed = true } +// LocalNegotiation returns a channel for connection negotiation. +// This should be piped to another peer connection. func (c *Conn) LocalNegotiation() <-chan Negotiation { return c.localNegotiator } +// AddRemoteNegotiation accepts a negotiation message for handshaking a connection. func (c *Conn) AddRemoteNegotiation(negotiation Negotiation) error { if negotiation.SessionDescription != nil { c.opts.Logger.Debug(context.Background(), "adding remote negotiation with session description") @@ -434,6 +359,52 @@ func (c *Conn) AddRemoteNegotiation(negotiation Negotiation) error { return nil } +func (c *Conn) pingChannel() (*Channel, error) { + c.pingOnce.Do(func() { + c.pingChan, c.pingError = c.dialChannel(context.Background(), "ping", &ChannelOptions{ + ID: c.pingChannelID, + Negotiated: true, + OpenOnDisconnect: true, + }) + if c.pingError != nil { + return + } + }) + return c.pingChan, c.pingError +} + +func (c *Conn) pingEchoChannel() (*Channel, error) { + c.pingEchoOnce.Do(func() { + c.pingEchoChan, c.pingEchoError = c.dialChannel(context.Background(), "echo", &ChannelOptions{ + ID: c.pingEchoChannelID, + Negotiated: true, + OpenOnDisconnect: true, + }) + if c.pingEchoError != nil { + return + } + go func() { + for { + data := make([]byte, pingDataLength) + bytesRead, err := c.pingEchoChan.Read(data) + if err != nil { + if c.isClosed() { + return + } + _ = c.CloseWithError(xerrors.Errorf("read ping echo channel: %w", err)) + return + } + _, err = c.pingEchoChan.Write(data[:bytesRead]) + if err != nil { + _ = c.CloseWithError(xerrors.Errorf("write ping echo channel: %w", err)) + return + } + } + }() + }) + return c.pingEchoChan, c.pingEchoError +} + // SetConfiguration applies options to the WebRTC connection. // Generally used for updating transport options, like ICE servers. func (c *Conn) SetConfiguration(configuration webrtc.Configuration) error { From f20ce9ddaac1f433520a116df406eedd11880fca Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 18:24:58 +0000 Subject: [PATCH 16/22] Revert to multi-channel setup --- peer/conn.go | 102 ++++++----- peer/conn_test.go | 12 +- peerbroker/dial.go | 80 ++++----- peerbroker/listen.go | 38 ++++- peerbroker/proto/peerbroker.pb.go | 271 ++++++++++++++---------------- peerbroker/proto/peerbroker.proto | 15 +- 6 files changed, 251 insertions(+), 267 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 2dc0d61a9f5af..045c5b9298e9a 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -63,22 +63,21 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp return nil, xerrors.Errorf("create peer connection: %w", err) } conn := &Conn{ - pingChannelID: 1, - pingEchoChannelID: 2, - opts: opts, - rtc: rtc, - offerrer: client, - closed: make(chan struct{}), - closedRTC: make(chan struct{}), - closedICE: make(chan struct{}), - dcOpenChannel: make(chan *webrtc.DataChannel), - dcDisconnectChannel: make(chan struct{}), - dcFailedChannel: make(chan struct{}), - // This channel needs to be bufferred otherwise slow consumers - // of this will cause a connection failure. - localNegotiator: make(chan Negotiation, 8), - remoteSessionDescription: make(chan webrtc.SessionDescription), - pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), + pingChannelID: 1, + pingEchoChannelID: 2, + opts: opts, + rtc: rtc, + offerrer: client, + closed: make(chan struct{}), + closedRTC: make(chan struct{}), + closedICE: make(chan struct{}), + dcOpenChannel: make(chan *webrtc.DataChannel), + dcDisconnectChannel: make(chan struct{}), + dcFailedChannel: make(chan struct{}), + localCandidateChannel: make(chan webrtc.ICECandidateInit), + localSessionDescriptionChannel: make(chan webrtc.SessionDescription), + remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription), + pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), } if client { // If we're the client, we want to flip the echo and @@ -122,10 +121,10 @@ type Conn struct { dcFailedListeners atomic.Uint32 dcClosedWaitGroup sync.WaitGroup - localNegotiator chan Negotiation - - remoteSessionDescription chan webrtc.SessionDescription - remoteSessionDescriptionMutex sync.Mutex + localCandidateChannel chan webrtc.ICECandidateInit + localSessionDescriptionChannel chan webrtc.SessionDescription + remoteSessionDescriptionChannel chan webrtc.SessionDescription + remoteSessionDescriptionMutex sync.Mutex pendingCandidatesToSend []webrtc.ICECandidateInit pendingCandidatesToSendMutex sync.Mutex @@ -177,9 +176,8 @@ func (c *Conn) init() error { } c.pendingCandidatesToSendMutex.Lock() defer c.pendingCandidatesToSendMutex.Unlock() - json := iceCandidate.ToJSON() if !c.pendingCandidatesFlushed { - c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, json) + c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) c.opts.Logger.Debug(context.Background(), "buffering local candidate") return } @@ -187,7 +185,7 @@ func (c *Conn) init() error { select { case <-c.closed: break - case c.localNegotiator <- Negotiation{nil, []webrtc.ICECandidateInit{json}}: + case c.localCandidateChannel <- iceCandidate.ToJSON(): } }) c.rtc.OnDataChannel(func(dc *webrtc.DataChannel) { @@ -266,7 +264,7 @@ func (c *Conn) negotiate() { select { case <-c.closed: return - case c.localNegotiator <- Negotiation{&offer, nil}: + case c.localSessionDescriptionChannel <- offer: } } @@ -274,7 +272,7 @@ func (c *Conn) negotiate() { select { case <-c.closed: return - case sessionDescription = <-c.remoteSessionDescription: + case sessionDescription = <-c.remoteSessionDescriptionChannel: } // This prevents candidates from being added while @@ -306,18 +304,19 @@ func (c *Conn) negotiate() { select { case <-c.closed: return - case c.localNegotiator <- Negotiation{&answer, nil}: + case c.localSessionDescriptionChannel <- answer: } } c.pendingCandidatesToSendMutex.Lock() defer c.pendingCandidatesToSendMutex.Unlock() - if len(c.pendingCandidatesToSend) > 0 { + for _, pendingCandidate := range c.pendingCandidatesToSend { select { case <-c.closed: return - case c.localNegotiator <- Negotiation{nil, c.pendingCandidatesToSend}: + case c.localCandidateChannel <- pendingCandidate: } + c.opts.Logger.Debug(context.Background(), "flushed buffered local candidate") } c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", slog.F("count", len(c.pendingCandidatesToSend)), @@ -326,37 +325,32 @@ func (c *Conn) negotiate() { c.pendingCandidatesFlushed = true } -// LocalNegotiation returns a channel for connection negotiation. -// This should be piped to another peer connection. -func (c *Conn) LocalNegotiation() <-chan Negotiation { - return c.localNegotiator +// AddRemoteCandidate adds a remote candidate to the RTC connection. +func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { + c.remoteSessionDescriptionMutex.Lock() + defer c.remoteSessionDescriptionMutex.Unlock() + c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("length", len(i.Candidate))) + return c.rtc.AddICECandidate(i) } -// AddRemoteNegotiation accepts a negotiation message for handshaking a connection. -func (c *Conn) AddRemoteNegotiation(negotiation Negotiation) error { - if negotiation.SessionDescription != nil { - c.opts.Logger.Debug(context.Background(), "adding remote negotiation with session description") - select { - case <-c.closed: - return nil - case c.remoteSessionDescription <- *negotiation.SessionDescription: - } +// SetRemoteSessionDescription sets the remote description for the WebRTC connection. +func (c *Conn) SetRemoteSessionDescription(sessionDescription webrtc.SessionDescription) { + select { + case <-c.closed: + case c.remoteSessionDescriptionChannel <- sessionDescription: } +} - if len(negotiation.ICECandidates) > 0 { - c.remoteSessionDescriptionMutex.Lock() - defer c.remoteSessionDescriptionMutex.Unlock() - c.opts.Logger.Debug(context.Background(), "adding remote negotiation with ice candidates", - slog.F("count", len(negotiation.ICECandidates))) - for _, iceCandidate := range negotiation.ICECandidates { - err := c.rtc.AddICECandidate(iceCandidate) - if err != nil { - return err - } - } - } +// LocalSessionDescription returns a channel that emits a session description +// when one is required to be exchanged. +func (c *Conn) LocalSessionDescription() <-chan webrtc.SessionDescription { + return c.localSessionDescriptionChannel +} - return nil +// LocalCandidate returns a channel that emits when a local candidate +// needs to be exchanged with a remote connection. +func (c *Conn) LocalCandidate() <-chan webrtc.ICECandidateInit { + return c.localCandidateChannel } func (c *Conn) pingChannel() (*Channel, error) { diff --git a/peer/conn_test.go b/peer/conn_test.go index 2bdf450cae144..ce3bdca39eeb7 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -307,8 +307,10 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R go func() { for { select { - case c := <-channel2.LocalNegotiation(): - _ = channel1.AddRemoteNegotiation(c) + case c := <-channel2.LocalCandidate(): + _ = channel1.AddRemoteCandidate(c) + case c := <-channel2.LocalSessionDescription(): + channel1.SetRemoteSessionDescription(c) case <-channel2.Closed(): return } @@ -318,8 +320,10 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R go func() { for { select { - case c := <-channel1.LocalNegotiation(): - _ = channel2.AddRemoteNegotiation(c) + case c := <-channel1.LocalCandidate(): + _ = channel2.AddRemoteCandidate(c) + case c := <-channel1.LocalSessionDescription(): + channel2.SetRemoteSessionDescription(c) case <-channel1.Closed(): return } diff --git a/peerbroker/dial.go b/peerbroker/dial.go index 8b3363b69f8ba..9af9c663bec6f 100644 --- a/peerbroker/dial.go +++ b/peerbroker/dial.go @@ -53,16 +53,29 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we select { case <-peerConn.Closed(): return - case localNegotiation := <-peerConn.LocalNegotiation(): + case sessionDescription := <-peerConn.LocalSessionDescription(): err = stream.Send(&proto.NegotiateConnection_ClientToServer{ - Message: &proto.NegotiateConnection_ClientToServer_Negotiation{ - Negotiation: convertLocalNegotiation(localNegotiation), + Message: &proto.NegotiateConnection_ClientToServer_Offer{ + Offer: &proto.WebRTCSessionDescription{ + SdpType: int32(sessionDescription.Type), + Sdp: sessionDescription.SDP, + }, }, }) if err != nil { _ = peerConn.CloseWithError(xerrors.Errorf("send local session description: %w", err)) return } + case iceCandidate := <-peerConn.LocalCandidate(): + err = stream.Send(&proto.NegotiateConnection_ClientToServer{ + Message: &proto.NegotiateConnection_ClientToServer_IceCandidate{ + IceCandidate: iceCandidate.Candidate, + }, + }) + if err != nil { + _ = peerConn.CloseWithError(xerrors.Errorf("send local candidate: %w", err)) + return + } } } }() @@ -74,14 +87,23 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we _ = peerConn.CloseWithError(xerrors.Errorf("recv: %w", err)) return } - if serverToClientMessage.GetNegotiation() == nil { - _ = peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(serverToClientMessage).String())) - return - } - err = peerConn.AddRemoteNegotiation(convertProtoNegotiation(serverToClientMessage.Negotiation)) - if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("add remote negotiation: %w", err)) + switch { + case serverToClientMessage.GetAnswer() != nil: + peerConn.SetRemoteSessionDescription(webrtc.SessionDescription{ + Type: webrtc.SDPType(serverToClientMessage.GetAnswer().SdpType), + SDP: serverToClientMessage.GetAnswer().Sdp, + }) + case serverToClientMessage.GetIceCandidate() != "": + err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ + Candidate: serverToClientMessage.GetIceCandidate(), + }) + if err != nil { + _ = peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) + return + } + default: + _ = peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(serverToClientMessage).String())) return } } @@ -89,41 +111,3 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we return peerConn, nil } - -func convertLocalNegotiation(localNegotiation peer.Negotiation) *proto.Negotiation { - protoNegotation := &proto.Negotiation{} - if localNegotiation.SessionDescription != nil { - protoNegotation.SessionDescription = &proto.WebRTCSessionDescription{ - SdpType: int32(localNegotiation.SessionDescription.Type), - Sdp: localNegotiation.SessionDescription.SDP, - } - } - if len(localNegotiation.ICECandidates) > 0 { - iceCandidates := make([]string, 0, len(localNegotiation.ICECandidates)) - for _, iceCandidate := range localNegotiation.ICECandidates { - iceCandidates = append(iceCandidates, iceCandidate.Candidate) - } - protoNegotation.IceCandidates = iceCandidates - } - return protoNegotation -} - -func convertProtoNegotiation(protoNegotiation *proto.Negotiation) peer.Negotiation { - localNegotiation := peer.Negotiation{} - if protoNegotiation.SessionDescription != nil { - localNegotiation.SessionDescription = &webrtc.SessionDescription{ - Type: webrtc.SDPType(protoNegotiation.SessionDescription.SdpType), - SDP: protoNegotiation.SessionDescription.Sdp, - } - } - if len(protoNegotiation.IceCandidates) > 0 { - candidates := make([]webrtc.ICECandidateInit, 0, len(protoNegotiation.IceCandidates)) - for _, iceCandidate := range protoNegotiation.IceCandidates { - candidates = append(candidates, webrtc.ICECandidateInit{ - Candidate: iceCandidate, - }) - } - localNegotiation.ICECandidates = candidates - } - return localNegotiation -} diff --git a/peerbroker/listen.go b/peerbroker/listen.go index 51c64cd2c83cd..db218a209fc8a 100644 --- a/peerbroker/listen.go +++ b/peerbroker/listen.go @@ -120,12 +120,27 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego select { case <-peerConn.Closed(): return - case localNegotiation := <-peerConn.LocalNegotiation(): + case sessionDescription := <-peerConn.LocalSessionDescription(): err = stream.Send(&proto.NegotiateConnection_ServerToClient{ - Negotiation: convertLocalNegotiation(localNegotiation), + Message: &proto.NegotiateConnection_ServerToClient_Answer{ + Answer: &proto.WebRTCSessionDescription{ + SdpType: int32(sessionDescription.Type), + Sdp: sessionDescription.SDP, + }, + }, }) if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("send local negotiation: %w", err)) + _ = peerConn.CloseWithError(xerrors.Errorf("send local session description: %w", err)) + return + } + case iceCandidate := <-peerConn.LocalCandidate(): + err = stream.Send(&proto.NegotiateConnection_ServerToClient{ + Message: &proto.NegotiateConnection_ServerToClient_IceCandidate{ + IceCandidate: iceCandidate.Candidate, + }, + }) + if err != nil { + _ = peerConn.CloseWithError(xerrors.Errorf("send local candidate: %w", err)) return } } @@ -141,11 +156,11 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego } switch { - case clientToServerMessage.GetNegotiation() != nil: - err = peerConn.AddRemoteNegotiation(convertProtoNegotiation(clientToServerMessage.GetNegotiation())) - if err != nil { - return peerConn.CloseWithError(xerrors.Errorf("add remote negotiation: %w", err)) - } + case clientToServerMessage.GetOffer() != nil: + peerConn.SetRemoteSessionDescription(webrtc.SessionDescription{ + Type: webrtc.SDPType(clientToServerMessage.GetOffer().SdpType), + SDP: clientToServerMessage.GetOffer().Sdp, + }) case clientToServerMessage.GetServers() != nil: // Convert protobuf ICE servers to the WebRTC type. iceServers := make([]webrtc.ICEServer, 0, len(clientToServerMessage.GetServers().Servers)) @@ -163,6 +178,13 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego if err != nil { return peerConn.CloseWithError(xerrors.Errorf("set ice configuration: %w", err)) } + case clientToServerMessage.GetIceCandidate() != "": + err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ + Candidate: clientToServerMessage.GetIceCandidate(), + }) + if err != nil { + return peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) + } default: return peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(clientToServerMessage).String())) } diff --git a/peerbroker/proto/peerbroker.pb.go b/peerbroker/proto/peerbroker.pb.go index 825f747a704f8..dc89ea8cb57eb 100644 --- a/peerbroker/proto/peerbroker.pb.go +++ b/peerbroker/proto/peerbroker.pb.go @@ -193,61 +193,6 @@ func (x *WebRTCICEServers) GetServers() []*WebRTCICEServer { return nil } -type Negotiation struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - SessionDescription *WebRTCSessionDescription `protobuf:"bytes,1,opt,name=session_description,json=sessionDescription,proto3" json:"session_description,omitempty"` - IceCandidates []string `protobuf:"bytes,2,rep,name=ice_candidates,json=iceCandidates,proto3" json:"ice_candidates,omitempty"` -} - -func (x *Negotiation) Reset() { - *x = Negotiation{} - if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *Negotiation) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Negotiation) ProtoMessage() {} - -func (x *Negotiation) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use Negotiation.ProtoReflect.Descriptor instead. -func (*Negotiation) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3} -} - -func (x *Negotiation) GetSessionDescription() *WebRTCSessionDescription { - if x != nil { - return x.SessionDescription - } - return nil -} - -func (x *Negotiation) GetIceCandidates() []string { - if x != nil { - return x.IceCandidates - } - return nil -} - type NegotiateConnection struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -257,7 +202,7 @@ type NegotiateConnection struct { func (x *NegotiateConnection) Reset() { *x = NegotiateConnection{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -270,7 +215,7 @@ func (x *NegotiateConnection) String() string { func (*NegotiateConnection) ProtoMessage() {} func (x *NegotiateConnection) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -283,7 +228,7 @@ func (x *NegotiateConnection) ProtoReflect() protoreflect.Message { // Deprecated: Use NegotiateConnection.ProtoReflect.Descriptor instead. func (*NegotiateConnection) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3} } type NegotiateConnection_ClientToServer struct { @@ -293,14 +238,15 @@ type NegotiateConnection_ClientToServer struct { // Types that are assignable to Message: // *NegotiateConnection_ClientToServer_Servers - // *NegotiateConnection_ClientToServer_Negotiation + // *NegotiateConnection_ClientToServer_Offer + // *NegotiateConnection_ClientToServer_IceCandidate Message isNegotiateConnection_ClientToServer_Message `protobuf_oneof:"message"` } func (x *NegotiateConnection_ClientToServer) Reset() { *x = NegotiateConnection_ClientToServer{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -313,7 +259,7 @@ func (x *NegotiateConnection_ClientToServer) String() string { func (*NegotiateConnection_ClientToServer) ProtoMessage() {} func (x *NegotiateConnection_ClientToServer) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -326,7 +272,7 @@ func (x *NegotiateConnection_ClientToServer) ProtoReflect() protoreflect.Message // Deprecated: Use NegotiateConnection_ClientToServer.ProtoReflect.Descriptor instead. func (*NegotiateConnection_ClientToServer) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4, 0} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3, 0} } func (m *NegotiateConnection_ClientToServer) GetMessage() isNegotiateConnection_ClientToServer_Message { @@ -343,13 +289,20 @@ func (x *NegotiateConnection_ClientToServer) GetServers() *WebRTCICEServers { return nil } -func (x *NegotiateConnection_ClientToServer) GetNegotiation() *Negotiation { - if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_Negotiation); ok { - return x.Negotiation +func (x *NegotiateConnection_ClientToServer) GetOffer() *WebRTCSessionDescription { + if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_Offer); ok { + return x.Offer } return nil } +func (x *NegotiateConnection_ClientToServer) GetIceCandidate() string { + if x, ok := x.GetMessage().(*NegotiateConnection_ClientToServer_IceCandidate); ok { + return x.IceCandidate + } + return "" +} + type isNegotiateConnection_ClientToServer_Message interface { isNegotiateConnection_ClientToServer_Message() } @@ -358,13 +311,19 @@ type NegotiateConnection_ClientToServer_Servers struct { Servers *WebRTCICEServers `protobuf:"bytes,1,opt,name=servers,proto3,oneof"` } -type NegotiateConnection_ClientToServer_Negotiation struct { - Negotiation *Negotiation `protobuf:"bytes,2,opt,name=negotiation,proto3,oneof"` +type NegotiateConnection_ClientToServer_Offer struct { + Offer *WebRTCSessionDescription `protobuf:"bytes,2,opt,name=offer,proto3,oneof"` +} + +type NegotiateConnection_ClientToServer_IceCandidate struct { + IceCandidate string `protobuf:"bytes,3,opt,name=ice_candidate,json=iceCandidate,proto3,oneof"` } func (*NegotiateConnection_ClientToServer_Servers) isNegotiateConnection_ClientToServer_Message() {} -func (*NegotiateConnection_ClientToServer_Negotiation) isNegotiateConnection_ClientToServer_Message() { +func (*NegotiateConnection_ClientToServer_Offer) isNegotiateConnection_ClientToServer_Message() {} + +func (*NegotiateConnection_ClientToServer_IceCandidate) isNegotiateConnection_ClientToServer_Message() { } type NegotiateConnection_ServerToClient struct { @@ -372,13 +331,16 @@ type NegotiateConnection_ServerToClient struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Negotiation *Negotiation `protobuf:"bytes,1,opt,name=negotiation,proto3" json:"negotiation,omitempty"` + // Types that are assignable to Message: + // *NegotiateConnection_ServerToClient_Answer + // *NegotiateConnection_ServerToClient_IceCandidate + Message isNegotiateConnection_ServerToClient_Message `protobuf_oneof:"message"` } func (x *NegotiateConnection_ServerToClient) Reset() { *x = NegotiateConnection_ServerToClient{} if protoimpl.UnsafeEnabled { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[6] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -391,7 +353,7 @@ func (x *NegotiateConnection_ServerToClient) String() string { func (*NegotiateConnection_ServerToClient) ProtoMessage() {} func (x *NegotiateConnection_ServerToClient) ProtoReflect() protoreflect.Message { - mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[6] + mi := &file_peerbroker_proto_peerbroker_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -404,16 +366,47 @@ func (x *NegotiateConnection_ServerToClient) ProtoReflect() protoreflect.Message // Deprecated: Use NegotiateConnection_ServerToClient.ProtoReflect.Descriptor instead. func (*NegotiateConnection_ServerToClient) Descriptor() ([]byte, []int) { - return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{4, 1} + return file_peerbroker_proto_peerbroker_proto_rawDescGZIP(), []int{3, 1} } -func (x *NegotiateConnection_ServerToClient) GetNegotiation() *Negotiation { - if x != nil { - return x.Negotiation +func (m *NegotiateConnection_ServerToClient) GetMessage() isNegotiateConnection_ServerToClient_Message { + if m != nil { + return m.Message + } + return nil +} + +func (x *NegotiateConnection_ServerToClient) GetAnswer() *WebRTCSessionDescription { + if x, ok := x.GetMessage().(*NegotiateConnection_ServerToClient_Answer); ok { + return x.Answer } return nil } +func (x *NegotiateConnection_ServerToClient) GetIceCandidate() string { + if x, ok := x.GetMessage().(*NegotiateConnection_ServerToClient_IceCandidate); ok { + return x.IceCandidate + } + return "" +} + +type isNegotiateConnection_ServerToClient_Message interface { + isNegotiateConnection_ServerToClient_Message() +} + +type NegotiateConnection_ServerToClient_Answer struct { + Answer *WebRTCSessionDescription `protobuf:"bytes,1,opt,name=answer,proto3,oneof"` +} + +type NegotiateConnection_ServerToClient_IceCandidate struct { + IceCandidate string `protobuf:"bytes,2,opt,name=ice_candidate,json=iceCandidate,proto3,oneof"` +} + +func (*NegotiateConnection_ServerToClient_Answer) isNegotiateConnection_ServerToClient_Message() {} + +func (*NegotiateConnection_ServerToClient_IceCandidate) isNegotiateConnection_ServerToClient_Message() { +} + var File_peerbroker_proto_peerbroker_proto protoreflect.FileDescriptor var file_peerbroker_proto_peerbroker_proto_rawDesc = []byte{ @@ -438,43 +431,40 @@ var file_peerbroker_proto_peerbroker_proto_rawDesc = []byte{ 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x49, 0x43, 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, - 0x22, 0x8b, 0x01, 0x0a, 0x0b, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x55, 0x0a, 0x13, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x64, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, - 0x43, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x12, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x69, 0x63, 0x65, 0x5f, 0x63, - 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0d, 0x69, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x73, 0x22, 0xf7, - 0x01, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x92, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x69, 0x65, 0x6e, - 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x07, 0x73, 0x65, 0x72, - 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x65, 0x65, - 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x49, 0x43, - 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x48, 0x00, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, - 0x65, 0x72, 0x73, 0x12, 0x3b, 0x0a, 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, - 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x48, 0x00, 0x52, 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x4b, 0x0a, 0x0e, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x39, 0x0a, - 0x0b, 0x6e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, - 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0b, 0x6e, 0x65, 0x67, - 0x6f, 0x74, 0x69, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x32, 0x87, 0x01, 0x0a, 0x0a, 0x50, 0x65, 0x65, - 0x72, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x79, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, - 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2e, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, - 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x1a, 0x2e, - 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, 0x67, 0x6f, - 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x28, 0x01, - 0x30, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, 0x65, 0x65, - 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0xd7, 0x02, 0x0a, 0x13, 0x4e, 0x65, 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xba, 0x01, 0x0a, 0x0e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x07, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, + 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, + 0x49, 0x43, 0x45, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x48, 0x00, 0x52, 0x07, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x3c, 0x0a, 0x05, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, + 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x66, + 0x66, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0d, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x61, 0x6e, 0x64, 0x69, + 0x64, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x69, 0x63, + 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x82, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x3e, 0x0a, 0x06, 0x61, 0x6e, 0x73, 0x77, + 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, + 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x57, 0x65, 0x62, 0x52, 0x54, 0x43, 0x53, 0x65, 0x73, 0x73, + 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, + 0x52, 0x06, 0x61, 0x6e, 0x73, 0x77, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0d, 0x69, 0x63, 0x65, 0x5f, + 0x63, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x0c, 0x69, 0x63, 0x65, 0x43, 0x61, 0x6e, 0x64, 0x69, 0x64, 0x61, 0x74, 0x65, 0x42, + 0x09, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x87, 0x01, 0x0a, 0x0a, 0x50, + 0x65, 0x65, 0x72, 0x42, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x12, 0x79, 0x0a, 0x13, 0x4e, 0x65, 0x67, + 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, + 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x1a, 0x2e, 0x2e, 0x70, 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2e, 0x4e, 0x65, + 0x67, 0x6f, 0x74, 0x69, 0x61, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x6f, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x70, + 0x65, 0x65, 0x72, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -489,29 +479,27 @@ func file_peerbroker_proto_peerbroker_proto_rawDescGZIP() []byte { return file_peerbroker_proto_peerbroker_proto_rawDescData } -var file_peerbroker_proto_peerbroker_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_peerbroker_proto_peerbroker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) var file_peerbroker_proto_peerbroker_proto_goTypes = []interface{}{ (*WebRTCSessionDescription)(nil), // 0: peerbroker.WebRTCSessionDescription (*WebRTCICEServer)(nil), // 1: peerbroker.WebRTCICEServer (*WebRTCICEServers)(nil), // 2: peerbroker.WebRTCICEServers - (*Negotiation)(nil), // 3: peerbroker.Negotiation - (*NegotiateConnection)(nil), // 4: peerbroker.NegotiateConnection - (*NegotiateConnection_ClientToServer)(nil), // 5: peerbroker.NegotiateConnection.ClientToServer - (*NegotiateConnection_ServerToClient)(nil), // 6: peerbroker.NegotiateConnection.ServerToClient + (*NegotiateConnection)(nil), // 3: peerbroker.NegotiateConnection + (*NegotiateConnection_ClientToServer)(nil), // 4: peerbroker.NegotiateConnection.ClientToServer + (*NegotiateConnection_ServerToClient)(nil), // 5: peerbroker.NegotiateConnection.ServerToClient } var file_peerbroker_proto_peerbroker_proto_depIdxs = []int32{ 1, // 0: peerbroker.WebRTCICEServers.servers:type_name -> peerbroker.WebRTCICEServer - 0, // 1: peerbroker.Negotiation.session_description:type_name -> peerbroker.WebRTCSessionDescription - 2, // 2: peerbroker.NegotiateConnection.ClientToServer.servers:type_name -> peerbroker.WebRTCICEServers - 3, // 3: peerbroker.NegotiateConnection.ClientToServer.negotiation:type_name -> peerbroker.Negotiation - 3, // 4: peerbroker.NegotiateConnection.ServerToClient.negotiation:type_name -> peerbroker.Negotiation - 5, // 5: peerbroker.PeerBroker.NegotiateConnection:input_type -> peerbroker.NegotiateConnection.ClientToServer - 6, // 6: peerbroker.PeerBroker.NegotiateConnection:output_type -> peerbroker.NegotiateConnection.ServerToClient - 6, // [6:7] is the sub-list for method output_type - 5, // [5:6] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 2, // 1: peerbroker.NegotiateConnection.ClientToServer.servers:type_name -> peerbroker.WebRTCICEServers + 0, // 2: peerbroker.NegotiateConnection.ClientToServer.offer:type_name -> peerbroker.WebRTCSessionDescription + 0, // 3: peerbroker.NegotiateConnection.ServerToClient.answer:type_name -> peerbroker.WebRTCSessionDescription + 4, // 4: peerbroker.PeerBroker.NegotiateConnection:input_type -> peerbroker.NegotiateConnection.ClientToServer + 5, // 5: peerbroker.PeerBroker.NegotiateConnection:output_type -> peerbroker.NegotiateConnection.ServerToClient + 5, // [5:6] is the sub-list for method output_type + 4, // [4:5] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_peerbroker_proto_peerbroker_proto_init() } @@ -557,18 +545,6 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } file_peerbroker_proto_peerbroker_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Negotiation); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_peerbroker_proto_peerbroker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NegotiateConnection); i { case 0: return &v.state @@ -580,7 +556,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { return nil } } - file_peerbroker_proto_peerbroker_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_peerbroker_proto_peerbroker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NegotiateConnection_ClientToServer); i { case 0: return &v.state @@ -592,7 +568,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { return nil } } - file_peerbroker_proto_peerbroker_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_peerbroker_proto_peerbroker_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*NegotiateConnection_ServerToClient); i { case 0: return &v.state @@ -605,9 +581,14 @@ func file_peerbroker_proto_peerbroker_proto_init() { } } } - file_peerbroker_proto_peerbroker_proto_msgTypes[5].OneofWrappers = []interface{}{ + file_peerbroker_proto_peerbroker_proto_msgTypes[4].OneofWrappers = []interface{}{ (*NegotiateConnection_ClientToServer_Servers)(nil), - (*NegotiateConnection_ClientToServer_Negotiation)(nil), + (*NegotiateConnection_ClientToServer_Offer)(nil), + (*NegotiateConnection_ClientToServer_IceCandidate)(nil), + } + file_peerbroker_proto_peerbroker_proto_msgTypes[5].OneofWrappers = []interface{}{ + (*NegotiateConnection_ServerToClient_Answer)(nil), + (*NegotiateConnection_ServerToClient_IceCandidate)(nil), } type x struct{} out := protoimpl.TypeBuilder{ @@ -615,7 +596,7 @@ func file_peerbroker_proto_peerbroker_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_peerbroker_proto_peerbroker_proto_rawDesc, NumEnums: 0, - NumMessages: 7, + NumMessages: 6, NumExtensions: 0, NumServices: 1, }, diff --git a/peerbroker/proto/peerbroker.proto b/peerbroker/proto/peerbroker.proto index e35d40f828086..7afec197a51b2 100644 --- a/peerbroker/proto/peerbroker.proto +++ b/peerbroker/proto/peerbroker.proto @@ -20,20 +20,19 @@ message WebRTCICEServers { repeated WebRTCICEServer servers = 1; } -message Negotiation { - WebRTCSessionDescription session_description = 1; - repeated string ice_candidates = 2; -} - message NegotiateConnection { message ClientToServer { oneof message { WebRTCICEServers servers = 1; - Negotiation negotiation = 2; + WebRTCSessionDescription offer = 2; + string ice_candidate = 3; } } message ServerToClient { - Negotiation negotiation = 1; + oneof message { + WebRTCSessionDescription answer = 1; + string ice_candidate = 2; + } } } @@ -46,4 +45,4 @@ service PeerBroker { // // See: https://davekilian.com/webrtc-the-hard-way.html rpc NegotiateConnection(stream NegotiateConnection.ClientToServer) returns (stream NegotiateConnection.ServerToClient); -} +} \ No newline at end of file From 09d84424558b3aa5ccf66d39d4dbab8a0f34a05f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 18:37:01 +0000 Subject: [PATCH 17/22] Properly close ICE gatherer --- .github/workflows/coder.yaml | 2 +- database/migrate_test.go | 5 ++ database/postgres/postgres_test.go | 5 ++ peer/channel.go | 4 +- peer/conn.go | 138 ++++++++++++++++++----------- peer/conn_test.go | 60 ++++++++----- peerbroker/dial.go | 6 +- peerbroker/listen.go | 5 +- 8 files changed, 141 insertions(+), 84 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index d40fd3ba5eba0..23bde4d0fceab 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -158,7 +158,7 @@ jobs: run: gotestsum --jsonfile="gotests.json" --packages="./..." -- -covermode=atomic -coverprofile="gotests.coverage" -timeout=3m - -count=3 -race -parallel=2 + -count=3 -race -short -parallel=2 - name: Test with PostgreSQL Database if: runner.os == 'Linux' diff --git a/database/migrate_test.go b/database/migrate_test.go index 5627e7606cb04..37aec1e96c818 100644 --- a/database/migrate_test.go +++ b/database/migrate_test.go @@ -20,6 +20,11 @@ func TestMain(m *testing.M) { func TestMigrate(t *testing.T) { t.Parallel() + if testing.Short() { + t.Skip() + return + } + t.Run("Once", func(t *testing.T) { t.Parallel() connection, closeFn, err := postgres.Open() diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index c9e8ef12cba11..30d027559c35d 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -21,6 +21,11 @@ func TestMain(m *testing.M) { func TestPostgres(t *testing.T) { t.Parallel() + if testing.Short() { + t.Skip() + return + } + connect, close, err := postgres.Open() require.NoError(t, err) defer close() diff --git a/peer/channel.go b/peer/channel.go index 150398f4d4474..b01154bcfaa25 100644 --- a/peer/channel.go +++ b/peer/channel.go @@ -141,9 +141,9 @@ func (c *Channel) init() { // A DataChannel can disconnect multiple times, so this needs to loop. for { select { - case <-c.closed: + case <-c.conn.closedRTC: // If this channel was closed, there's no need to close again. - return + err = c.conn.closeError case <-c.conn.Closed(): // If the RTC connection closed with an error, this channel // should end with the same one. diff --git a/peer/conn.go b/peer/conn.go index 045c5b9298e9a..df619a7f759e8 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -108,11 +108,13 @@ type Conn struct { // Determines whether this connection will send the offer or the answer. offerrer bool - closed chan struct{} - closedRTC chan struct{} - closedICE chan struct{} - closeMutex sync.Mutex - closeError error + closed chan struct{} + closedRTC chan struct{} + closedRTCMutex sync.Mutex + closedICE chan struct{} + closedICEMutex sync.Mutex + closeMutex sync.Mutex + closeError error dcOpenChannel chan *webrtc.DataChannel dcDisconnectChannel chan struct{} @@ -128,7 +130,6 @@ type Conn struct { pendingCandidatesToSend []webrtc.ICECandidateInit pendingCandidatesToSendMutex sync.Mutex - pendingCandidatesFlushed bool pingChannelID uint16 pingEchoChannelID uint16 @@ -155,6 +156,8 @@ func (c *Conn) init() error { slog.F("state", iceConnectionState)) if iceConnectionState == webrtc.ICEConnectionStateClosed { + c.closedICEMutex.Lock() + defer c.closedICEMutex.Unlock() select { case <-c.closedICE: default: @@ -169,24 +172,36 @@ func (c *Conn) init() error { c.rtc.OnICEGatheringStateChange(func(iceGatherState webrtc.ICEGathererState) { c.opts.Logger.Debug(context.Background(), "ice gathering state updated", slog.F("state", iceGatherState)) + + if iceGatherState == webrtc.ICEGathererStateClosed { + c.closedICEMutex.Lock() + defer c.closedICEMutex.Unlock() + select { + case <-c.closedICE: + default: + close(c.closedICE) + } + } }) c.rtc.OnICECandidate(func(iceCandidate *webrtc.ICECandidate) { if iceCandidate == nil { return } - c.pendingCandidatesToSendMutex.Lock() - defer c.pendingCandidatesToSendMutex.Unlock() - if !c.pendingCandidatesFlushed { - c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) - c.opts.Logger.Debug(context.Background(), "buffering local candidate") - return - } - c.opts.Logger.Debug(context.Background(), "sending local candidate") - select { - case <-c.closed: - break - case c.localCandidateChannel <- iceCandidate.ToJSON(): - } + go func() { + c.pendingCandidatesToSendMutex.Lock() + defer c.pendingCandidatesToSendMutex.Unlock() + if c.rtc.RemoteDescription() == nil { + c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) + c.opts.Logger.Debug(context.Background(), "buffering local candidate") + return + } + c.opts.Logger.Debug(context.Background(), "sending local candidate") + select { + case <-c.closed: + break + case c.localCandidateChannel <- iceCandidate.ToJSON(): + } + }() }) c.rtc.OnDataChannel(func(dc *webrtc.DataChannel) { select { @@ -197,6 +212,11 @@ func (c *Conn) init() error { } }) c.rtc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) { + if c.isClosed() { + return + } + // Pion executes this handler multiple times in a rare condition. + // This prevents logging from happening after close. c.opts.Logger.Debug(context.Background(), "rtc connection updated", slog.F("state", peerConnectionState)) @@ -215,18 +235,20 @@ func (c *Conn) init() error { default: } } - } - - if peerConnectionState == webrtc.PeerConnectionStateClosed { + case webrtc.PeerConnectionStateClosed: // Pion executes event handlers after close is called // on the RTC connection. This ensures our Close() // handler properly cleans up before returning. // // Pion can execute this multiple times, so we check // if it's open before closing. + c.closedRTCMutex.Lock() + defer c.closedRTCMutex.Unlock() select { case <-c.closedRTC: + c.opts.Logger.Debug(context.Background(), "closedRTC channel already closed") default: + c.opts.Logger.Debug(context.Background(), "closedRTC channel closing...") close(c.closedRTC) } } @@ -248,6 +270,8 @@ func (c *Conn) init() error { // uses trickle ICE by default. See: https://webrtchacks.com/trickle-ice/ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") + c.remoteSessionDescriptionMutex.Lock() + defer c.remoteSessionDescriptionMutex.Unlock() if c.offerrer { offer, err := c.rtc.CreateOffer(&webrtc.OfferOptions{}) @@ -255,7 +279,9 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("create offer: %w", err)) return } + c.closeMutex.Lock() err = c.rtc.SetLocalDescription(offer) + c.closeMutex.Unlock() if err != nil { _ = c.CloseWithError(xerrors.Errorf("set local description: %w", err)) return @@ -266,25 +292,23 @@ func (c *Conn) negotiate() { return case c.localSessionDescriptionChannel <- offer: } + c.opts.Logger.Debug(context.Background(), "sent offer") } var sessionDescription webrtc.SessionDescription + c.opts.Logger.Debug(context.Background(), "awaiting remote description...") select { case <-c.closed: return case sessionDescription = <-c.remoteSessionDescriptionChannel: } - // This prevents candidates from being added while - // the remote description is being set. - c.remoteSessionDescriptionMutex.Lock() c.opts.Logger.Debug(context.Background(), "setting remote description") err := c.rtc.SetRemoteDescription(sessionDescription) if err != nil { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) return } - c.remoteSessionDescriptionMutex.Unlock() if !c.offerrer { answer, err := c.rtc.CreateAnswer(&webrtc.AnswerOptions{}) @@ -306,31 +330,44 @@ func (c *Conn) negotiate() { return case c.localSessionDescriptionChannel <- answer: } + c.opts.Logger.Debug(context.Background(), "sent answer") } - c.pendingCandidatesToSendMutex.Lock() - defer c.pendingCandidatesToSendMutex.Unlock() - for _, pendingCandidate := range c.pendingCandidatesToSend { - select { - case <-c.closed: - return - case c.localCandidateChannel <- pendingCandidate: + go func() { + c.pendingCandidatesToSendMutex.Lock() + defer c.pendingCandidatesToSendMutex.Unlock() + for _, pendingCandidate := range c.pendingCandidatesToSend { + select { + case <-c.closed: + return + case c.localCandidateChannel <- pendingCandidate: + } + c.opts.Logger.Debug(context.Background(), "flushed buffered local candidate") } - c.opts.Logger.Debug(context.Background(), "flushed buffered local candidate") - } - c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", - slog.F("count", len(c.pendingCandidatesToSend)), - ) - c.pendingCandidatesToSend = make([]webrtc.ICECandidateInit, 0) - c.pendingCandidatesFlushed = true + c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", + slog.F("count", len(c.pendingCandidatesToSend)), + ) + c.pendingCandidatesToSend = make([]webrtc.ICECandidateInit, 0) + }() } // AddRemoteCandidate adds a remote candidate to the RTC connection. -func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) error { - c.remoteSessionDescriptionMutex.Lock() - defer c.remoteSessionDescriptionMutex.Unlock() - c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("length", len(i.Candidate))) - return c.rtc.AddICECandidate(i) +func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) { + if c.isClosed() { + return + } + go func() { + c.remoteSessionDescriptionMutex.Lock() + defer c.remoteSessionDescriptionMutex.Unlock() + if c.isClosed() { + return + } + c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("length", len(i.Candidate))) + err := c.rtc.AddICECandidate(i) + if err != nil { + _ = c.CloseWithError(xerrors.Errorf("accept candidate: %w", err)) + } + }() } // SetRemoteSessionDescription sets the remote description for the WebRTC connection. @@ -528,7 +565,6 @@ func (c *Conn) CloseWithError(err error) error { } else { c.closeError = err } - close(c.closed) if ch, _ := c.pingChannel(); ch != nil { _ = ch.closeWithError(c.closeError) @@ -538,10 +574,6 @@ func (c *Conn) CloseWithError(err error) error { // closing an already closed connection isn't an issue for us. _ = c.rtc.Close() - // Waits for all DataChannels to exit before officially labeling as closed. - // All logging, goroutines, and async functionality is cleaned up after this. - c.dcClosedWaitGroup.Wait() - if c.rtc.ConnectionState() != webrtc.PeerConnectionStateNew { c.opts.Logger.Debug(context.Background(), "waiting for rtc connection close...") <-c.closedRTC @@ -552,5 +584,11 @@ func (c *Conn) CloseWithError(err error) error { <-c.closedICE } + // Waits for all DataChannels to exit before officially labeling as closed. + // All logging, goroutines, and async functionality is cleaned up after this. + c.dcClosedWaitGroup.Wait() + + close(c.closed) + c.opts.Logger.Debug(context.Background(), "closed") return err } diff --git a/peer/conn_test.go b/peer/conn_test.go index ce3bdca39eeb7..bf27b0ccf00cc 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -48,14 +48,7 @@ var ( ) func TestMain(m *testing.M) { - // pion/ice doesn't properly close immediately. The solution for this isn't yet known. See: - // https://github.com/pion/ice/pull/413 - goleak.VerifyTestMain(m, - goleak.IgnoreTopFunction("github.com/pion/ice/v2.(*Agent).startOnConnectionStateChangeRoutine.func1"), - goleak.IgnoreTopFunction("github.com/pion/ice/v2.(*Agent).startOnConnectionStateChangeRoutine.func2"), - goleak.IgnoreTopFunction("github.com/pion/ice/v2.(*Agent).taskLoop"), - goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), - ) + goleak.VerifyTestMain(m) } func TestConn(t *testing.T) { @@ -64,6 +57,7 @@ func TestConn(t *testing.T) { t.Run("Ping", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) _, err := client.Ping() require.NoError(t, err) _, err = server.Ping() @@ -72,7 +66,8 @@ func TestConn(t *testing.T) { t.Run("PingNetworkOffline", func(t *testing.T) { t.Parallel() - _, server, wan := createPair(t) + client, server, wan := createPair(t) + exchange(client, server) _, err := server.Ping() require.NoError(t, err) err = wan.Stop() @@ -83,7 +78,8 @@ func TestConn(t *testing.T) { t.Run("PingReconnect", func(t *testing.T) { t.Parallel() - _, server, wan := createPair(t) + client, server, wan := createPair(t) + exchange(client, server) _, err := server.Ping() require.NoError(t, err) // Create a channel that closes on disconnect. @@ -104,6 +100,7 @@ func TestConn(t *testing.T) { t.Run("Accept", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) @@ -119,6 +116,7 @@ func TestConn(t *testing.T) { t.Run("AcceptNetworkOffline", func(t *testing.T) { t.Parallel() client, server, wan := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) @@ -135,6 +133,7 @@ func TestConn(t *testing.T) { t.Run("Buffering", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) cch, err := client.Dial(context.Background(), "hello", &peer.ChannelOptions{}) require.NoError(t, err) sch, err := server.Accept(context.Background()) @@ -161,6 +160,7 @@ func TestConn(t *testing.T) { t.Run("NetConn", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) srv, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) defer srv.Close() @@ -213,6 +213,7 @@ func TestConn(t *testing.T) { t.Run("CloseBeforeNegotiate", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) err := client.Close() require.NoError(t, err) err = server.Close() @@ -232,6 +233,7 @@ func TestConn(t *testing.T) { t.Run("PingConcurrent", func(t *testing.T) { t.Parallel() client, server, _ := createPair(t) + exchange(client, server) var wg sync.WaitGroup wg.Add(2) go func() { @@ -246,6 +248,18 @@ func TestConn(t *testing.T) { }() wg.Wait() }) + + t.Run("CandidateBeforeSessionDescription", func(t *testing.T) { + t.Parallel() + client, server, _ := createPair(t) + server.SetRemoteSessionDescription(<-client.LocalSessionDescription()) + sdp := <-server.LocalSessionDescription() + client.AddRemoteCandidate(<-server.LocalCandidate()) + client.SetRemoteSessionDescription(sdp) + server.AddRemoteCandidate(<-client.LocalCandidate()) + _, err := client.Ping() + require.NoError(t, err) + }) } func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.Router) { @@ -304,14 +318,18 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R _ = wan.Stop() }) + return channel1, channel2, wan +} + +func exchange(client, server *peer.Conn) { go func() { for { select { - case c := <-channel2.LocalCandidate(): - _ = channel1.AddRemoteCandidate(c) - case c := <-channel2.LocalSessionDescription(): - channel1.SetRemoteSessionDescription(c) - case <-channel2.Closed(): + case c := <-server.LocalCandidate(): + client.AddRemoteCandidate(c) + case c := <-server.LocalSessionDescription(): + client.SetRemoteSessionDescription(c) + case <-server.Closed(): return } } @@ -320,15 +338,13 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R go func() { for { select { - case c := <-channel1.LocalCandidate(): - _ = channel2.AddRemoteCandidate(c) - case c := <-channel1.LocalSessionDescription(): - channel2.SetRemoteSessionDescription(c) - case <-channel1.Closed(): + case c := <-client.LocalCandidate(): + server.AddRemoteCandidate(c) + case c := <-client.LocalSessionDescription(): + server.SetRemoteSessionDescription(c) + case <-client.Closed(): return } } }() - - return channel1, channel2, wan } diff --git a/peerbroker/dial.go b/peerbroker/dial.go index 9af9c663bec6f..7253e44f0f582 100644 --- a/peerbroker/dial.go +++ b/peerbroker/dial.go @@ -95,13 +95,9 @@ func Dial(stream proto.DRPCPeerBroker_NegotiateConnectionClient, iceServers []we SDP: serverToClientMessage.GetAnswer().Sdp, }) case serverToClientMessage.GetIceCandidate() != "": - err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ + peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ Candidate: serverToClientMessage.GetIceCandidate(), }) - if err != nil { - _ = peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) - return - } default: _ = peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(serverToClientMessage).String())) return diff --git a/peerbroker/listen.go b/peerbroker/listen.go index db218a209fc8a..fa68023689c7e 100644 --- a/peerbroker/listen.go +++ b/peerbroker/listen.go @@ -179,12 +179,9 @@ func (b *peerBrokerService) NegotiateConnection(stream proto.DRPCPeerBroker_Nego return peerConn.CloseWithError(xerrors.Errorf("set ice configuration: %w", err)) } case clientToServerMessage.GetIceCandidate() != "": - err = peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ + peerConn.AddRemoteCandidate(webrtc.ICECandidateInit{ Candidate: clientToServerMessage.GetIceCandidate(), }) - if err != nil { - return peerConn.CloseWithError(xerrors.Errorf("add remote candidate: %w", err)) - } default: return peerConn.CloseWithError(xerrors.Errorf("unhandled message: %s", reflect.TypeOf(clientToServerMessage).String())) } From 1facda93d0ed82166fbe022621c57bcd1e407507 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 22:29:01 +0000 Subject: [PATCH 18/22] Improve comments --- peer/conn.go | 126 +++++++++++++++++++++++++++------------------------ 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index df619a7f759e8..3bddc815582ee 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -126,7 +126,8 @@ type Conn struct { localCandidateChannel chan webrtc.ICECandidateInit localSessionDescriptionChannel chan webrtc.SessionDescription remoteSessionDescriptionChannel chan webrtc.SessionDescription - remoteSessionDescriptionMutex sync.Mutex + + negotiateMutex sync.Mutex pendingCandidatesToSend []webrtc.ICECandidateInit pendingCandidatesToSendMutex sync.Mutex @@ -143,12 +144,6 @@ type Conn struct { pingError error } -// Negotiation represents a handshake message between peer connections. -type Negotiation struct { - SessionDescription *webrtc.SessionDescription - ICECandidates []webrtc.ICECandidateInit -} - func (c *Conn) init() error { c.rtc.OnNegotiationNeeded(c.negotiate) c.rtc.OnICEConnectionStateChange(func(iceConnectionState webrtc.ICEConnectionState) { @@ -156,6 +151,9 @@ func (c *Conn) init() error { slog.F("state", iceConnectionState)) if iceConnectionState == webrtc.ICEConnectionStateClosed { + // pion/webrtc can update this state multiple times. + // A connection can never become un-closed, so we + // close the channel if it isn't already. c.closedICEMutex.Lock() defer c.closedICEMutex.Unlock() select { @@ -165,15 +163,14 @@ func (c *Conn) init() error { } } }) - c.rtc.OnSignalingStateChange(func(signalState webrtc.SignalingState) { - c.opts.Logger.Debug(context.Background(), "signal state updated", - slog.F("state", signalState)) - }) c.rtc.OnICEGatheringStateChange(func(iceGatherState webrtc.ICEGathererState) { c.opts.Logger.Debug(context.Background(), "ice gathering state updated", slog.F("state", iceGatherState)) if iceGatherState == webrtc.ICEGathererStateClosed { + // pion/webrtc can update this state multiple times. + // A connection can never become un-closed, so we + // close the channel if it isn't already. c.closedICEMutex.Lock() defer c.closedICEMutex.Unlock() select { @@ -183,40 +180,11 @@ func (c *Conn) init() error { } } }) - c.rtc.OnICECandidate(func(iceCandidate *webrtc.ICECandidate) { - if iceCandidate == nil { - return - } - go func() { - c.pendingCandidatesToSendMutex.Lock() - defer c.pendingCandidatesToSendMutex.Unlock() - if c.rtc.RemoteDescription() == nil { - c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) - c.opts.Logger.Debug(context.Background(), "buffering local candidate") - return - } - c.opts.Logger.Debug(context.Background(), "sending local candidate") - select { - case <-c.closed: - break - case c.localCandidateChannel <- iceCandidate.ToJSON(): - } - }() - }) - c.rtc.OnDataChannel(func(dc *webrtc.DataChannel) { - select { - case <-c.closed: - return - case c.dcOpenChannel <- dc: - default: - } - }) c.rtc.OnConnectionStateChange(func(peerConnectionState webrtc.PeerConnectionState) { if c.isClosed() { + // Make sure we don't log after Close() has been called. return } - // Pion executes this handler multiple times in a rare condition. - // This prevents logging from happening after close. c.opts.Logger.Debug(context.Background(), "rtc connection updated", slog.F("state", peerConnectionState)) @@ -236,23 +204,55 @@ func (c *Conn) init() error { } } case webrtc.PeerConnectionStateClosed: - // Pion executes event handlers after close is called - // on the RTC connection. This ensures our Close() - // handler properly cleans up before returning. - // - // Pion can execute this multiple times, so we check - // if it's open before closing. + // pion/webrtc can update this state multiple times. + // A connection can never become un-closed, so we + // close the channel if it isn't already. c.closedRTCMutex.Lock() defer c.closedRTCMutex.Unlock() select { case <-c.closedRTC: - c.opts.Logger.Debug(context.Background(), "closedRTC channel already closed") default: - c.opts.Logger.Debug(context.Background(), "closedRTC channel closing...") close(c.closedRTC) } } }) + c.rtc.OnSignalingStateChange(func(signalState webrtc.SignalingState) { + c.opts.Logger.Debug(context.Background(), "signaling state updated", + slog.F("state", signalState)) + }) + c.rtc.OnICECandidate(func(iceCandidate *webrtc.ICECandidate) { + if iceCandidate == nil { + return + } + // Run this in a goroutine so we don't block pion/webrtc + // from continuing. + go func() { + c.pendingCandidatesToSendMutex.Lock() + defer c.pendingCandidatesToSendMutex.Unlock() + // If the remote description hasn't been set yet, we queue the send of these candidates. + // It may work to send these immediately, but at the time of writing this package is + // unstable, so better being safe than sorry. + if c.rtc.RemoteDescription() == nil { + c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) + c.opts.Logger.Debug(context.Background(), "buffering local candidate") + return + } + c.opts.Logger.Debug(context.Background(), "sending local candidate") + select { + case <-c.closed: + break + case c.localCandidateChannel <- iceCandidate.ToJSON(): + } + }() + }) + c.rtc.OnDataChannel(func(dc *webrtc.DataChannel) { + select { + case <-c.closed: + return + case c.dcOpenChannel <- dc: + default: + } + }) _, err := c.pingChannel() if err != nil { return err @@ -265,13 +265,14 @@ func (c *Conn) init() error { return nil } -// Negotiate exchanges ICECandidate pairs over the exposed channels. -// The diagram below shows the expected handshake. pion/webrtc v3 -// uses trickle ICE by default. See: https://webrtchacks.com/trickle-ice/ +// negotiate is triggered when a connection is ready to be established. +// See trickle ICE for the expected exchange: https://webrtchacks.com/trickle-ice/ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") - c.remoteSessionDescriptionMutex.Lock() - defer c.remoteSessionDescriptionMutex.Unlock() + // ICE candidates cannot be added until SessionDescriptions have been + // exchanged between peers. + c.negotiateMutex.Lock() + defer c.negotiateMutex.Unlock() if c.offerrer { offer, err := c.rtc.CreateOffer(&webrtc.OfferOptions{}) @@ -279,6 +280,8 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("create offer: %w", err)) return } + // pion/webrtc will panic if Close is called while this + // function is being executed. c.closeMutex.Lock() err = c.rtc.SetLocalDescription(offer) c.closeMutex.Unlock() @@ -302,8 +305,8 @@ func (c *Conn) negotiate() { return case sessionDescription = <-c.remoteSessionDescriptionChannel: } - c.opts.Logger.Debug(context.Background(), "setting remote description") + err := c.rtc.SetRemoteDescription(sessionDescription) if err != nil { _ = c.CloseWithError(xerrors.Errorf("set remote description (closed %v): %w", c.isClosed(), err)) @@ -316,8 +319,9 @@ func (c *Conn) negotiate() { _ = c.CloseWithError(xerrors.Errorf("create answer: %w", err)) return } + // pion/webrtc will panic if Close is called while this + // function is being executed. c.closeMutex.Lock() - // pion doesn't handle a close properly if it occurs during this function. err = c.rtc.SetLocalDescription(answer) c.closeMutex.Unlock() if err != nil { @@ -333,6 +337,7 @@ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "sent answer") } + // Flush bufferred candidates after both sides have been negotiated! go func() { c.pendingCandidatesToSendMutex.Lock() defer c.pendingCandidatesToSendMutex.Unlock() @@ -356,9 +361,11 @@ func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) { if c.isClosed() { return } + // This must occur in a goroutine to allow the SessionDescriptions + // to be exchanged first. go func() { - c.remoteSessionDescriptionMutex.Lock() - defer c.remoteSessionDescriptionMutex.Unlock() + c.negotiateMutex.Lock() + defer c.negotiateMutex.Unlock() if c.isClosed() { return } @@ -574,11 +581,12 @@ func (c *Conn) CloseWithError(err error) error { // closing an already closed connection isn't an issue for us. _ = c.rtc.Close() + // Waiting for pion/webrtc to report closed state on both of these + // ensures no goroutine leaks. if c.rtc.ConnectionState() != webrtc.PeerConnectionStateNew { c.opts.Logger.Debug(context.Background(), "waiting for rtc connection close...") <-c.closedRTC } - if c.rtc.ICEConnectionState() != webrtc.ICEConnectionStateNew { c.opts.Logger.Debug(context.Background(), "waiting for ice connection close...") <-c.closedICE From f58a9808acde2f5e2f8384ce9c4f2714f7d6abcb Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 30 Jan 2022 22:49:07 +0000 Subject: [PATCH 19/22] Try removing buffered candidates --- peer/conn.go | 51 +++++++++++++-------------------------------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index 3bddc815582ee..cddbd0eee2952 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -77,7 +77,6 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp localCandidateChannel: make(chan webrtc.ICECandidateInit), localSessionDescriptionChannel: make(chan webrtc.SessionDescription), remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription), - pendingCandidatesToSend: make([]webrtc.ICECandidateInit, 0), } if client { // If we're the client, we want to flip the echo and @@ -128,9 +127,7 @@ type Conn struct { remoteSessionDescriptionChannel chan webrtc.SessionDescription negotiateMutex sync.Mutex - - pendingCandidatesToSend []webrtc.ICECandidateInit - pendingCandidatesToSendMutex sync.Mutex + hasNegotiated bool pingChannelID uint16 pingEchoChannelID uint16 @@ -145,6 +142,9 @@ type Conn struct { } func (c *Conn) init() error { + // The negotiation needed callback can take a little bit to execute! + c.negotiateMutex.Lock() + c.rtc.OnNegotiationNeeded(c.negotiate) c.rtc.OnICEConnectionStateChange(func(iceConnectionState webrtc.ICEConnectionState) { c.opts.Logger.Debug(context.Background(), "ice connection state updated", @@ -227,17 +227,7 @@ func (c *Conn) init() error { // Run this in a goroutine so we don't block pion/webrtc // from continuing. go func() { - c.pendingCandidatesToSendMutex.Lock() - defer c.pendingCandidatesToSendMutex.Unlock() - // If the remote description hasn't been set yet, we queue the send of these candidates. - // It may work to send these immediately, but at the time of writing this package is - // unstable, so better being safe than sorry. - if c.rtc.RemoteDescription() == nil { - c.pendingCandidatesToSend = append(c.pendingCandidatesToSend, iceCandidate.ToJSON()) - c.opts.Logger.Debug(context.Background(), "buffering local candidate") - return - } - c.opts.Logger.Debug(context.Background(), "sending local candidate") + c.opts.Logger.Debug(context.Background(), "sending local candidate", slog.F("candidate", iceCandidate.ToJSON().Candidate)) select { case <-c.closed: break @@ -271,7 +261,10 @@ func (c *Conn) negotiate() { c.opts.Logger.Debug(context.Background(), "negotiating") // ICE candidates cannot be added until SessionDescriptions have been // exchanged between peers. - c.negotiateMutex.Lock() + if c.hasNegotiated { + c.negotiateMutex.Lock() + } + c.hasNegotiated = true defer c.negotiateMutex.Unlock() if c.offerrer { @@ -336,24 +329,6 @@ func (c *Conn) negotiate() { } c.opts.Logger.Debug(context.Background(), "sent answer") } - - // Flush bufferred candidates after both sides have been negotiated! - go func() { - c.pendingCandidatesToSendMutex.Lock() - defer c.pendingCandidatesToSendMutex.Unlock() - for _, pendingCandidate := range c.pendingCandidatesToSend { - select { - case <-c.closed: - return - case c.localCandidateChannel <- pendingCandidate: - } - c.opts.Logger.Debug(context.Background(), "flushed buffered local candidate") - } - c.opts.Logger.Debug(context.Background(), "flushed buffered local candidates", - slog.F("count", len(c.pendingCandidatesToSend)), - ) - c.pendingCandidatesToSend = make([]webrtc.ICECandidateInit, 0) - }() } // AddRemoteCandidate adds a remote candidate to the RTC connection. @@ -366,12 +341,12 @@ func (c *Conn) AddRemoteCandidate(i webrtc.ICECandidateInit) { go func() { c.negotiateMutex.Lock() defer c.negotiateMutex.Unlock() - if c.isClosed() { - return - } - c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("length", len(i.Candidate))) + c.opts.Logger.Debug(context.Background(), "accepting candidate", slog.F("candidate", i.Candidate)) err := c.rtc.AddICECandidate(i) if err != nil { + if c.rtc.ConnectionState() == webrtc.PeerConnectionStateClosed { + return + } _ = c.CloseWithError(xerrors.Errorf("accept candidate: %w", err)) } }() From b29448176981c6b7a2eea72765e4eb81c618d23c Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 31 Jan 2022 00:00:15 +0000 Subject: [PATCH 20/22] Buffer local and remote messages --- peer/conn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index cddbd0eee2952..24cb6d0f58a1a 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -75,8 +75,8 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp dcDisconnectChannel: make(chan struct{}), dcFailedChannel: make(chan struct{}), localCandidateChannel: make(chan webrtc.ICECandidateInit), - localSessionDescriptionChannel: make(chan webrtc.SessionDescription), - remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription), + localSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), + remoteSessionDescriptionChannel: make(chan webrtc.SessionDescription, 1), } if client { // If we're the client, we want to flip the echo and From 65c58c375ff879329ff98e4686853d24c8b4b2bc Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 31 Jan 2022 01:20:54 +0000 Subject: [PATCH 21/22] Log dTLS transport state --- peer/conn.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/peer/conn.go b/peer/conn.go index 24cb6d0f58a1a..e0913dbe5af3d 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -220,6 +220,10 @@ func (c *Conn) init() error { c.opts.Logger.Debug(context.Background(), "signaling state updated", slog.F("state", signalState)) }) + c.rtc.SCTP().Transport().OnStateChange(func(dtlsTransportState webrtc.DTLSTransportState) { + c.opts.Logger.Debug(context.Background(), "dtls transport state updated", + slog.F("state", dtlsTransportState)) + }) c.rtc.OnICECandidate(func(iceCandidate *webrtc.ICECandidate) { if iceCandidate == nil { return From cd775abfbd347992ef1e64d12883cbaa939d72c7 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 31 Jan 2022 01:31:14 +0000 Subject: [PATCH 22/22] Add pion logging --- peer/conn.go | 12 +++++++----- peer/conn_test.go | 8 ++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/peer/conn.go b/peer/conn.go index e0913dbe5af3d..32e0c72bbfa32 100644 --- a/peer/conn.go +++ b/peer/conn.go @@ -49,12 +49,10 @@ func newWithClientOrServer(servers []webrtc.ICEServer, client bool, opts *ConnOp opts = &ConnOptions{} } - // Enables preference to STUN. - opts.SettingEngine.SetSrflxAcceptanceMinWait(0) opts.SettingEngine.DetachDataChannels() - lf := logging.NewDefaultLoggerFactory() - lf.DefaultLogLevel = logging.LogLevelDisabled - opts.SettingEngine.LoggerFactory = lf + factory := logging.NewDefaultLoggerFactory() + factory.DefaultLogLevel = logging.LogLevelDisabled + opts.SettingEngine.LoggerFactory = factory api := webrtc.NewAPI(webrtc.WithSettingEngine(opts.SettingEngine)) rtc, err := api.NewPeerConnection(webrtc.Configuration{ ICEServers: servers, @@ -224,6 +222,10 @@ func (c *Conn) init() error { c.opts.Logger.Debug(context.Background(), "dtls transport state updated", slog.F("state", dtlsTransportState)) }) + c.rtc.SCTP().Transport().ICETransport().OnSelectedCandidatePairChange(func(candidatePair *webrtc.ICECandidatePair) { + c.opts.Logger.Debug(context.Background(), "selected candidate pair changed", + slog.F("local", candidatePair.Local), slog.F("remote", candidatePair.Remote)) + }) c.rtc.OnICECandidate(func(iceCandidate *webrtc.ICECandidate) { if iceCandidate == nil { return diff --git a/peer/conn_test.go b/peer/conn_test.go index bf27b0ccf00cc..67d22483fe6cb 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -287,9 +287,7 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R c1SettingEngine.SetVNet(c1Net) c1SettingEngine.SetPrflxAcceptanceMinWait(0) c1SettingEngine.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval) - channel1, err := peer.Client([]webrtc.ICEServer{{ - URLs: []string{"stun:stun.l.google.com:19302"}, - }}, &peer.ConnOptions{ + channel1, err := peer.Client([]webrtc.ICEServer{{}}, &peer.ConnOptions{ SettingEngine: c1SettingEngine, Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), }) @@ -301,9 +299,7 @@ func createPair(t *testing.T) (client *peer.Conn, server *peer.Conn, wan *vnet.R c2SettingEngine.SetVNet(c2Net) c2SettingEngine.SetPrflxAcceptanceMinWait(0) c2SettingEngine.SetICETimeouts(disconnectedTimeout, failedTimeout, keepAliveInterval) - channel2, err := peer.Server([]webrtc.ICEServer{{ - URLs: []string{"stun:stun.l.google.com:19302"}, - }}, &peer.ConnOptions{ + channel2, err := peer.Server([]webrtc.ICEServer{{}}, &peer.ConnOptions{ SettingEngine: c2SettingEngine, Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), })