From 6cc039c9d370048b44a5c413a6711688319ce433 Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Fri, 28 Aug 2026 09:05:04 +0000 Subject: [PATCH 1/3] fix(coderd): make chatd stream sync poller a callback fanout to fix send-on-closed-channel panic The stream sync poller delivered hints over per-subscriber channels that unregister closed under the poller mutex, while pollOnce sent on them from a lock-free snapshot. A subscriber unregistering mid-poll made pollOnce panic with "send on closed channel", crashing coderd. Any authenticated user could trigger this by opening and closing chat streams (GHSA-7x3x-59xg-4hrc). Convert the poller to a callback fanout, mirroring pubsub.Subscribe: Register now takes a deliver callback and returns only an unregister func, unregister is delete-only (nothing is ever closed), and the consumer funnels poller hints into its existing consumer-owned updateCh with the same streamCtx guard already used for pubsub hints. This makes the panic unrepresentable and also removes the nil-poller closed-channel return that instantly terminated streams via the ok-check. Adds a regression race test churning register/unregister against pollOnce; the previous implementation panics under it. --- coderd/x/chatd/stream_subscribe.go | 14 ++-- coderd/x/chatd/stream_sync_poller.go | 26 +++---- .../chatd/stream_sync_poller_internal_test.go | 74 +++++++++++++++++++ 3 files changed, 92 insertions(+), 22 deletions(-) create mode 100644 coderd/x/chatd/stream_sync_poller_internal_test.go diff --git a/coderd/x/chatd/stream_subscribe.go b/coderd/x/chatd/stream_subscribe.go index 9545b707c11..248122e2fd3 100644 --- a/coderd/x/chatd/stream_subscribe.go +++ b/coderd/x/chatd/stream_subscribe.go @@ -58,7 +58,12 @@ func (p *Server) subscribeStreamLoop( return subscribeWithInitialError(chatID, "failed to subscribe to chat updates") } - pollerCh, unregisterPoller := p.streamSyncPoller.Register(chatID) + unregisterPoller := p.streamSyncPoller.Register(chatID, func(hint streamSyncHint) { + select { + case updateCh <- hint: + case <-streamCtx.Done(): + } + }) loop := newStreamLoop(chat, p.db, logger, afterMessageID) // The immediate sync builds the initial snapshot returned to the caller // and the relay target for the forwarder. Hints only fire on state @@ -97,13 +102,6 @@ func (p *Server) subscribeStreamLoop( if !p.runStreamSync(streamCtx, loop, relay, events, hint) { return } - case hint, ok := <-pollerCh: - if !ok { - return - } - if !p.runStreamSync(streamCtx, loop, relay, events, hint) { - return - } case part, ok := <-relay.Parts(): if !ok { return diff --git a/coderd/x/chatd/stream_sync_poller.go b/coderd/x/chatd/stream_sync_poller.go index 11e9171687e..b3b1601fdc6 100644 --- a/coderd/x/chatd/stream_sync_poller.go +++ b/coderd/x/chatd/stream_sync_poller.go @@ -27,8 +27,8 @@ type streamSyncPoller struct { } type streamSyncPollerSubscriber struct { - chatID uuid.UUID - hints chan streamSyncHint + chatID uuid.UUID + deliver func(streamSyncHint) } func newStreamSyncPoller( @@ -68,15 +68,17 @@ func (p *streamSyncPoller) Close() { p.cancel() } -func (p *streamSyncPoller) Register(chatID uuid.UUID) (<-chan streamSyncHint, func()) { +// Register subscribes deliver to poll hints for chatID until the returned +// unregister func is called. deliver is invoked outside the poller's mutex and +// may race with unregister, so it must remain safe to call after unregister +// returns (e.g. by guarding on the subscriber's own context). +func (p *streamSyncPoller) Register(chatID uuid.UUID, deliver func(streamSyncHint)) (unregister func()) { if p == nil { - ch := make(chan streamSyncHint) - close(ch) - return ch, func() {} + return func() {} } subscriber := &streamSyncPollerSubscriber{ - chatID: chatID, - hints: make(chan streamSyncHint, 1), + chatID: chatID, + deliver: deliver, } p.mu.Lock() if p.subscribers[chatID] == nil { @@ -85,7 +87,7 @@ func (p *streamSyncPoller) Register(chatID uuid.UUID) (<-chan streamSyncHint, fu p.subscribers[chatID][subscriber] = struct{}{} p.mu.Unlock() - return subscriber.hints, func() { + return func() { p.unregister(subscriber) } } @@ -101,7 +103,6 @@ func (p *streamSyncPoller) unregister(subscriber *streamSyncPollerSubscriber) { if len(chatSubscribers) == 0 { delete(p.subscribers, subscriber.chatID) } - close(subscriber.hints) } func (p *streamSyncPoller) loop() { @@ -132,10 +133,7 @@ func (p *streamSyncPoller) pollOnce() { for _, row := range rows { hint := streamSyncHintFromPollRow(row) for _, subscriber := range subscribers[row.ID] { - select { - case subscriber.hints <- hint: - default: - } + subscriber.deliver(hint) } } } diff --git a/coderd/x/chatd/stream_sync_poller_internal_test.go b/coderd/x/chatd/stream_sync_poller_internal_test.go new file mode 100644 index 00000000000..63b2c01c192 --- /dev/null +++ b/coderd/x/chatd/stream_sync_poller_internal_test.go @@ -0,0 +1,74 @@ +package chatd + +import ( + "context" + "sync" + "testing" + + "github.com/google/uuid" + "go.uber.org/mock/gomock" + + "cdr.dev/slog/v3/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbmock" +) + +// TestStreamSyncPollerConcurrentRegisterUnregister churns subscriber +// registration while pollOnce delivers hints concurrently. Before the poller +// became a callback fanout, unregister closed the subscriber's hint channel +// while pollOnce could still be mid-send on its lock-free snapshot, panicking +// with "send on closed channel" (GHSA-7x3x-59xg-4hrc). Run with -race. +func TestStreamSyncPollerConcurrentRegisterUnregister(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + db.EXPECT().GetChatStreamSyncRows(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(_ context.Context, ids []uuid.UUID) ([]database.GetChatStreamSyncRowsRow, error) { + rows := make([]database.GetChatStreamSyncRowsRow, 0, len(ids)) + for _, id := range ids { + rows = append(rows, database.GetChatStreamSyncRowsRow{ID: id}) + } + return rows, nil + }, + ) + + poller := newStreamSyncPoller(context.Background(), db, nil, slogtest.Make(t, nil)) + defer poller.Close() + + chatID := uuid.New() + done := make(chan struct{}) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-done: + return + default: + } + unregister := poller.Register(chatID, func(streamSyncHint) {}) + unregister() + } + }() + } + for range 1000 { + poller.pollOnce() + } + close(done) + wg.Wait() +} + +// TestStreamSyncPollerNilRegister verifies a nil poller degrades to a no-op +// registration instead of terminating subscribers. +func TestStreamSyncPollerNilRegister(t *testing.T) { + t.Parallel() + + var poller *streamSyncPoller + unregister := poller.Register(uuid.New(), func(streamSyncHint) { + t.Fatal("nil poller must never deliver hints") + }) + unregister() +} From e763895ce1fcf049defd7ced2fc241da2e7516df Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Wed, 2 Sep 2026 11:21:30 +0200 Subject: [PATCH 2/3] fix(coderd/x/chatd): drop poller hints when stream update channel is full --- coderd/x/chatd/stream_subscribe.go | 2 +- coderd/x/chatd/stream_sync_poller.go | 5 +- .../chatd/stream_sync_poller_internal_test.go | 48 +++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/coderd/x/chatd/stream_subscribe.go b/coderd/x/chatd/stream_subscribe.go index 248122e2fd3..0eed04ab10f 100644 --- a/coderd/x/chatd/stream_subscribe.go +++ b/coderd/x/chatd/stream_subscribe.go @@ -61,7 +61,7 @@ func (p *Server) subscribeStreamLoop( unregisterPoller := p.streamSyncPoller.Register(chatID, func(hint streamSyncHint) { select { case updateCh <- hint: - case <-streamCtx.Done(): + default: } }) loop := newStreamLoop(chat, p.db, logger, afterMessageID) diff --git a/coderd/x/chatd/stream_sync_poller.go b/coderd/x/chatd/stream_sync_poller.go index b3b1601fdc6..676c387dd12 100644 --- a/coderd/x/chatd/stream_sync_poller.go +++ b/coderd/x/chatd/stream_sync_poller.go @@ -71,7 +71,10 @@ func (p *streamSyncPoller) Close() { // Register subscribes deliver to poll hints for chatID until the returned // unregister func is called. deliver is invoked outside the poller's mutex and // may race with unregister, so it must remain safe to call after unregister -// returns (e.g. by guarding on the subscriber's own context). +// returns (e.g. by sending to a channel that is never closed). deliver runs +// synchronously on the shared poll loop, so it must not block; a blocked +// callback stalls polling for every chat on the replica. Dropped hints are +// re-delivered on the next poll tick. func (p *streamSyncPoller) Register(chatID uuid.UUID, deliver func(streamSyncHint)) (unregister func()) { if p == nil { return func() {} diff --git a/coderd/x/chatd/stream_sync_poller_internal_test.go b/coderd/x/chatd/stream_sync_poller_internal_test.go index 63b2c01c192..cb89ab733e7 100644 --- a/coderd/x/chatd/stream_sync_poller_internal_test.go +++ b/coderd/x/chatd/stream_sync_poller_internal_test.go @@ -4,6 +4,7 @@ import ( "context" "sync" "testing" + "time" "github.com/google/uuid" "go.uber.org/mock/gomock" @@ -11,6 +12,7 @@ import ( "cdr.dev/slog/v3/sloggers/slogtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbmock" + "github.com/coder/coder/v2/testutil" ) // TestStreamSyncPollerConcurrentRegisterUnregister churns subscriber @@ -61,6 +63,52 @@ func TestStreamSyncPollerConcurrentRegisterUnregister(t *testing.T) { wg.Wait() } +// TestStreamSyncPollerStalledSubscriber verifies pollOnce completes even when +// a subscriber's consumer has stopped draining its update channel. The +// subscriber callback must stay nonblocking (drop-on-full); a blocking send +// would stall the shared poll loop for every chat on the replica. +func TestStreamSyncPollerStalledSubscriber(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + db.EXPECT().GetChatStreamSyncRows(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func(_ context.Context, ids []uuid.UUID) ([]database.GetChatStreamSyncRowsRow, error) { + rows := make([]database.GetChatStreamSyncRowsRow, 0, len(ids)) + for _, id := range ids { + rows = append(rows, database.GetChatStreamSyncRowsRow{ID: id}) + } + return rows, nil + }, + ) + + poller := newStreamSyncPoller(context.Background(), db, nil, slogtest.Make(t, nil)) + defer poller.Close() + + // A full, never-drained channel simulates a stalled consumer. The + // drop-on-full callback mirrors the consumer in subscribeStreamLoop. + stalled := make(chan streamSyncHint, 1) + stalled <- streamSyncHint{} + unregister := poller.Register(uuid.New(), func(hint streamSyncHint) { + select { + case stalled <- hint: + default: + } + }) + defer unregister() + + pollDone := make(chan struct{}) + go func() { + defer close(pollDone) + poller.pollOnce() + }() + select { + case <-pollDone: + case <-time.After(testutil.WaitShort): + t.Fatal("pollOnce blocked on a stalled subscriber") + } +} + // TestStreamSyncPollerNilRegister verifies a nil poller degrades to a no-op // registration instead of terminating subscribers. func TestStreamSyncPollerNilRegister(t *testing.T) { From 518736fbf9675c9d2eb9711ee5c70f4ee13d39cd Mon Sep 17 00:00:00 2001 From: Ethan Dickson Date: Wed, 2 Sep 2026 10:20:47 +0000 Subject: [PATCH 3/3] fix(coderd/x/chatd): avoid closing poller hint channels --- coderd/x/chatd/stream_subscribe.go | 14 +- coderd/x/chatd/stream_sync_poller.go | 28 ++-- .../chatd/stream_sync_poller_internal_test.go | 122 ------------------ 3 files changed, 21 insertions(+), 143 deletions(-) delete mode 100644 coderd/x/chatd/stream_sync_poller_internal_test.go diff --git a/coderd/x/chatd/stream_subscribe.go b/coderd/x/chatd/stream_subscribe.go index 0eed04ab10f..9545b707c11 100644 --- a/coderd/x/chatd/stream_subscribe.go +++ b/coderd/x/chatd/stream_subscribe.go @@ -58,12 +58,7 @@ func (p *Server) subscribeStreamLoop( return subscribeWithInitialError(chatID, "failed to subscribe to chat updates") } - unregisterPoller := p.streamSyncPoller.Register(chatID, func(hint streamSyncHint) { - select { - case updateCh <- hint: - default: - } - }) + pollerCh, unregisterPoller := p.streamSyncPoller.Register(chatID) loop := newStreamLoop(chat, p.db, logger, afterMessageID) // The immediate sync builds the initial snapshot returned to the caller // and the relay target for the forwarder. Hints only fire on state @@ -102,6 +97,13 @@ func (p *Server) subscribeStreamLoop( if !p.runStreamSync(streamCtx, loop, relay, events, hint) { return } + case hint, ok := <-pollerCh: + if !ok { + return + } + if !p.runStreamSync(streamCtx, loop, relay, events, hint) { + return + } case part, ok := <-relay.Parts(): if !ok { return diff --git a/coderd/x/chatd/stream_sync_poller.go b/coderd/x/chatd/stream_sync_poller.go index 676c387dd12..2a87a9af121 100644 --- a/coderd/x/chatd/stream_sync_poller.go +++ b/coderd/x/chatd/stream_sync_poller.go @@ -27,8 +27,8 @@ type streamSyncPoller struct { } type streamSyncPollerSubscriber struct { - chatID uuid.UUID - deliver func(streamSyncHint) + chatID uuid.UUID + hints chan streamSyncHint } func newStreamSyncPoller( @@ -68,20 +68,15 @@ func (p *streamSyncPoller) Close() { p.cancel() } -// Register subscribes deliver to poll hints for chatID until the returned -// unregister func is called. deliver is invoked outside the poller's mutex and -// may race with unregister, so it must remain safe to call after unregister -// returns (e.g. by sending to a channel that is never closed). deliver runs -// synchronously on the shared poll loop, so it must not block; a blocked -// callback stalls polling for every chat on the replica. Dropped hints are -// re-delivered on the next poll tick. -func (p *streamSyncPoller) Register(chatID uuid.UUID, deliver func(streamSyncHint)) (unregister func()) { +func (p *streamSyncPoller) Register(chatID uuid.UUID) (<-chan streamSyncHint, func()) { if p == nil { - return func() {} + ch := make(chan streamSyncHint) + close(ch) + return ch, func() {} } subscriber := &streamSyncPollerSubscriber{ - chatID: chatID, - deliver: deliver, + chatID: chatID, + hints: make(chan streamSyncHint, 1), } p.mu.Lock() if p.subscribers[chatID] == nil { @@ -90,7 +85,7 @@ func (p *streamSyncPoller) Register(chatID uuid.UUID, deliver func(streamSyncHin p.subscribers[chatID][subscriber] = struct{}{} p.mu.Unlock() - return func() { + return subscriber.hints, func() { p.unregister(subscriber) } } @@ -136,7 +131,10 @@ func (p *streamSyncPoller) pollOnce() { for _, row := range rows { hint := streamSyncHintFromPollRow(row) for _, subscriber := range subscribers[row.ID] { - subscriber.deliver(hint) + select { + case subscriber.hints <- hint: + default: + } } } } diff --git a/coderd/x/chatd/stream_sync_poller_internal_test.go b/coderd/x/chatd/stream_sync_poller_internal_test.go deleted file mode 100644 index cb89ab733e7..00000000000 --- a/coderd/x/chatd/stream_sync_poller_internal_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package chatd - -import ( - "context" - "sync" - "testing" - "time" - - "github.com/google/uuid" - "go.uber.org/mock/gomock" - - "cdr.dev/slog/v3/sloggers/slogtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbmock" - "github.com/coder/coder/v2/testutil" -) - -// TestStreamSyncPollerConcurrentRegisterUnregister churns subscriber -// registration while pollOnce delivers hints concurrently. Before the poller -// became a callback fanout, unregister closed the subscriber's hint channel -// while pollOnce could still be mid-send on its lock-free snapshot, panicking -// with "send on closed channel" (GHSA-7x3x-59xg-4hrc). Run with -race. -func TestStreamSyncPollerConcurrentRegisterUnregister(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - db := dbmock.NewMockStore(ctrl) - db.EXPECT().GetChatStreamSyncRows(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(_ context.Context, ids []uuid.UUID) ([]database.GetChatStreamSyncRowsRow, error) { - rows := make([]database.GetChatStreamSyncRowsRow, 0, len(ids)) - for _, id := range ids { - rows = append(rows, database.GetChatStreamSyncRowsRow{ID: id}) - } - return rows, nil - }, - ) - - poller := newStreamSyncPoller(context.Background(), db, nil, slogtest.Make(t, nil)) - defer poller.Close() - - chatID := uuid.New() - done := make(chan struct{}) - var wg sync.WaitGroup - for range 8 { - wg.Add(1) - go func() { - defer wg.Done() - for { - select { - case <-done: - return - default: - } - unregister := poller.Register(chatID, func(streamSyncHint) {}) - unregister() - } - }() - } - for range 1000 { - poller.pollOnce() - } - close(done) - wg.Wait() -} - -// TestStreamSyncPollerStalledSubscriber verifies pollOnce completes even when -// a subscriber's consumer has stopped draining its update channel. The -// subscriber callback must stay nonblocking (drop-on-full); a blocking send -// would stall the shared poll loop for every chat on the replica. -func TestStreamSyncPollerStalledSubscriber(t *testing.T) { - t.Parallel() - - ctrl := gomock.NewController(t) - db := dbmock.NewMockStore(ctrl) - db.EXPECT().GetChatStreamSyncRows(gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(_ context.Context, ids []uuid.UUID) ([]database.GetChatStreamSyncRowsRow, error) { - rows := make([]database.GetChatStreamSyncRowsRow, 0, len(ids)) - for _, id := range ids { - rows = append(rows, database.GetChatStreamSyncRowsRow{ID: id}) - } - return rows, nil - }, - ) - - poller := newStreamSyncPoller(context.Background(), db, nil, slogtest.Make(t, nil)) - defer poller.Close() - - // A full, never-drained channel simulates a stalled consumer. The - // drop-on-full callback mirrors the consumer in subscribeStreamLoop. - stalled := make(chan streamSyncHint, 1) - stalled <- streamSyncHint{} - unregister := poller.Register(uuid.New(), func(hint streamSyncHint) { - select { - case stalled <- hint: - default: - } - }) - defer unregister() - - pollDone := make(chan struct{}) - go func() { - defer close(pollDone) - poller.pollOnce() - }() - select { - case <-pollDone: - case <-time.After(testutil.WaitShort): - t.Fatal("pollOnce blocked on a stalled subscriber") - } -} - -// TestStreamSyncPollerNilRegister verifies a nil poller degrades to a no-op -// registration instead of terminating subscribers. -func TestStreamSyncPollerNilRegister(t *testing.T) { - t.Parallel() - - var poller *streamSyncPoller - unregister := poller.Register(uuid.New(), func(streamSyncHint) { - t.Fatal("nil poller must never deliver hints") - }) - unregister() -}