Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e4b6f56

Browse files
kylecarbskylecarbs
authored
chore: separate pubsub into a new package (#8017)
* chore: rename store to dbmock for consistency * chore: remove redundant dbtype package This wasn't necessary and forked how we do DB types. * chore: separate pubsub into a new package This didn't need to be in database and was bloating it.
1 parent 2c843f4 commit e4b6f56

17 files changed

+95
-85
lines changed

‎cli/server.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ import (
6868
"github.com/coder/coder/coderd/database/dbmetrics"
6969
"github.com/coder/coder/coderd/database/dbpurge"
7070
"github.com/coder/coder/coderd/database/migrations"
71+
"github.com/coder/coder/coderd/database/pubsub"
7172
"github.com/coder/coder/coderd/devtunnel"
7273
"github.com/coder/coder/coderd/gitauth"
7374
"github.com/coder/coder/coderd/gitsshkey"
@@ -463,7 +464,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
463464
Logger: logger.Named("coderd"),
464465
Database: dbfake.New(),
465466
DERPMap: derpMap,
466-
Pubsub: database.NewPubsubInMemory(),
467+
Pubsub: pubsub.NewInMemory(),
467468
CacheDir: cacheDir,
468469
GoogleTokenValidator: googleTokenValidator,
469470
GitAuthConfigs: gitAuthConfigs,
@@ -589,7 +590,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
589590
if cfg.InMemoryDatabase {
590591
// This is only used for testing.
591592
options.Database = dbmetrics.New(dbfake.New(), options.PrometheusRegistry)
592-
options.Pubsub = database.NewPubsubInMemory()
593+
options.Pubsub = pubsub.NewInMemory()
593594
} else {
594595
sqlDB, err := connectToPostgres(ctx, logger, sqlDriver, cfg.PostgresURL.String())
595596
if err != nil {
@@ -600,7 +601,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
600601
}()
601602

602603
options.Database = dbmetrics.New(database.New(sqlDB), options.PrometheusRegistry)
603-
options.Pubsub, err = database.NewPubsub(ctx, sqlDB, cfg.PostgresURL.String())
604+
options.Pubsub, err = pubsub.New(ctx, sqlDB, cfg.PostgresURL.String())
604605
if err != nil {
605606
return xerrors.Errorf("create pubsub: %w", err)
606607
}

‎coderd/coderd.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import (
4848
"github.com/coder/coder/coderd/database"
4949
"github.com/coder/coder/coderd/database/dbauthz"
5050
"github.com/coder/coder/coderd/database/dbmetrics"
51+
"github.com/coder/coder/coderd/database/pubsub"
5152
"github.com/coder/coder/coderd/gitauth"
5253
"github.com/coder/coder/coderd/gitsshkey"
5354
"github.com/coder/coder/coderd/healthcheck"
@@ -95,7 +96,7 @@ type Options struct {
9596
AppHostnameRegex *regexp.Regexp
9697
Logger slog.Logger
9798
Database database.Store
98-
Pubsub database.Pubsub
99+
Pubsub pubsub.Pubsub
99100

100101
// CacheDir is used for caching files served by the API.
101102
CacheDir string

‎coderd/coderdtest/coderdtest.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ import (
5959
"github.com/coder/coder/coderd/database"
6060
"github.com/coder/coder/coderd/database/dbauthz"
6161
"github.com/coder/coder/coderd/database/dbtestutil"
62+
"github.com/coder/coder/coderd/database/pubsub"
6263
"github.com/coder/coder/coderd/gitauth"
6364
"github.com/coder/coder/coderd/gitsshkey"
6465
"github.com/coder/coder/coderd/healthcheck"
@@ -130,7 +131,7 @@ type Options struct {
130131
// It should only be used in cases where multiple Coder
131132
// test instances are running against the same database.
132133
Database database.Store
133-
Pubsub database.Pubsub
134+
Pubsub pubsub.Pubsub
134135

135136
ConfigSSH codersdk.SSHConfigResponse
136137

‎coderd/database/dbtestutil/db.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@ import (
1111
"github.com/coder/coder/coderd/database"
1212
"github.com/coder/coder/coderd/database/dbfake"
1313
"github.com/coder/coder/coderd/database/postgres"
14+
"github.com/coder/coder/coderd/database/pubsub"
1415
)
1516

16-
func NewDB(t testing.TB) (database.Store, database.Pubsub) {
17+
func NewDB(t testing.TB) (database.Store, pubsub.Pubsub) {
1718
t.Helper()
1819

1920
db := dbfake.New()
20-
pubsub := database.NewPubsubInMemory()
21+
ps := pubsub.NewInMemory()
2122
if os.Getenv("DB") != "" {
2223
connectionURL := os.Getenv("CODER_PG_CONNECTION_URL")
2324
if connectionURL == "" {
@@ -36,12 +37,12 @@ func NewDB(t testing.TB) (database.Store, database.Pubsub) {
3637
})
3738
db = database.New(sqlDB)
3839

39-
pubsub, err = database.NewPubsub(context.Background(), sqlDB, connectionURL)
40+
ps, err = pubsub.New(context.Background(), sqlDB, connectionURL)
4041
require.NoError(t, err)
4142
t.Cleanup(func() {
42-
_ = pubsub.Close()
43+
_ = ps.Close()
4344
})
4445
}
4546

46-
return db, pubsub
47+
return db, ps
4748
}

‎coderd/database/pubsub.go renamed to ‎coderd/database/pubsub/pubsub.go

+13-13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package database
1+
package pubsub
22

33
import (
44
"context"
@@ -48,7 +48,7 @@ type msgOrErr struct {
4848
type msgQueue struct {
4949
ctx context.Context
5050
cond *sync.Cond
51-
q [PubsubBufferSize]msgOrErr
51+
q [BufferSize]msgOrErr
5252
front int
5353
size int
5454
closed bool
@@ -82,7 +82,7 @@ func (q *msgQueue) run() {
8282
return
8383
}
8484
item := q.q[q.front]
85-
q.front = (q.front + 1) % PubsubBufferSize
85+
q.front = (q.front + 1) % BufferSize
8686
q.size--
8787
q.cond.L.Unlock()
8888

@@ -111,20 +111,20 @@ func (q *msgQueue) enqueue(msg []byte) {
111111
q.cond.L.Lock()
112112
defer q.cond.L.Unlock()
113113

114-
if q.size == PubsubBufferSize {
114+
if q.size == BufferSize {
115115
// queue is full, so we're going to drop the msg we got called with.
116116
// We also need to record that messages are being dropped, which we
117117
// do at the last message in the queue. This potentially makes us
118118
// lose 2 messages instead of one, but it's more important at this
119119
// point to warn the subscriber that they're losing messages so they
120120
// can do something about it.
121-
back := (q.front + PubsubBufferSize - 1) % PubsubBufferSize
121+
back := (q.front + BufferSize - 1) % BufferSize
122122
q.q[back].msg = nil
123123
q.q[back].err = ErrDroppedMessages
124124
return
125125
}
126126
// queue is not full, insert the message
127-
next := (q.front + q.size) % PubsubBufferSize
127+
next := (q.front + q.size) % BufferSize
128128
q.q[next].msg = msg
129129
q.q[next].err = nil
130130
q.size++
@@ -143,17 +143,17 @@ func (q *msgQueue) dropped() {
143143
q.cond.L.Lock()
144144
defer q.cond.L.Unlock()
145145

146-
if q.size == PubsubBufferSize {
146+
if q.size == BufferSize {
147147
// queue is full, but we need to record that messages are being dropped,
148148
// which we do at the last message in the queue. This potentially drops
149149
// another message, but it's more important for the subscriber to know.
150-
back := (q.front + PubsubBufferSize - 1) % PubsubBufferSize
150+
back := (q.front + BufferSize - 1) % BufferSize
151151
q.q[back].msg = nil
152152
q.q[back].err = ErrDroppedMessages
153153
return
154154
}
155155
// queue is not full, insert the error
156-
next := (q.front + q.size) % PubsubBufferSize
156+
next := (q.front + q.size) % BufferSize
157157
q.q[next].msg = nil
158158
q.q[next].err = ErrDroppedMessages
159159
q.size++
@@ -171,9 +171,9 @@ type pgPubsub struct {
171171
queues map[string]map[uuid.UUID]*msgQueue
172172
}
173173

174-
// PubsubBufferSize is the maximum number of unhandled messages we will buffer
174+
// BufferSize is the maximum number of unhandled messages we will buffer
175175
// for a subscriber before dropping messages.
176-
const PubsubBufferSize = 2048
176+
const BufferSize = 2048
177177

178178
// Subscribe calls the listener when an event matching the name is received.
179179
func (p *pgPubsub) Subscribe(event string, listener Listener) (cancel func(), err error) {
@@ -295,8 +295,8 @@ func (p *pgPubsub) recordReconnect() {
295295
}
296296
}
297297

298-
// NewPubsub creates a new Pubsub implementation using a PostgreSQL connection.
299-
func NewPubsub(ctx context.Context, database *sql.DB, connectURL string) (Pubsub, error) {
298+
// New creates a new Pubsub implementation using a PostgreSQL connection.
299+
func New(ctx context.Context, database *sql.DB, connectURL string) (Pubsub, error) {
300300
// Creates a new listener using pq.
301301
errCh := make(chan error)
302302
listener := pq.NewListener(connectURL, time.Second, time.Minute, func(_ pq.ListenerEventType, err error) {

‎coderd/database/pubsub_internal_test.go renamed to ‎coderd/database/pubsub/pubsub_internal_test.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package database
1+
package pubsub
22

33
import (
44
"context"
@@ -26,7 +26,7 @@ func Test_msgQueue_ListenerWithError(t *testing.T) {
2626
// PubsubBufferSize is 2048, which is a power of 2, so a pattern of 5 will not be aligned
2727
// when we wrap around the end of the circular buffer. This tests that we correctly handle
2828
// the wrapping and aren't dequeueing misaligned data.
29-
cycles := (PubsubBufferSize / 5) * 2 // almost twice around the ring
29+
cycles := (BufferSize / 5) * 2 // almost twice around the ring
3030
for j := 0; j < cycles; j++ {
3131
for i := 0; i < 4; i++ {
3232
uut.enqueue([]byte(fmt.Sprintf("%d%d", j, i)))
@@ -75,7 +75,7 @@ func Test_msgQueue_Listener(t *testing.T) {
7575
// PubsubBufferSize is 2048, which is a power of 2, so a pattern of 5 will not be aligned
7676
// when we wrap around the end of the circular buffer. This tests that we correctly handle
7777
// the wrapping and aren't dequeueing misaligned data.
78-
cycles := (PubsubBufferSize / 5) * 2 // almost twice around the ring
78+
cycles := (BufferSize / 5) * 2 // almost twice around the ring
7979
for j := 0; j < cycles; j++ {
8080
for i := 0; i < 4; i++ {
8181
uut.enqueue([]byte(fmt.Sprintf("%d%d", j, i)))
@@ -119,7 +119,7 @@ func Test_msgQueue_Full(t *testing.T) {
119119
// we send 2 more than the capacity. One extra because the call to the ListenerFunc blocks
120120
// but only after we've dequeued a message, and then another extra because we want to exceed
121121
// the capacity, not just reach it.
122-
for i := 0; i < PubsubBufferSize+2; i++ {
122+
for i := 0; i < BufferSize+2; i++ {
123123
uut.enqueue([]byte(fmt.Sprintf("%d", i)))
124124
// ensure the first dequeue has happened before proceeding, so that this function isn't racing
125125
// against the goroutine that dequeues items.
@@ -136,5 +136,5 @@ func Test_msgQueue_Full(t *testing.T) {
136136
// Ok, so we sent 2 more than capacity, but we only read the capacity, that's because the last
137137
// message we send doesn't get queued, AND, it bumps a message out of the queue to make room
138138
// for the error, so we read 2 less than we sent.
139-
require.Equal(t, PubsubBufferSize, n)
139+
require.Equal(t, BufferSize, n)
140140
}

‎coderd/database/pubsub_memory.go renamed to ‎coderd/database/pubsub/pubsub_memory.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package database
1+
package pubsub
22

33
import (
44
"context"
@@ -87,7 +87,7 @@ func (*memoryPubsub) Close() error {
8787
return nil
8888
}
8989

90-
func NewPubsubInMemory() Pubsub {
90+
func NewInMemory() Pubsub {
9191
return &memoryPubsub{
9292
listeners: make(map[string]map[uuid.UUID]genericListener),
9393
}

‎coderd/database/pubsub_memory_test.go renamed to ‎coderd/database/pubsub/pubsub_memory_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package database_test
1+
package pubsub_test
22

33
import (
44
"context"
@@ -7,7 +7,7 @@ import (
77
"github.com/stretchr/testify/assert"
88
"github.com/stretchr/testify/require"
99

10-
"github.com/coder/coder/coderd/database"
10+
"github.com/coder/coder/coderd/database/pubsub"
1111
)
1212

1313
func TestPubsubMemory(t *testing.T) {
@@ -16,7 +16,7 @@ func TestPubsubMemory(t *testing.T) {
1616
t.Run("Legacy", func(t *testing.T) {
1717
t.Parallel()
1818

19-
pubsub := database.NewPubsubInMemory()
19+
pubsub := pubsub.NewInMemory()
2020
event := "test"
2121
data := "testing"
2222
messageChannel := make(chan []byte)
@@ -36,7 +36,7 @@ func TestPubsubMemory(t *testing.T) {
3636
t.Run("WithErr", func(t *testing.T) {
3737
t.Parallel()
3838

39-
pubsub := database.NewPubsubInMemory()
39+
pubsub := pubsub.NewInMemory()
4040
event := "test"
4141
data := "testing"
4242
messageChannel := make(chan []byte)

0 commit comments

Comments
 (0)