Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
16a4f03
feat: stem chat search terms with the english text search config
f0ssel Aug 19, 2026
471548e
fix: track chat search vector config instead of resetting in migration
f0ssel Aug 20, 2026
5115af5
Merge remote-tracking branch 'origin/main' into garrett/codagt-867-ba…
f0ssel Aug 20, 2026
4290fac
fix(coderd): renumber chat search migration to 000580
f0ssel Aug 20, 2026
930e3df
fix(coderd/database): query stale chat search vectors with their own …
f0ssel Aug 20, 2026
2f2580d
fix(coderd/database): reset english vectors in down migration for par…
f0ssel Aug 24, 2026
52f07ec
Merge remote-tracking branch 'origin/main' into garrett/codagt-867-ba…
f0ssel Aug 24, 2026
74dbd5c
fix(coderd): renumber chat search migration to 000585
f0ssel Aug 24, 2026
e7f7fc3
fix(coderd/database): reindex stale chat search vectors without an in…
f0ssel Aug 24, 2026
73a7c82
test(coderd/database/dbpurge): expect stale reindex query in mocked m…
f0ssel Aug 24, 2026
a49bea8
fix(coderd): keep title search on ILIKE and scope english config to m…
f0ssel Aug 24, 2026
36c522d
fix(coderd): scope english search config to message bodies only
f0ssel Aug 24, 2026
515e6a0
docs(coderd): scope stemming claim in chat search annotation to messa…
f0ssel Aug 24, 2026
1f84987
refactor(coderd): trim chat search comments per review
f0ssel Aug 25, 2026
5af097f
refactor(coderd/database): use an enum for chat_messages.search_tsv_c…
f0ssel Aug 25, 2026
e8d715c
refactor(coderd/database/dbpurge): drop chatSearchStaleDrained field …
f0ssel Aug 25, 2026
73ed324
refactor(coderd/database): drop query explanations from chats.sql
f0ssel Aug 25, 2026
a2459f8
fix(coderd/database): stamp existing vectors simple and match config …
f0ssel Aug 25, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -7187,6 +7187,13 @@ func (q *querier) RegisterWorkspaceProxy(ctx context.Context, arg database.Regis
return updateWithReturn(q.log, q.auth, fetch, q.db.RegisterWorkspaceProxy)(ctx, arg)
}

func (q *querier) ReindexStaleChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceChat); err != nil {
return 0, err
}
return q.db.ReindexStaleChatMessagesSearchTsv(ctx, batchSize)
}

func (q *querier) ReleaseExternalAuthLinkRefreshLease(ctx context.Context, arg database.ReleaseExternalAuthLinkRefreshLeaseParams) error {
fetch := func(ctx context.Context, arg database.ReleaseExternalAuthLinkRefreshLeaseParams) (database.ExternalAuthLink, error) {
return q.db.GetExternalAuthLink(ctx, database.GetExternalAuthLinkParams{UserID: arg.UserID, ProviderID: arg.ProviderID})
Expand Down
4 changes: 4 additions & 0 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1079,6 +1079,10 @@ func (s *MethodTestSuite) TestChats() {
dbm.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), int32(100)).Return(int64(0), nil).AnyTimes()
check.Args(int32(100)).Asserts(rbac.ResourceChat, policy.ActionUpdate)
}))
s.Run("ReindexStaleChatMessagesSearchTsv", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
dbm.EXPECT().ReindexStaleChatMessagesSearchTsv(gomock.Any(), int32(100)).Return(int64(0), nil).AnyTimes()
check.Args(int32(100)).Asserts(rbac.ResourceChat, policy.ActionUpdate)
}))
s.Run("GetChatRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
dbm.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(30), nil).AnyTimes()
check.Args().Asserts()
Expand Down
8 changes: 8 additions & 0 deletions coderd/database/dbmetrics/querymetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 30 additions & 0 deletions coderd/database/dbpurge/dbpurge.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,10 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.

chatConfigErr := errors.Join(chatRetentionErr, chatDebugRetentionErr)

// Set inside the transaction closure, applied to the instance only
// after the transaction commits.
var staleDrained bool

// Start a transaction to grab advisory lock, we don't want to run
// multiple purges at the same time (multiple replicas).
err := db.InTx(func(tx database.Store) error {
Expand Down Expand Up @@ -354,6 +358,28 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
}
}

// Rewrite vectors produced with a stale text search config
// ('simple' rows from before migration 000585, or rows written by
// an old binary mid rolling upgrade). Upgraded binaries always
// stamp search_tsv_config, so this backlog is finite: once a pass
// returns fewer rows than the batch size the scan has reached the
// end of the table and is skipped for the process lifetime.
var reindexedChatSearchRows int64
if !i.chatSearchStaleDrained {
for range i.chatSearchBackfillMaxBatches {
n, err := tx.ReindexStaleChatMessagesSearchTsv(ctx, i.chatSearchBackfillBatchSize)
if err != nil {
return xerrors.Errorf("reindex stale chat_messages.search_tsv: %w", err)
}
reindexedChatSearchRows += n
if n < int64(i.chatSearchBackfillBatchSize) {
staleDrained = true
break
}
}
}
backfilledChatSearchRows += reindexedChatSearchRows

i.logger.Debug(ctx, "purged old database entries",
slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs),
slog.F("expired_api_keys", expiredAPIKeys),
Expand Down Expand Up @@ -399,6 +425,9 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
if err != nil {
return err
}
if staleDrained {
i.chatSearchStaleDrained = true
}

// Surface the deferred chat-config error so doTick records
// the failed iteration metric.
Expand All @@ -420,6 +449,7 @@ type instance struct {
chatSearchRowsBackfilled prometheus.Counter
chatSearchBackfillBatchSize int32
chatSearchBackfillMaxBatches int
chatSearchStaleDrained bool
}

func (i *instance) Close() error {
Expand Down
83 changes: 80 additions & 3 deletions coderd/database/dbpurge/dbpurge_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ func TestMetrics(t *testing.T) {
mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().ReindexStaleChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
Expand Down Expand Up @@ -308,6 +309,7 @@ func TestMetrics(t *testing.T) {
mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().ReindexStaleChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1)
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
Expand Down Expand Up @@ -3113,15 +3115,19 @@ func TestBackfillChatMessagesSearchTsv(t *testing.T) {
isNull, _ := searchTsv(ctx, t, rawDB, id)
require.False(t, isNull, msg)
}
// Asserts the row's tsvector matches expectedText, not just non-NULL.
// Asserts the row's tsvector matches expectedText and that the sweep
// stamped the config that produced it.
requireTsvFor := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64, expectedText string) {
t.Helper()
var matches bool
var config sql.NullString
err := rawDB.QueryRowContext(ctx,
"SELECT search_tsv = to_tsvector('simple', $2::text) FROM chat_messages WHERE id = $1", id, expectedText).
Scan(&matches)
"SELECT search_tsv = to_tsvector('english', $2::text), search_tsv_config FROM chat_messages WHERE id = $1", id, expectedText).
Scan(&matches, &config)
require.NoError(t, err)
require.True(t, matches, "search_tsv should contain the lexemes of %q", expectedText)
require.Equal(t, sql.NullString{String: "english", Valid: true}, config,
"backfilled rows must record the config that produced the vector")
}
requireNotBackfilled := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64, msg string) {
t.Helper()
Expand Down Expand Up @@ -3284,6 +3290,76 @@ func TestBackfillChatMessagesSearchTsv(t *testing.T) {
requireBackfilled(ctx, t, rawDB, fresh.ID, "message inserted after drain should be backfilled on the next tick")
})

//nolint:paralleltest // It uses LockIDDBPurge.
t.Run("ReindexesStaleConfigVectors", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
clk := quartz.NewMock(t)
clk.Set(now).MustWait(ctx)
db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure())
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
deps := setupDeps(t, db)

countStale := func() int {
var count int
err := rawDB.QueryRowContext(ctx, `
SELECT COUNT(*) FROM chat_messages
WHERE search_tsv IS NOT NULL
AND search_tsv_config IS DISTINCT FROM 'english'
AND deleted = false
AND visibility IN ('user', 'both')
AND role IN ('user', 'assistant')`).Scan(&count)
require.NoError(t, err)
return count
}
makeStale := func(id int64, config sql.NullString) {
// 'simple' simulates a vector stamped by migration 000585; NULL
// simulates a binary that predates search_tsv_config (e.g. an
// old replica winning the dbpurge lock mid rolling upgrade).
_, err := rawDB.ExecContext(ctx, `
UPDATE chat_messages
SET search_tsv = to_tsvector('simple', chat_message_search_text(content)),
search_tsv_config = $2
WHERE id = $1`, id, config)
require.NoError(t, err)
}
simpleConfig := sql.NullString{String: "simple", Valid: true}
nullConfig := sql.NullString{}

preexisting := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("refactoring the deployment"))
makeStale(preexisting.ID, simpleConfig)
oldBinary := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("rotating the credentials"))
makeStale(oldBinary.ID, nullConfig)
require.Equal(t, 2, countStale())

tick := awaitDoTicks(ctx, t, clk, 2)
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk))

// The first tick rewrites the stale backlog with 'english' and
// latches off the stale scan for the process lifetime.
tick()
requireTsvFor(ctx, t, rawDB, preexisting.ID, "refactoring the deployment")
requireTsvFor(ctx, t, rawDB, oldBinary.ID, "rotating the credentials")
require.Zero(t, countStale())

// A stale row appearing after the latch (old replica racing the
// tail of a rolling upgrade) is intentionally not rewritten by
// this process; it is repaired after a restart re-runs one stale
// pass.
makeStale(preexisting.ID, nullConfig)
tick()
require.Equal(t, 1, countStale(), "stale scan must stay latched off after drain")
require.NoError(t, closer.Close())

// A new dbpurge instance (process restart) re-runs one stale pass
// and repairs the row.
tick = awaitDoTicks(ctx, t, clk, 1)
closer = dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk))
defer closer.Close()
tick()
requireTsvFor(ctx, t, rawDB, preexisting.ID, "refactoring the deployment")
require.Zero(t, countStale())
})

//nolint:paralleltest // It uses LockIDDBPurge.
t.Run("SteadyStateNoop", func(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitLong)
Expand Down Expand Up @@ -3341,6 +3417,7 @@ func TestBackfillChatMessagesSearchTsv(t *testing.T) {
Return(int32(0), nil).AnyTimes()
mDB.EXPECT().TryAcquireLock(gomock.Any(), int64(database.LockIDDBPurge)).Return(false, nil).AnyTimes()
mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Times(0)
mDB.EXPECT().ReindexStaleChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Times(0)
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
return f(mDB)
Expand Down
Loading
Loading