From 78b9472ad22d9db7c636d2fb182a9dc2621fc8fc Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 Jul 2026 15:27:01 +0000 Subject: [PATCH 01/12] feat(coderd): add chat search schema This commit adds the required schema for chat search: - search_tsv tsvector column on chat_messages (default NULL). It is not queried or referenced directly in Go code. - Partial GIN full-text search index on chat_messages.search_tsv where non-null - btree index on chat_messages.search_tsv where null - GIN full-text-search indexes on chats.title and chat_diff_statuses.pull_request_title - chat_message_search_text stored function to extract message content It also modifies the existing trigger functions set_chat_message_revision_before and update_chat_history_after_message_update to disregard search_tsv. This is needed to avoid spurious chat history revision updates. --- coderd/database/dump.sql | 29 ++- .../000541_chat_search_schema.down.sql | 68 +++++ .../000541_chat_search_schema.up.sql | 91 +++++++ coderd/database/migrations/migrate_test.go | 243 ++++++++++++++++++ coderd/database/models.go | 1 + coderd/database/queries.sql.go | 24 +- coderd/x/chatd/chatstate/trigger_test.go | 74 ++++++ 7 files changed, 519 insertions(+), 11 deletions(-) create mode 100644 coderd/database/migrations/000541_chat_search_schema.down.sql create mode 100644 coderd/database/migrations/000541_chat_search_schema.up.sql diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index ed1680dec42..3e70b0ce0e8 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -766,6 +766,16 @@ BEGIN END; $$; +CREATE FUNCTION chat_message_search_text(content jsonb) RETURNS text + LANGUAGE sql IMMUTABLE PARALLEL SAFE + AS $$ + SELECT CASE WHEN jsonb_typeof(content) = 'array' THEN ( + SELECT string_agg(part->>'text', ' ' ORDER BY ordinality) + FROM jsonb_array_elements(content) WITH ORDINALITY AS t(part, ordinality) + WHERE part->>'type' = 'text' + ) END +$$; + CREATE FUNCTION check_workspace_agent_name_unique() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1348,6 +1358,7 @@ CREATE FUNCTION set_chat_message_revision_before() RETURNS trigger AS $$ DECLARE chat_snapshot_version bigint; + cmp chat_messages; BEGIN IF TG_OP = 'INSERT' AND NEW.revision IS NOT NULL THEN RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; @@ -1362,7 +1373,9 @@ BEGIN RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; END IF; - IF OLD IS NOT DISTINCT FROM NEW THEN + cmp := NEW; + cmp.search_tsv := OLD.search_tsv; + IF OLD IS NOT DISTINCT FROM cmp THEN RETURN NEW; END IF; END IF; @@ -1429,7 +1442,8 @@ BEGIN SELECT DISTINCT n.chat_id FROM chat_message_history_new_rows n JOIN chat_message_history_old_rows o ON o.id = n.id - WHERE o IS DISTINCT FROM n + -- jsonb-minus here: transition-table rows have no composite-copy idiom in pure SQL. + WHERE (to_jsonb(o) - 'search_tsv') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv') ) AS affected WHERE c.id = affected.chat_id AND ( @@ -1922,7 +1936,8 @@ CREATE TABLE chat_messages ( deleted boolean DEFAULT false NOT NULL, provider_response_id text, api_key_id text, - revision bigint NOT NULL + revision bigint NOT NULL, + search_tsv tsvector ); CREATE SEQUENCE chat_messages_id_seq @@ -4683,6 +4698,8 @@ CREATE UNIQUE INDEX idx_chat_debug_steps_run_step ON chat_debug_steps USING btre CREATE INDEX idx_chat_debug_steps_stale ON chat_debug_steps USING btree (updated_at) WHERE (finished_at IS NULL); +CREATE INDEX idx_chat_diff_statuses_pr_title_fts ON chat_diff_statuses USING gin (to_tsvector('simple'::regconfig, pull_request_title)); + CREATE INDEX idx_chat_diff_statuses_stale_at ON chat_diff_statuses USING btree (stale_at); CREATE INDEX idx_chat_diff_statuses_url_lower ON chat_diff_statuses USING btree (lower(url)) WHERE ((url IS NOT NULL) AND (url <> ''::text)); @@ -4703,6 +4720,10 @@ CREATE INDEX idx_chat_messages_created_at ON chat_messages USING btree (created_ CREATE INDEX idx_chat_messages_owner_spend ON chat_messages USING btree (chat_id, created_at) WHERE (total_cost_micros IS NOT NULL); +CREATE INDEX idx_chat_messages_search_tsv ON chat_messages USING gin (search_tsv) WHERE ((search_tsv IS NOT NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); + +CREATE INDEX idx_chat_messages_search_tsv_pending ON chat_messages USING btree (id DESC) WHERE ((search_tsv IS NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); + CREATE INDEX idx_chat_messages_user_prompts ON chat_messages USING btree (chat_id, id DESC) WHERE ((deleted = false) AND (role = 'user'::chat_message_role) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility]))); CREATE INDEX idx_chat_model_configs_ai_provider_id ON chat_model_configs USING btree (ai_provider_id); @@ -4731,6 +4752,8 @@ CREATE INDEX idx_chats_pending ON chats USING btree (status) WHERE (status = 'pe CREATE INDEX idx_chats_root_chat_id ON chats USING btree (root_chat_id); +CREATE INDEX idx_chats_title_fts ON chats USING gin (to_tsvector('simple'::regconfig, title)); + CREATE INDEX idx_chats_worker_acquisition_candidates ON chats USING btree (status, updated_at, id) WHERE (archived = false); CREATE INDEX idx_chats_workspace ON chats USING btree (workspace_id); diff --git a/coderd/database/migrations/000541_chat_search_schema.down.sql b/coderd/database/migrations/000541_chat_search_schema.down.sql new file mode 100644 index 00000000000..05de502cc2e --- /dev/null +++ b/coderd/database/migrations/000541_chat_search_schema.down.sql @@ -0,0 +1,68 @@ +-- Restore the original trigger bodies from 000519. +CREATE OR REPLACE FUNCTION set_chat_message_revision_before() +RETURNS trigger AS $$ +DECLARE + chat_snapshot_version bigint; +BEGIN + IF TG_OP = 'INSERT' AND NEW.revision IS NOT NULL THEN + RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF OLD.chat_id IS DISTINCT FROM NEW.chat_id THEN + RAISE EXCEPTION 'chat_messages.chat_id is immutable'; + END IF; + + IF OLD.revision IS DISTINCT FROM NEW.revision THEN + RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; + END IF; + + IF OLD IS NOT DISTINCT FROM NEW THEN + RETURN NEW; + END IF; + END IF; + + SELECT snapshot_version INTO chat_snapshot_version + FROM chats WHERE id = NEW.chat_id; + + IF chat_snapshot_version IS NULL THEN + RAISE EXCEPTION 'chat % does not exist', NEW.chat_id; + END IF; + + NEW.revision = chat_snapshot_version; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION update_chat_history_after_message_update() +RETURNS trigger AS $$ +BEGIN + UPDATE chats c + SET history_version = c.snapshot_version, + generation_attempt = 0 + FROM ( + SELECT DISTINCT n.chat_id + FROM chat_message_history_new_rows n + JOIN chat_message_history_old_rows o ON o.id = n.id + WHERE o IS DISTINCT FROM n + ) AS affected + WHERE c.id = affected.chat_id + AND ( + c.history_version IS DISTINCT FROM c.snapshot_version + OR c.generation_attempt <> 0 + ); + RETURN NULL; +END; +$$ LANGUAGE plpgsql; + +DROP INDEX IF EXISTS idx_chat_diff_statuses_pr_title_fts; + +DROP INDEX IF EXISTS idx_chats_title_fts; + +DROP INDEX IF EXISTS idx_chat_messages_search_tsv_pending; + +DROP INDEX IF EXISTS idx_chat_messages_search_tsv; + +ALTER TABLE chat_messages DROP COLUMN IF EXISTS search_tsv; + +DROP FUNCTION IF EXISTS chat_message_search_text(jsonb); diff --git a/coderd/database/migrations/000541_chat_search_schema.up.sql b/coderd/database/migrations/000541_chat_search_schema.up.sql new file mode 100644 index 00000000000..eaa4ec95f29 --- /dev/null +++ b/coderd/database/migrations/000541_chat_search_schema.up.sql @@ -0,0 +1,91 @@ +-- CASE guard: jsonb_array_elements raises on non-array input. Legacy +-- content_version=0 rows store scalar JSON strings; excluded from search +-- by design. IMMUTABLE for expression index use. +CREATE FUNCTION chat_message_search_text(content jsonb) RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE WHEN jsonb_typeof(content) = 'array' THEN ( + SELECT string_agg(part->>'text', ' ' ORDER BY ordinality) + FROM jsonb_array_elements(content) WITH ORDINALITY AS t(part, ordinality) + WHERE part->>'type' = 'text' + ) END +$$; + +-- Populated by a background sweep, not at insert time. NULL means pending. +ALTER TABLE chat_messages ADD COLUMN search_tsv tsvector; + +CREATE INDEX idx_chat_messages_search_tsv ON chat_messages +USING GIN (search_tsv) +WHERE ((search_tsv IS NOT NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); + +-- Sweep uses this to find pending rows. Queries must repeat the full predicate. +CREATE INDEX idx_chat_messages_search_tsv_pending ON chat_messages USING btree (id DESC) +WHERE ((search_tsv IS NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); + +CREATE INDEX idx_chats_title_fts ON chats + USING GIN (to_tsvector('simple', title)); + +CREATE INDEX idx_chat_diff_statuses_pr_title_fts ON chat_diff_statuses + USING GIN (to_tsvector('simple', pull_request_title)); + +-- search_tsv is system-maintained; the backfill must not perturb message +-- revision, chat history_version, generation_attempt, or retry state. +-- Both triggers therefore ignore changes confined to search_tsv. +CREATE OR REPLACE FUNCTION set_chat_message_revision_before() +RETURNS trigger AS $$ +DECLARE + chat_snapshot_version bigint; + cmp chat_messages; +BEGIN + IF TG_OP = 'INSERT' AND NEW.revision IS NOT NULL THEN + RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; + END IF; + + IF TG_OP = 'UPDATE' THEN + IF OLD.chat_id IS DISTINCT FROM NEW.chat_id THEN + RAISE EXCEPTION 'chat_messages.chat_id is immutable'; + END IF; + + IF OLD.revision IS DISTINCT FROM NEW.revision THEN + RAISE EXCEPTION 'chat_messages.revision must be assigned by trigger'; + END IF; + + cmp := NEW; + cmp.search_tsv := OLD.search_tsv; + IF OLD IS NOT DISTINCT FROM cmp THEN + RETURN NEW; + END IF; + END IF; + + SELECT snapshot_version INTO chat_snapshot_version + FROM chats WHERE id = NEW.chat_id; + + IF chat_snapshot_version IS NULL THEN + RAISE EXCEPTION 'chat % does not exist', NEW.chat_id; + END IF; + + NEW.revision = chat_snapshot_version; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION update_chat_history_after_message_update() +RETURNS trigger AS $$ +BEGIN + UPDATE chats c + SET history_version = c.snapshot_version, + generation_attempt = 0 + FROM ( + SELECT DISTINCT n.chat_id + FROM chat_message_history_new_rows n + JOIN chat_message_history_old_rows o ON o.id = n.id + -- jsonb-minus here: transition-table rows have no composite-copy idiom in pure SQL. + WHERE (to_jsonb(o) - 'search_tsv') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv') + ) AS affected + WHERE c.id = affected.chat_id + AND ( + c.history_version IS DISTINCT FROM c.snapshot_version + OR c.generation_attempt <> 0 + ); + RETURN NULL; +END; +$$ LANGUAGE plpgsql; diff --git a/coderd/database/migrations/migrate_test.go b/coderd/database/migrations/migrate_test.go index f148860bc5f..769184d8898 100644 --- a/coderd/database/migrations/migrate_test.go +++ b/coderd/database/migrations/migrate_test.go @@ -19,11 +19,13 @@ import ( "github.com/golang-migrate/migrate/v4/source/stub" "github.com/google/uuid" "github.com/lib/pq" + "github.com/sqlc-dev/pqtype" "github.com/stretchr/testify/require" "go.uber.org/goleak" "golang.org/x/sync/errgroup" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/migrations" "github.com/coder/coder/v2/testutil" @@ -1792,3 +1794,244 @@ func TestMigration000498SoftDeleteStaleWorkspaceAgents(t *testing.T) { // TestSoftDeleteWorkspaceAgentsByWorkspaceID, plus integration tests // under coderd/coderd_test.go; not retested here. } + +func TestMigration000541ChatMessageSearchText(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + sqlDB := testSQLDB(t) + require.NoError(t, migrations.Up(sqlDB)) + + cases := []struct { + name string + content sql.NullString + want sql.NullString + }{ + { + name: "SingleTextPart", + content: sql.NullString{String: `[{"type":"text","text":"hello world"}]`, Valid: true}, + want: sql.NullString{String: "hello world", Valid: true}, + }, + { + name: "TextInterleavedWithNonText", + content: sql.NullString{String: `[ + {"type":"text","text":"first"}, + {"type":"reasoning","text":"thinking"}, + {"type":"tool-call","toolName":"execute"}, + {"type":"text","text":"second"} + ]`, Valid: true}, + want: sql.NullString{String: "first second", Valid: true}, + }, + { + name: "OnlyNonTextParts", + content: sql.NullString{String: `[{"type":"reasoning","text":"thinking"}]`, Valid: true}, + want: sql.NullString{}, + }, + { + name: "ScalarContent", + content: sql.NullString{String: `"hello"`, Valid: true}, + want: sql.NullString{}, + }, + { + name: "EmptyArray", + content: sql.NullString{String: `[]`, Valid: true}, + want: sql.NullString{}, + }, + { + name: "NullInput", + content: sql.NullString{}, + want: sql.NullString{}, + }, + { + name: "ElementsMissingTypeOrText", + content: sql.NullString{String: `[{"text":"no type"},{"type":"text"},{"type":"text","text":"kept"}]`, Valid: true}, + want: sql.NullString{String: "kept", Valid: true}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + var got sql.NullString + err := sqlDB.QueryRowContext(ctx, + `SELECT chat_message_search_text($1::jsonb)`, tc.content, + ).Scan(&got) + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } +} + +// Shared eligibility predicate of the two partial chat_messages search +// indexes. Queries must repeat it verbatim. +const eligibilityPredicate = `deleted = false + AND visibility IN ('user', 'both') + AND role IN ('user', 'assistant')` + +func TestMigration000541ChatSearchSchemaIndexes(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + sqlDB := testSQLDB(t) + require.NoError(t, migrations.Up(sqlDB)) + + cases := []struct { + name string + table string + partial bool + }{ + {name: "idx_chat_messages_search_tsv", table: "chat_messages", partial: true}, + {name: "idx_chat_messages_search_tsv_pending", table: "chat_messages", partial: true}, + {name: "idx_chats_title_fts", table: "chats", partial: false}, + {name: "idx_chat_diff_statuses_pr_title_fts", table: "chat_diff_statuses", partial: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + var table string + var partial bool + err := sqlDB.QueryRowContext(ctx, ` + SELECT i.tablename, x.indpred IS NOT NULL + FROM pg_indexes i + JOIN pg_class c ON c.relname = i.indexname + JOIN pg_index x ON x.indexrelid = c.oid + WHERE i.indexname = $1`, tc.name, + ).Scan(&table, &partial) + require.NoError(t, err, "index %s should exist", tc.name) + require.Equal(t, tc.table, table, "index %s table", tc.name) + require.Equal(t, tc.partial, partial, "index %s partial", tc.name) + }) + } +} + +func TestMigration000541ChatSearchSchemaBehavior(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + sqlDB := testSQLDB(t) + require.NoError(t, migrations.Up(sqlDB)) + db := database.New(sqlDB) + ctx := testutil.Context(t, testutil.WaitLong) + + org := dbgen.Organization(t, db, database.Organization{}) + owner := dbgen.User(t, db, database.User{}) + _ = dbgen.ChatProvider(t, db, database.ChatProvider{Provider: "openai", DisplayName: "OpenAI"}) + modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ + CreatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true}, + IsDefault: true, + }) + chat := dbgen.Chat(t, db, database.Chat{ + OrganizationID: org.ID, + OwnerID: owner.ID, + LastModelConfigID: modelCfg.ID, + }) + + newMsg := func(role database.ChatMessageRole, visibility database.ChatMessageVisibility, content string) database.ChatMessage { + seed := database.ChatMessage{ + ChatID: chat.ID, + CreatedBy: uuid.NullUUID{UUID: owner.ID, Valid: true}, + Role: role, + Visibility: visibility, + } + if content != "" { + seed.Content = pqtype.NullRawMessage{RawMessage: []byte(content), Valid: true} + } + return dbgen.ChatMessage(t, db, seed) + } + textContent := func(text string) string { + return `[{"type":"text","text":"` + text + `"}]` + } + + pendingIDs := func(ctx context.Context, limit int) []int64 { + rows, err := sqlDB.QueryContext(ctx, ` + SELECT id FROM chat_messages + WHERE search_tsv IS NULL AND `+eligibilityPredicate+` + ORDER BY id DESC + LIMIT $1`, limit) + require.NoError(t, err) + defer rows.Close() + var ids []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + ids = append(ids, id) + } + require.NoError(t, rows.Err()) + return ids + } + + // Insert regression: RETURNING * must survive the new column, and new + // rows must start with search_tsv NULL so they enter the pending queue. + eligibleText := newMsg(database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("deploy the search feature")) + var tsvIsNull bool + err := sqlDB.QueryRowContext(ctx, + `SELECT search_tsv IS NULL FROM chat_messages WHERE id = $1`, eligibleText.ID, + ).Scan(&tsvIsNull) + require.NoError(t, err) + require.True(t, tsvIsNull, "new rows must have search_tsv NULL") + + eligibleNoText := newMsg(database.ChatMessageRoleAssistant, database.ChatMessageVisibilityUser, `[{"type":"reasoning","text":"thinking"}]`) + toolMsg := newMsg(database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, textContent("tool output about deploy")) + modelOnly := newMsg(database.ChatMessageRoleUser, database.ChatMessageVisibilityModel, textContent("model-only deploy note")) + deletedMsg := newMsg(database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("deleted deploy message")) + _, err = sqlDB.ExecContext(ctx, `UPDATE chat_messages SET deleted = true WHERE id = $1`, deletedMsg.ID) + require.NoError(t, err) + + // Only eligible rows appear in the queue, newest first. The tool-role, + // model-only, and soft-deleted rows are excluded even though their + // search_tsv is NULL. + require.Equal(t, []int64{eligibleNoText.ID, eligibleText.ID}, pendingIDs(ctx, 10)) + + // Sweep-style UPDATE. The '' sentinel (not NULL) marks no-text rows as + // swept; NULL means pending, so COALESCE is what drains them from the + // queue. + _, err = sqlDB.ExecContext(ctx, ` + UPDATE chat_messages + SET search_tsv = COALESCE(to_tsvector('simple', chat_message_search_text(content)), ''::tsvector) + WHERE id = ANY($1)`, pq.Array([]int64{eligibleText.ID, eligibleNoText.ID})) + require.NoError(t, err) + require.Empty(t, pendingIDs(ctx, 10), "swept rows must leave the queue, including no-text rows") + + // Soft-deleting an unswept row removes it from the queue without a sweep. + unswept := newMsg(database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("unswept deploy row")) + require.Equal(t, []int64{unswept.ID}, pendingIDs(ctx, 10)) + _, err = sqlDB.ExecContext(ctx, `UPDATE chat_messages SET deleted = true WHERE id = $1`, unswept.ID) + require.NoError(t, err) + require.Empty(t, pendingIDs(ctx, 10)) + + // Search contract: populate search_tsv on every row (including + // ineligible ones) and assert the search-index predicate filters them. + _, err = sqlDB.ExecContext(ctx, ` + UPDATE chat_messages + SET search_tsv = COALESCE(to_tsvector('simple', chat_message_search_text(content)), ''::tsvector) + WHERE chat_id = $1`, chat.ID) + require.NoError(t, err) + + rows, err := sqlDB.QueryContext(ctx, ` + SELECT id FROM chat_messages + WHERE search_tsv @@ websearch_to_tsquery('simple', $1) + AND search_tsv IS NOT NULL + AND `+eligibilityPredicate+` + ORDER BY id`, "deploy") + require.NoError(t, err) + defer rows.Close() + var matched []int64 + for rows.Next() { + var id int64 + require.NoError(t, rows.Scan(&id)) + matched = append(matched, id) + } + require.NoError(t, rows.Err()) + require.Equal(t, []int64{eligibleText.ID}, matched, + "search must exclude deleted, model-only, and tool-role rows (%d %d %d)", + toolMsg.ID, modelOnly.ID, deletedMsg.ID) +} diff --git a/coderd/database/models.go b/coderd/database/models.go index 5f91bd7c37b..f466fae7ef8 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -4979,6 +4979,7 @@ type ChatMessage struct { ProviderResponseID sql.NullString `db:"provider_response_id" json:"provider_response_id"` APIKeyID sql.NullString `db:"api_key_id" json:"api_key_id"` Revision int64 `db:"revision" json:"revision"` + SearchTsv interface{} `db:"search_tsv" json:"search_tsv"` } type ChatModelConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 451a233abcf..2e26ece9c7d 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -7526,7 +7526,7 @@ func (q *sqlQuerier) GetChatHeartbeat(ctx context.Context, arg GetChatHeartbeatP const getChatMessageByID = `-- name: GetChatMessageByID :one SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -7561,6 +7561,7 @@ func (q *sqlQuerier) GetChatMessageByID(ctx context.Context, id int64) (ChatMess &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ) return i, err } @@ -7650,7 +7651,7 @@ func (q *sqlQuerier) GetChatMessageSummariesPerChat(ctx context.Context, created const getChatMessagesByChatID = `-- name: GetChatMessagesByChatID :many SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -7700,6 +7701,7 @@ func (q *sqlQuerier) GetChatMessagesByChatID(ctx context.Context, arg GetChatMes &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } @@ -7716,7 +7718,7 @@ func (q *sqlQuerier) GetChatMessagesByChatID(ctx context.Context, arg GetChatMes const getChatMessagesByChatIDAscPaginated = `-- name: GetChatMessagesByChatIDAscPaginated :many SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -7769,6 +7771,7 @@ func (q *sqlQuerier) GetChatMessagesByChatIDAscPaginated(ctx context.Context, ar &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } @@ -7785,7 +7788,7 @@ func (q *sqlQuerier) GetChatMessagesByChatIDAscPaginated(ctx context.Context, ar const getChatMessagesByChatIDDescPaginated = `-- name: GetChatMessagesByChatIDDescPaginated :many SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -7851,6 +7854,7 @@ func (q *sqlQuerier) GetChatMessagesByChatIDDescPaginated(ctx context.Context, a &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } @@ -7867,7 +7871,7 @@ func (q *sqlQuerier) GetChatMessagesByChatIDDescPaginated(ctx context.Context, a const getChatMessagesByRevisionForStream = `-- name: GetChatMessagesByRevisionForStream :many SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -7916,6 +7920,7 @@ func (q *sqlQuerier) GetChatMessagesByRevisionForStream(ctx context.Context, arg &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } @@ -7948,7 +7953,7 @@ WITH latest_compressed_summary AS ( 1 ) SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -8022,6 +8027,7 @@ func (q *sqlQuerier) GetChatMessagesForPromptByChatID(ctx context.Context, chatI &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } @@ -9256,7 +9262,7 @@ func (q *sqlQuerier) GetDatabaseNow(ctx context.Context) (time.Time, error) { const getLastChatMessageByRole = `-- name: GetLastChatMessageByRole :one SELECT - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv FROM chat_messages WHERE @@ -9301,6 +9307,7 @@ func (q *sqlQuerier) GetLastChatMessageByRole(ctx context.Context, arg GetLastCh &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ) return i, err } @@ -9800,7 +9807,7 @@ SELECT NULLIF(UNNEST($17::bigint[]), 0), NULLIF(UNNEST($18::bigint[]), 0) RETURNING - id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision + id, chat_id, model_config_id, created_at, role, content, visibility, input_tokens, output_tokens, total_tokens, reasoning_tokens, cache_creation_tokens, cache_read_tokens, context_limit, compressed, created_by, content_version, total_cost_micros, runtime_ms, deleted, provider_response_id, api_key_id, revision, search_tsv ` type InsertChatMessagesParams struct { @@ -9876,6 +9883,7 @@ func (q *sqlQuerier) InsertChatMessages(ctx context.Context, arg InsertChatMessa &i.ProviderResponseID, &i.APIKeyID, &i.Revision, + &i.SearchTsv, ); err != nil { return nil, err } diff --git a/coderd/x/chatd/chatstate/trigger_test.go b/coderd/x/chatd/chatstate/trigger_test.go index 5d0bfcb04c4..41a95cf58f2 100644 --- a/coderd/x/chatd/chatstate/trigger_test.go +++ b/coderd/x/chatd/chatstate/trigger_test.go @@ -260,6 +260,80 @@ func TestNoopMessageUpdateDoesNotAdvanceHistoryVersion(t *testing.T) { "no-op update must NOT advance message revision") } +// TestSearchTsvBackfillDoesNotTouchChatState verifies that the +// search_tsv backfill UPDATE leaves message revision, history_version, +// generation_attempt, and retry_state untouched. search_tsv is a +// system-maintained column; populating it is not a content change. +func TestSearchTsvBackfillDoesNotTouchChatState(t *testing.T) { + t.Parallel() + tf := newTriggerFixture(t) + f := tf.f + ctx := testutil.Context(t, testutil.WaitShort) + created := createTestChat(t, f) + + msgs, err := f.DB.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ + ChatID: created.Chat.ID, + }) + require.NoError(t, err) + require.NotEmpty(t, msgs) + target := msgs[0] + require.Equal(t, database.ChatMessageRoleUser, target.Role) + require.False(t, target.Deleted) + originalRevision := target.Revision + + var tsvPending bool + err = tf.sqlDB.QueryRowContext(ctx, + `SELECT search_tsv IS NULL FROM chat_messages WHERE id = $1`, target.ID, + ).Scan(&tsvPending) + require.NoError(t, err) + require.True(t, tsvPending, "fresh message starts with search_tsv pending") + + attempt, err := f.DB.IncrementChatGenerationAttempt(ctx, created.Chat.ID) + require.NoError(t, err) + require.Equal(t, int64(1), attempt) + + withRetry, err := f.DB.UpdateChatRetryState(ctx, database.UpdateChatRetryStateParams{ + ID: created.Chat.ID, + RetryState: []byte(`{"attempt":1,"delay_ms":250,"error":"retry","retrying_at":"2026-05-29T00:00:00Z"}`), + }) + require.NoError(t, err) + require.True(t, withRetry.RetryState.Valid) + + bumped, err := f.DB.LockChatAndBumpSnapshotVersion(ctx, created.Chat.ID) + require.NoError(t, err) + require.NotEqual(t, bumped.SnapshotVersion, bumped.HistoryVersion, + "snapshot bump leaves history_version trailing") + + // Backfill-shaped UPDATE: only search_tsv changes. + _, err = tf.sqlDB.ExecContext(ctx, ` + UPDATE chat_messages + SET search_tsv = COALESCE(to_tsvector('simple', chat_message_search_text(content)), ''::tsvector) + WHERE id = $1 + `, target.ID) + require.NoError(t, err) + + reloadedMsg, err := f.DB.GetChatMessageByID(ctx, target.ID) + require.NoError(t, err) + require.Equal(t, originalRevision, reloadedMsg.Revision, + "backfill must NOT advance message revision") + err = tf.sqlDB.QueryRowContext(ctx, + `SELECT search_tsv IS NULL FROM chat_messages WHERE id = $1`, target.ID, + ).Scan(&tsvPending) + require.NoError(t, err) + require.False(t, tsvPending, "backfill populates search_tsv") + + after, err := f.DB.GetChatByID(ctx, created.Chat.ID) + require.NoError(t, err) + require.Equal(t, bumped.HistoryVersion, after.HistoryVersion, + "backfill must NOT advance history_version") + require.Equal(t, int64(1), after.GenerationAttempt, + "backfill must NOT reset generation_attempt") + require.True(t, after.RetryState.Valid, + "backfill must NOT clear retry_state") + require.Equal(t, withRetry.RetryStateVersion, after.RetryStateVersion, + "backfill must NOT change retry_state_version") +} + // Queue version triggers // TestQueueInsertUpdatesQueueVersion verifies that an INSERT into From 4ebe375659a54e36eef48af247be9b7266c23aa0 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 6 Jul 2026 15:18:44 +0000 Subject: [PATCH 02/12] feat(coderd/database/dbpurge): populate chat_messages.search_tsv - Adds a new query `BackfillChatMessagesSearchTsv` to populate search_tsv for a batch of chat_messages rows. This leverages idx_chat_messages_search_tsv_pending added in the previous commit. - Adds Prometheus metric coderd_dbpurge_chat_search_rows_backfilled_total so operators can track chat message indexing progress. - Adds a background task to dbpurge to index up to 50,000 rows in 10,000 row batches. --- coderd/database/dbauthz/dbauthz.go | 10 +- coderd/database/dbauthz/dbauthz_test.go | 4 + coderd/database/dbmetrics/querymetrics.go | 8 + coderd/database/dbmock/dbmock.go | 15 + coderd/database/dbpurge/dbpurge.go | 75 ++++- coderd/database/dbpurge/dbpurge_test.go | 383 ++++++++++++++++++++++ coderd/database/querier.go | 6 + coderd/database/queries.sql.go | 30 ++ coderd/database/queries/chats.sql | 21 ++ docs/admin/integrations/prometheus.md | 1 + scripts/metricsdocgen/generated_metrics | 3 + 11 files changed, 541 insertions(+), 15 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 10f6bcd855a..6b4ff3a0aee 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -697,7 +697,8 @@ var ( rbac.ResourceApiKey.Type: {policy.ActionDelete}, rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionDelete}, - // Chat auto-archive sets archived=true on inactive chats. + // Chat auto-archive sets archived=true on inactive chats; the + // search_tsv backfill also updates chat messages. rbac.ResourceChat.Type: {policy.ActionRead, policy.ActionUpdate}, // Purge old boundary logs past the retention period. rbac.ResourceBoundaryLog.Type: {policy.ActionDelete}, @@ -1752,6 +1753,13 @@ func (q *querier) AutoArchiveInactiveChats(ctx context.Context, arg database.Aut return q.db.AutoArchiveInactiveChats(ctx, arg) } +func (q *querier) BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) { + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceChat); err != nil { + return 0, err + } + return q.db.BackfillChatMessagesSearchTsv(ctx, batchSize) +} + func (q *querier) BackoffChatDiffStatus(ctx context.Context, arg database.BackoffChatDiffStatusParams) error { // This is a system-level operation used by the gitsync // background worker to reschedule failed refreshes. Same diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 67c7cee24c6..0da18ab923f 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1012,6 +1012,10 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().DeleteOldChats(gomock.Any(), database.DeleteOldChatsParams{}).Return(int64(0), nil).AnyTimes() check.Args(database.DeleteOldChatsParams{}).Asserts(rbac.ResourceSystem, policy.ActionDelete) })) + s.Run("BackfillChatMessagesSearchTsv", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), int32(100)).Return(int64(0), nil).AnyTimes() + check.Args(int32(100)).Asserts(rbac.ResourceChat, policy.ActionUpdate) + })) s.Run("GetChatRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { dbm.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(30), nil).AnyTimes() check.Args().Asserts() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index a2592c270ec..d0cd769fcbc 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -185,6 +185,14 @@ func (m queryMetricsStore) AutoArchiveInactiveChats(ctx context.Context, arg dat return r0, r1 } +func (m queryMetricsStore) BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) { + start := time.Now() + r0, r1 := m.s.BackfillChatMessagesSearchTsv(ctx, batchSize) + m.queryLatencies.WithLabelValues("BackfillChatMessagesSearchTsv").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "BackfillChatMessagesSearchTsv").Inc() + return r0, r1 +} + func (m queryMetricsStore) BackoffChatDiffStatus(ctx context.Context, arg database.BackoffChatDiffStatusParams) error { start := time.Now() r0 := m.s.BackoffChatDiffStatus(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index ba2884a0d44..9fbabdc3e4f 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -193,6 +193,21 @@ func (mr *MockStoreMockRecorder) AutoArchiveInactiveChats(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoArchiveInactiveChats", reflect.TypeOf((*MockStore)(nil).AutoArchiveInactiveChats), ctx, arg) } +// BackfillChatMessagesSearchTsv mocks base method. +func (m *MockStore) BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BackfillChatMessagesSearchTsv", ctx, batchSize) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BackfillChatMessagesSearchTsv indicates an expected call of BackfillChatMessagesSearchTsv. +func (mr *MockStoreMockRecorder) BackfillChatMessagesSearchTsv(ctx, batchSize any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackfillChatMessagesSearchTsv", reflect.TypeOf((*MockStore)(nil).BackfillChatMessagesSearchTsv), ctx, batchSize) +} + // BackoffChatDiffStatus mocks base method. func (m *MockStore) BackoffChatDiffStatus(ctx context.Context, arg database.BackoffChatDiffStatusParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index 51e034417e0..ca658801397 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -51,6 +51,11 @@ const ( // Chat debug run deletions can cascade into steps with large JSONB // payloads, so they use the same conservative batch size. chatDebugRunsBatchSize = 1000 + // 10k rows take ~800ms; capping at 5 batches bounds per-tick + // transaction growth at a few seconds. Larger backlogs drain + // across ticks. + chatSearchBackfillBatchSize = 10000 + chatSearchBackfillMaxBatches = 5 ) type Option func(*instance) @@ -61,6 +66,14 @@ func WithClock(clk quartz.Clock) Option { return func(i *instance) { i.clk = clk } } +// WithChatSearchBackfillLimits overrides backfill batch size and cap. For tests. +func WithChatSearchBackfillLimits(batchSize int32, maxBatches int) Option { + return func(i *instance) { + i.chatSearchBackfillBatchSize = batchSize + i.chatSearchBackfillMaxBatches = maxBatches + } +} + // New creates a new periodically purging database instance. // Callers must Close the returned instance. func New(ctx context.Context, logger slog.Logger, db database.Store, vals *codersdk.DeploymentValues, reg prometheus.Registerer, opts ...Option) io.Closer { @@ -87,14 +100,26 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder }, []string{"record_type"}) reg.MustRegister(recordsPurged) + // Separate counter: the backfill updates rows, not purges them. + chatSearchRowsBackfilled := prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "dbpurge", + Name: "chat_search_rows_backfilled_total", + Help: "Total number of chat message rows whose search_tsv was backfilled.", + }) + reg.MustRegister(chatSearchRowsBackfilled) + inst := &instance{ - cancel: cancelFunc, - closed: closed, - logger: logger, - vals: vals, - clk: quartz.NewReal(), - iterationDuration: iterationDuration, - recordsPurged: recordsPurged, + cancel: cancelFunc, + closed: closed, + logger: logger, + vals: vals, + clk: quartz.NewReal(), + iterationDuration: iterationDuration, + recordsPurged: recordsPurged, + chatSearchRowsBackfilled: chatSearchRowsBackfilled, + chatSearchBackfillBatchSize: chatSearchBackfillBatchSize, + chatSearchBackfillMaxBatches: chatSearchBackfillMaxBatches, } for _, opt := range opts { opt(inst) @@ -310,6 +335,21 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } } + // Safe to run incrementally: queue membership is per-row, content is + // immutable after insert, and soft-deleted rows leave the index + // automatically. + var backfilledChatSearchRows int64 + for range i.chatSearchBackfillMaxBatches { + n, err := tx.BackfillChatMessagesSearchTsv(ctx, i.chatSearchBackfillBatchSize) + if err != nil { + return xerrors.Errorf("backfill chat_messages.search_tsv: %w", err) + } + backfilledChatSearchRows += n + if n < int64(i.chatSearchBackfillBatchSize) { + break + } + } + i.logger.Debug(ctx, "purged old database entries", slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs), slog.F("expired_api_keys", expiredAPIKeys), @@ -322,6 +362,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. slog.F("chats", purgedChats), slog.F("chat_files", purgedChatFiles), slog.F("chat_debug_runs", purgedChatDebugRuns), + slog.F("chat_search_rows_backfilled", backfilledChatSearchRows), slog.F("duration", i.clk.Since(start)), ) @@ -338,6 +379,9 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns)) i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles)) } + if i.chatSearchRowsBackfilled != nil { + i.chatSearchRowsBackfilled.Add(float64(backfilledChatSearchRows)) + } // chatConfigErr is returned after the tx, so do not record this // iteration as successful when only the deferred config read failed. @@ -362,13 +406,16 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } type instance struct { - cancel context.CancelFunc - closed chan struct{} - logger slog.Logger - vals *codersdk.DeploymentValues - clk quartz.Clock - iterationDuration *prometheus.HistogramVec - recordsPurged *prometheus.CounterVec + cancel context.CancelFunc + closed chan struct{} + logger slog.Logger + vals *codersdk.DeploymentValues + clk quartz.Clock + iterationDuration *prometheus.HistogramVec + recordsPurged *prometheus.CounterVec + chatSearchRowsBackfilled prometheus.Counter + chatSearchBackfillBatchSize int32 + chatSearchBackfillMaxBatches int } func (i *instance) Close() error { diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index e73583075da..cd3fd0d3103 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" + "github.com/sqlc-dev/pqtype" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -254,6 +255,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error { @@ -305,6 +307,7 @@ func TestMetrics(t *testing.T) { mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes() mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1) mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). @@ -2861,3 +2864,383 @@ func TestDeleteOldChatFiles(t *testing.T) { }) } } + +// awaitDoTicks returns a function that blocks until the next purge tick +// completes. The first call waits for the initial tick fired by +// dbpurge.New; each subsequent call advances the mock clock by the purge +// interval and waits for that tick to finish. Unlike awaitDoTick, it does +// not trigger an extra trailing tick, so database state is stable between +// calls. tick may be called at most n times. +func awaitDoTicks(ctx context.Context, t *testing.T, clk *quartz.Mock, n int) func() { + t.Helper() + completed := make(chan struct{}) + advance := make(chan struct{}) + trapNow := clk.Trap().Now() + trapStop := clk.Trap().TickerStop() + trapReset := clk.Trap().TickerReset() + go func() { + defer close(completed) + defer trapReset.Close() + defer trapStop.Close() + defer trapNow.Close() + // Initial tick: Now() trap. Completion: TickerReset trap. + trapNow.MustWait(ctx).MustRelease(ctx) + trapReset.MustWait(ctx).MustRelease(ctx) + select { + case completed <- struct{}{}: + case <-ctx.Done(): + return + } + for i := 1; i < n; i++ { + select { + case <-advance: + case <-ctx.Done(): + return + } + d, w := clk.AdvanceNext() + if !assert.Equal(t, 10*time.Minute, d) { + return + } + w.MustWait(ctx) + // The purge loop stops the ticker, runs doTick, then resets. + trapStop.MustWait(ctx).MustRelease(ctx) + trapReset.MustWait(ctx).MustRelease(ctx) + select { + case completed <- struct{}{}: + case <-ctx.Done(): + return + } + } + }() + first := true + return func() { + t.Helper() + if !first { + testutil.RequireSend(ctx, t, advance, struct{}{}) + } + first = false + testutil.TryReceive(ctx, t, completed) + } +} + +//nolint:paralleltest // It uses LockIDDBPurge. +func TestBackfillChatMessagesSearchTsv(t *testing.T) { + now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC) + + type chatSearchDeps struct { + user database.User + modelConfig database.ChatModelConfig + chat database.Chat + } + setupDeps := func(t *testing.T, db database.Store) chatSearchDeps { + t.Helper() + user := dbgen.User(t, db, database.User{}) + org := dbgen.Organization(t, db, database.Organization{}) + _ = dbgen.OrganizationMember(t, db, database.OrganizationMember{ + UserID: user.ID, + OrganizationID: org.ID, + }) + _ = dbgen.ChatProvider(t, db, database.ChatProvider{ + Provider: "openai", + DisplayName: "OpenAI", + }) + modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{ + Model: "test-model", + ContextLimit: 8192, + }) + chat := dbgen.Chat(t, db, database.Chat{ + OrganizationID: org.ID, + OwnerID: user.ID, + LastModelConfigID: modelConfig.ID, + Title: "search-backfill-test-chat", + }) + return chatSearchDeps{user: user, modelConfig: modelConfig, chat: chat} + } + textContent := func(text string) pqtype.NullRawMessage { + return pqtype.NullRawMessage{ + RawMessage: json.RawMessage(fmt.Sprintf(`[{"type":"text","text":%q}]`, text)), + Valid: true, + } + } + createMessage := func(t *testing.T, db database.Store, deps chatSearchDeps, role database.ChatMessageRole, visibility database.ChatMessageVisibility, content pqtype.NullRawMessage) database.ChatMessage { + t.Helper() + return dbgen.ChatMessage(t, db, database.ChatMessage{ + ChatID: deps.chat.ID, + CreatedBy: uuid.NullUUID{UUID: deps.user.ID, Valid: true}, + ModelConfigID: uuid.NullUUID{UUID: deps.modelConfig.ID, Valid: true}, + Role: role, + Visibility: visibility, + Content: content, + }) + } + softDelete := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64) { + t.Helper() + _, err := rawDB.ExecContext(ctx, "UPDATE chat_messages SET deleted = true WHERE id = $1", id) + require.NoError(t, err) + } + // Repeats the predicate of idx_chat_messages_search_tsv_pending. + countPending := func(ctx context.Context, t *testing.T, rawDB *sql.DB) int { + t.Helper() + var count int + err := rawDB.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM chat_messages + WHERE search_tsv IS NULL + AND deleted = false + AND visibility IN ('user', 'both') + AND role IN ('user', 'assistant')`).Scan(&count) + require.NoError(t, err) + return count + } + searchTsv := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64) (isNull bool, text string) { + t.Helper() + err := rawDB.QueryRowContext(ctx, + "SELECT search_tsv IS NULL, COALESCE(search_tsv::text, '') FROM chat_messages WHERE id = $1", id). + Scan(&isNull, &text) + require.NoError(t, err) + return isNull, text + } + requireBackfilled := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64, msg string) { + t.Helper() + isNull, _ := searchTsv(ctx, t, rawDB, id) + require.False(t, isNull, msg) + } + // Asserts the row's tsvector matches expectedText, not just non-NULL. + requireTsvFor := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64, expectedText string) { + t.Helper() + var matches bool + err := rawDB.QueryRowContext(ctx, + "SELECT search_tsv = to_tsvector('simple', $2::text) FROM chat_messages WHERE id = $1", id, expectedText). + Scan(&matches) + require.NoError(t, err) + require.True(t, matches, "search_tsv should contain the lexemes of %q", expectedText) + } + requireNotBackfilled := func(ctx context.Context, t *testing.T, rawDB *sql.DB, id int64, msg string) { + t.Helper() + isNull, _ := searchTsv(ctx, t, rawDB, id) + require.True(t, isNull, msg) + } + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("DrainConverges", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + eligibleBoth := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("hello world")) + eligibleUserVis := createMessage(t, db, deps, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityUser, textContent("assistant reply")) + eligibleNoText := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, pqtype.NullRawMessage{RawMessage: json.RawMessage(`[]`), Valid: true}) + toolMsg := createMessage(t, db, deps, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, textContent("tool output")) + modelOnlyMsg := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityModel, textContent("model only")) + deletedMsg := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("deleted message")) + softDelete(ctx, t, rawDB, deletedMsg.ID) + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk)) + defer closer.Close() + tick() + + require.Zero(t, countPending(ctx, t, rawDB), "queue should be drained") + requireTsvFor(ctx, t, rawDB, eligibleBoth.ID, "hello world") + requireTsvFor(ctx, t, rawDB, eligibleUserVis.ID, "assistant reply") + requireBackfilled(ctx, t, rawDB, eligibleNoText.ID, "eligible message with no text should be backfilled (sentinel)") + requireNotBackfilled(ctx, t, rawDB, toolMsg.ID, "tool message should never be backfilled") + requireNotBackfilled(ctx, t, rawDB, modelOnlyMsg.ID, "model-only message should never be backfilled") + requireNotBackfilled(ctx, t, rawDB, deletedMsg.ID, "deleted message should never be backfilled") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("BackfillsNewestFirst", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + var ids []int64 + for i := range 5 { + msg := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent(fmt.Sprintf("message %d", i))) + ids = append(ids, msg.ID) + } + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), + dbpurge.WithClock(clk), dbpurge.WithChatSearchBackfillLimits(2, 1)) + defer closer.Close() + tick() + + slices.Sort(ids) + requireBackfilled(ctx, t, rawDB, ids[4], "newest message should be backfilled first") + requireBackfilled(ctx, t, rawDB, ids[3], "second-newest message should be backfilled first") + for _, id := range ids[:3] { + requireNotBackfilled(ctx, t, rawDB, id, "older messages should remain pending after one batch") + } + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("NoTextSentinel", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + emptyArr := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, pqtype.NullRawMessage{RawMessage: json.RawMessage(`[]`), Valid: true}) + noTextParts := createMessage(t, db, deps, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"tool_call","id":"x"}]`), Valid: true}) + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk)) + defer closer.Close() + tick() + + for _, id := range []int64{emptyArr.ID, noTextParts.ID} { + isNull, text := searchTsv(ctx, t, rawDB, id) + require.False(t, isNull, "no-text row should get the empty-tsvector sentinel, not stay NULL") + require.Empty(t, text, "no-text row should have an empty tsvector") + } + require.Zero(t, countPending(ctx, t, rawDB), "sentinel rows should not reappear as pending") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("PerTickBound", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + for i := range 6 { + createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent(fmt.Sprintf("message %d", i))) + } + + tick := awaitDoTicks(ctx, t, clk, 2) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), + dbpurge.WithClock(clk), dbpurge.WithChatSearchBackfillLimits(2, 2)) + defer closer.Close() + + tick() + require.Equal(t, 2, countPending(ctx, t, rawDB), "one tick backfills at most maxBatches*batchSize rows") + + tick() + require.Zero(t, countPending(ctx, t, rawDB), "next tick continues draining") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("SkipsDeletedRows", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + msg := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("soft deleted before backfill")) + softDelete(ctx, t, rawDB, msg.ID) + require.Zero(t, countPending(ctx, t, rawDB), "deleted rows should not appear as pending") + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk)) + defer closer.Close() + tick() + + requireNotBackfilled(ctx, t, rawDB, msg.ID, "deleted row should never be backfilled") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("BackfillsNewMessagesAfterDrain", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + + initial := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("initial message")) + + tick := awaitDoTicks(ctx, t, clk, 2) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk)) + defer closer.Close() + + tick() + requireBackfilled(ctx, t, rawDB, initial.ID, "initial message should be backfilled") + require.Zero(t, countPending(ctx, t, rawDB)) + + fresh := createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent("post drain message")) + tick() + requireBackfilled(ctx, t, rawDB, fresh.ID, "message inserted after drain should be backfilled on the next tick") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("SteadyStateNoop", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + _ = setupDeps(t, db) + reg := prometheus.NewRegistry() + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, reg, dbpurge.WithClock(clk)) + defer closer.Close() + tick() + + require.Zero(t, countPending(ctx, t, rawDB)) + backfilled := promhelp.CounterValue(t, reg, "coderd_dbpurge_chat_search_rows_backfilled_total", nil) + require.Zero(t, backfilled, "empty queue should backfill zero rows") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("MetricsCountsBackfilledRows", func(t *testing.T) { + ctx := testutil.Context(t, testutil.WaitLong) + clk := quartz.NewMock(t) + clk.Set(now).MustWait(ctx) + db, _, _ := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure()) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + deps := setupDeps(t, db) + reg := prometheus.NewRegistry() + + for i := range 3 { + createMessage(t, db, deps, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, textContent(fmt.Sprintf("message %d", i))) + } + createMessage(t, db, deps, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, textContent("tool output")) + + tick := awaitDoTicks(ctx, t, clk, 1) + closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, reg, dbpurge.WithClock(clk)) + defer closer.Close() + tick() + + backfilled := promhelp.CounterValue(t, reg, "coderd_dbpurge_chat_search_rows_backfilled_total", nil) + require.Equal(t, 3, backfilled, "counter should count exactly the eligible backfilled rows") + }) + + //nolint:paralleltest // It uses LockIDDBPurge. + t.Run("SkippedWhenLockHeld", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + defer cancel() + + clk := quartz.NewMock(t) + ctrl := gomock.NewController(t) + mDB := dbmock.NewMockStore(ctrl) + mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes() + mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays). + Return(int32(0), nil).AnyTimes() + mDB.EXPECT().TryAcquireLock(gomock.Any(), int64(database.LockIDDBPurge)).Return(false, nil).AnyTimes() + mDB.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), gomock.Any()).Times(0) + mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")). + DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error { + return f(mDB) + }).MinTimes(1) + + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + done := awaitDoTick(ctx, t, clk) + closer := dbpurge.New(ctx, logger, mDB, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), dbpurge.WithClock(clk)) + defer closer.Close() + testutil.TryReceive(ctx, t, done) + }) +} diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 218688f7b40..f6534bd93b5 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -70,6 +70,12 @@ type sqlcQuerier interface { // created_at ASC flows through to dbpurge's digest truncation; see // buildDigestData in dbpurge.go for the tradeoff rationale. AutoArchiveInactiveChats(ctx context.Context, arg AutoArchiveInactiveChatsParams) ([]AutoArchiveInactiveChatsRow, error) + // Backfills chat_messages.search_tsv for pending rows, newest first. + // The WHERE clause must match the predicate of + // idx_chat_messages_search_tsv_pending exactly so the partial index + // serves this query. + // NULL means "pending", '' means "backfilled, no text". + BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) BackoffChatDiffStatus(ctx context.Context, arg BackoffChatDiffStatusParams) error // Deletes heartbeat rows for the supplied (chat_id, runner_id) pairs. BatchDeleteChatHeartbeats(ctx context.Context, arg BatchDeleteChatHeartbeatsParams) (int64, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 2e26ece9c7d..0a942b22491 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6163,6 +6163,36 @@ func (q *sqlQuerier) AutoArchiveInactiveChats(ctx context.Context, arg AutoArchi return items, nil } +const backfillChatMessagesSearchTsv = `-- name: BackfillChatMessagesSearchTsv :execrows +WITH batch AS ( + SELECT id FROM chat_messages + WHERE search_tsv IS NULL + AND deleted = false + AND visibility IN ('user', 'both') + AND role IN ('user', 'assistant') + ORDER BY id DESC + LIMIT $1::int +) +UPDATE chat_messages cm +SET search_tsv = COALESCE( + to_tsvector('simple', chat_message_search_text(cm.content)), + ''::tsvector) +FROM batch WHERE cm.id = batch.id +` + +// Backfills chat_messages.search_tsv for pending rows, newest first. +// The WHERE clause must match the predicate of +// idx_chat_messages_search_tsv_pending exactly so the partial index +// serves this query. +// NULL means "pending", ” means "backfilled, no text". +func (q *sqlQuerier) BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) { + result, err := q.db.ExecContext(ctx, backfillChatMessagesSearchTsv, batchSize) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const backoffChatDiffStatus = `-- name: BackoffChatDiffStatus :exec UPDATE chat_diff_statuses diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index f5458a835af..8740afb75e3 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -310,6 +310,27 @@ SET WHERE id = @id::bigint; +-- name: BackfillChatMessagesSearchTsv :execrows +-- Backfills chat_messages.search_tsv for pending rows, newest first. +-- The WHERE clause must match the predicate of +-- idx_chat_messages_search_tsv_pending exactly so the partial index +-- serves this query. +WITH batch AS ( + SELECT id FROM chat_messages + WHERE search_tsv IS NULL + AND deleted = false + AND visibility IN ('user', 'both') + AND role IN ('user', 'assistant') + ORDER BY id DESC + LIMIT @batch_size::int +) +UPDATE chat_messages cm +-- NULL means "pending", '' means "backfilled, no text". +SET search_tsv = COALESCE( + to_tsvector('simple', chat_message_search_text(cm.content)), + ''::tsvector) +FROM batch WHERE cm.id = batch.id; + -- name: GetChatByID :one SELECT * FROM chats_expanded diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 03b625f8d3d..26f71eca7aa 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -238,6 +238,7 @@ deployment. They will always be available from the agent. | `coderd_db_query_latencies_seconds` | histogram | Latency distribution of queries in seconds. | `query` | | `coderd_db_tx_duration_seconds` | histogram | Duration of transactions in seconds. | `success` `tx_id` | | `coderd_db_tx_executions_count` | counter | Total count of transactions executed. 'retries' is expected to be 0 for a successful transaction. | `retries` `success` `tx_id` | +| `coderd_dbpurge_chat_search_rows_backfilled_total` | counter | Total number of chat message rows whose search_tsv was backfilled. | | | `coderd_dbpurge_iteration_duration_seconds` | histogram | Duration of each dbpurge iteration in seconds. | `success` | | `coderd_dbpurge_records_purged_total` | counter | Total number of records purged by type. | `record_type` | | `coderd_experiments` | gauge | Indicates whether each experiment is enabled (1) or not (0) | `experiment` | diff --git a/scripts/metricsdocgen/generated_metrics b/scripts/metricsdocgen/generated_metrics index d045636a811..65e613adf6b 100644 --- a/scripts/metricsdocgen/generated_metrics +++ b/scripts/metricsdocgen/generated_metrics @@ -316,6 +316,9 @@ coderd_db_tx_duration_seconds{success="",tx_id=""} 0 # HELP coderd_db_tx_executions_count Total count of transactions executed. 'retries' is expected to be 0 for a successful transaction. # TYPE coderd_db_tx_executions_count counter coderd_db_tx_executions_count{success="",retries="",tx_id=""} 0 +# HELP coderd_dbpurge_chat_search_rows_backfilled_total Total number of chat message rows whose search_tsv was backfilled. +# TYPE coderd_dbpurge_chat_search_rows_backfilled_total counter +coderd_dbpurge_chat_search_rows_backfilled_total 0 # HELP coderd_dbpurge_iteration_duration_seconds Duration of each dbpurge iteration in seconds. # TYPE coderd_dbpurge_iteration_duration_seconds histogram coderd_dbpurge_iteration_duration_seconds{success=""} 0 From 6707b18fad1ee9a0a2fec6dae0c2f0a349c28dba Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 6 Jul 2026 17:29:39 +0000 Subject: [PATCH 03/12] feat(coderd/database): add search parameter to GetChats - Adds a `search` parameter to the `GetChats` query that matches over chat title, PR title, or message body. PostgreSQL full-text search is used for all matches. - Adds database-level tests for chat FTS. NB: a search query that is completely numeric is treated as a PR number search (exact match). --- coderd/database/modelqueries.go | 1 + coderd/database/querier_test.go | 232 ++++++++++++++++++++++++++++++ coderd/database/queries.sql.go | 46 +++++- coderd/database/queries/chats.sql | 40 ++++++ 4 files changed, 317 insertions(+), 2 deletions(-) diff --git a/coderd/database/modelqueries.go b/coderd/database/modelqueries.go index e5618d5564e..09449aa5c90 100644 --- a/coderd/database/modelqueries.go +++ b/coderd/database/modelqueries.go @@ -786,6 +786,7 @@ func (q *sqlQuerier) GetAuthorizedChats(ctx context.Context, arg GetChatsParams, arg.PrNumber, arg.RepoQuery, arg.PrTitleQuery, + arg.Search, arg.OffsetOpt, arg.LimitOpt, ) diff --git a/coderd/database/querier_test.go b/coderd/database/querier_test.go index 00afbbd34f8..4a8f2fe88a4 100644 --- a/coderd/database/querier_test.go +++ b/coderd/database/querier_test.go @@ -10,6 +10,7 @@ import ( "net" "slices" "sort" + "strconv" "strings" "testing" "time" @@ -15291,6 +15292,237 @@ func TestGetChatsFilter(t *testing.T) { } } +func TestGetChatsSearch(t *testing.T) { + t.Parallel() + + store, _, sqlDB := dbtestutil.NewDBWithSQLDB(t) + ctx := context.Background() + + org := dbgen.Organization(t, store, database.Organization{}) + user := dbgen.User(t, store, database.User{}) + dbgen.OrganizationMember(t, store, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID}) + + provider := dbgen.AIProviderWithOptionalKey(t, store, database.AIProvider{ + Type: database.AIProviderTypeOpenai, + }, "test-key") + + modelCfg, err := store.InsertChatModelConfig(ctx, database.InsertChatModelConfigParams{ + AIProviderID: uuid.NullUUID{UUID: provider.ID, Valid: true}, + Model: "test-model-" + uuid.NewString(), + DisplayName: "Test Model", + CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true}, + Enabled: true, + IsDefault: true, + ContextLimit: 128000, + CompressionThreshold: 80, + Options: json.RawMessage(`{}`), + }) + require.NoError(t, err) + + createRoot := func(title string) database.Chat { + t.Helper() + chat, err := store.InsertChat(ctx, database.InsertChatParams{ + OrganizationID: org.ID, + Status: database.ChatStatusWaiting, + ClientType: database.ChatClientTypeUi, + OwnerID: user.ID, + LastModelConfigID: modelCfg.ID, + Title: title, + }) + require.NoError(t, err) + return chat + } + + createChild := func(root database.Chat, title string) database.Chat { + t.Helper() + chat, err := store.InsertChat(ctx, database.InsertChatParams{ + OrganizationID: org.ID, + Status: database.ChatStatusWaiting, + ClientType: database.ChatClientTypeUi, + OwnerID: user.ID, + LastModelConfigID: modelCfg.ID, + Title: title, + ParentChatID: uuid.NullUUID{UUID: root.ID, Valid: true}, + RootChatID: uuid.NullUUID{UUID: root.ID, Valid: true}, + }) + require.NoError(t, err) + return chat + } + + insertMsg := func(chatID uuid.UUID, role database.ChatMessageRole, visibility database.ChatMessageVisibility, text string) database.ChatMessage { + t.Helper() + msgs, err := store.InsertChatMessages(ctx, database.InsertChatMessagesParams{ + ChatID: chatID, + CreatedBy: []uuid.UUID{user.ID}, + ModelConfigID: []uuid.UUID{modelCfg.ID}, + Role: []database.ChatMessageRole{role}, + Content: []string{`[{"type":"text","text":` + strconv.Quote(text) + `}]`}, + ContentVersion: []int16{1}, + Visibility: []database.ChatMessageVisibility{visibility}, + InputTokens: []int64{0}, + OutputTokens: []int64{0}, + TotalTokens: []int64{0}, + ReasoningTokens: []int64{0}, + CacheCreationTokens: []int64{0}, + CacheReadTokens: []int64{0}, + ContextLimit: []int64{0}, + Compressed: []bool{false}, + TotalCostMicros: []int64{0}, + RuntimeMs: []int64{0}, + }) + require.NoError(t, err) + require.Len(t, msgs, 1) + return msgs[0] + } + + linkPR := func(chatID uuid.UUID, url, state, prTitle string, prNumber int32, gitRemoteOrigin string) { + t.Helper() + now := time.Now() + _, err := store.UpsertChatDiffStatusReference(ctx, database.UpsertChatDiffStatusReferenceParams{ + ChatID: chatID, + Url: sql.NullString{String: url, Valid: true}, + GitBranch: "main", + GitRemoteOrigin: gitRemoteOrigin, + StaleAt: now.Add(time.Hour), + }) + require.NoError(t, err) + _, err = store.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{ + ChatID: chatID, + Url: sql.NullString{String: url, Valid: true}, + PullRequestState: sql.NullString{String: state, Valid: true}, + PullRequestTitle: prTitle, + PrNumber: sql.NullInt32{Int32: prNumber, Valid: prNumber > 0}, + Additions: 1, + Deletions: 1, + ChangedFiles: 1, + RefreshedAt: now, + StaleAt: now.Add(time.Hour), + }) + require.NoError(t, err) + } + + titleChat := createRoot("deploy pipeline alpha") + + archivedChat := createRoot("deploy pipeline beta") + + prTitleChat := createRoot("widget work") + linkPR(prTitleChat.ID, "https://github.com/acme/widget/pull/42", "open", "Fix authentication bug", 42, "https://github.com/acme/widget.git") + + mergedChat := createRoot("other work") + linkPR(mergedChat.ID, "https://github.com/acme/other-repo/pull/7", "merged", "Fix authentication flow", 7, "https://github.com/acme/other-repo.git") + + msgChat := createRoot("plain one") + insertMsg(msgChat.ID, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, "kubernetes cluster restart") + + assistantMsgChat := createRoot("plain assistant") + insertMsg(assistantMsgChat.ID, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, "grafana dashboard tuning") + + userVisMsgChat := createRoot("plain uservis") + insertMsg(userVisMsgChat.ID, database.ChatMessageRoleUser, database.ChatMessageVisibilityUser, "vault token rotation") + + assistantUserVisMsgChat := createRoot("plain assistant uservis") + insertMsg(assistantUserVisMsgChat.ID, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityUser, "redis eviction policy") + + deletedMsgChat := createRoot("plain two") + deletedMsg := insertMsg(deletedMsgChat.ID, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, "terraform apply failure") + + childParent := createRoot("plain parent") + childChat := createChild(childParent, "plain child") + insertMsg(childChat.ID, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityUser, "orchestrator saga") + + ineligibleChat := createRoot("plain three") + toolMsg := insertMsg(ineligibleChat.ID, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, "forbidden secret token") + modelOnlyMsg := insertMsg(ineligibleChat.ID, database.ChatMessageRoleUser, database.ChatMessageVisibilityModel, "forbidden secret token") + + // Ineligible rows keep search_tsv NULL after backfill. + _, err = store.BackfillChatMessagesSearchTsv(ctx, 1000) + require.NoError(t, err) + + // Soft-deleted rows stay excluded even though search_tsv remains + // populated. + err = store.SoftDeleteChatMessageByID(ctx, deletedMsg.ID) + require.NoError(t, err) + + // Inserted after backfill: search_tsv IS NULL, must match nothing. + pendingChat := createRoot("plain four") + insertMsg(pendingChat.ID, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, "elasticsearch indexing") + + // Prove role/visibility predicates exclude rows even when search_tsv + // is set. + _, err = sqlDB.ExecContext(ctx, + `UPDATE chat_messages SET search_tsv = to_tsvector('simple', 'forbidden secret token') WHERE id = ANY($1)`, + pq.Array([]int64{toolMsg.ID, modelOnlyMsg.ID})) + require.NoError(t, err) + + _, err = store.ArchiveChatByID(ctx, archivedChat.ID) + require.NoError(t, err) + + allRootIDs := []uuid.UUID{ + titleChat.ID, archivedChat.ID, prTitleChat.ID, mergedChat.ID, + msgChat.ID, assistantMsgChat.ID, userVisMsgChat.ID, + assistantUserVisMsgChat.ID, deletedMsgChat.ID, childParent.ID, + ineligibleChat.ID, pendingChat.ID, + } + + tests := []struct { + name string + params database.GetChatsParams + want []uuid.UUID + }{ + {"Title/Match", database.GetChatsParams{Search: "pipeline alpha"}, []uuid.UUID{titleChat.ID}}, + {"Title/CaseInsensitiveMultiWord", database.GetChatsParams{Search: "ALPHA DEPLOY"}, []uuid.UUID{titleChat.ID}}, + {"Title/AndSemantics", database.GetChatsParams{Search: "deploy nonexistent"}, nil}, + {"PRTitle/Match", database.GetChatsParams{Search: "authentication"}, []uuid.UUID{prTitleChat.ID, mergedChat.ID}}, + {"Message/Match", database.GetChatsParams{Search: "kubernetes restart"}, []uuid.UUID{msgChat.ID}}, + {"Message/AssistantRoleMatch", database.GetChatsParams{Search: "grafana tuning"}, []uuid.UUID{assistantMsgChat.ID}}, + {"Message/UserVisibilityMatch", database.GetChatsParams{Search: "vault rotation"}, []uuid.UUID{userVisMsgChat.ID}}, + {"Message/AssistantUserVisibilityMatch", database.GetChatsParams{Search: "redis eviction"}, []uuid.UUID{assistantUserVisMsgChat.ID}}, + {"PRNumber/Match", database.GetChatsParams{Search: "42"}, []uuid.UUID{prTitleChat.ID}}, + {"PRNumber/NonNumericNoMatch", database.GetChatsParams{Search: "42abc"}, nil}, + {"PRNumber/OversizedDigitsNoError", database.GetChatsParams{Search: "1111111111111111111111111"}, nil}, + {"NoMatch", database.GetChatsParams{Search: "zzzqqq"}, nil}, + {"Message/PendingBackfillNoMatch", database.GetChatsParams{Search: "elasticsearch"}, nil}, + {"Message/DeletedNoMatch", database.GetChatsParams{Search: "terraform"}, nil}, + // Parent also excluded: EXISTS is per-chat, not per-tree. + {"Message/ChildNotSurfaced", database.GetChatsParams{Search: "orchestrator saga"}, nil}, + {"Message/IneligibleMessagesNoMatch", database.GetChatsParams{Search: "forbidden secret"}, nil}, + {"Composed/ArchivedDefaultIncludesAll", database.GetChatsParams{Search: "deploy pipeline"}, []uuid.UUID{titleChat.ID, archivedChat.ID}}, + {"Composed/ArchivedFalseExcludes", database.GetChatsParams{Search: "deploy pipeline", Archived: sql.NullBool{Bool: false, Valid: true}}, []uuid.UUID{titleChat.ID}}, + {"Composed/ArchivedTrueOnly", database.GetChatsParams{Search: "deploy pipeline", Archived: sql.NullBool{Bool: true, Valid: true}}, []uuid.UUID{archivedChat.ID}}, + {"Composed/SearchAndRepo", database.GetChatsParams{Search: "authentication", RepoQuery: "acme/widget"}, []uuid.UUID{prTitleChat.ID}}, + {"Composed/SearchAndPRStatus", database.GetChatsParams{Search: "authentication", PullRequestStatuses: []string{"merged"}}, []uuid.UUID{mergedChat.ID}}, + {"EmptySearch/ReturnsAll", database.GetChatsParams{Search: ""}, allRootIDs}, + {"WhitespaceSearch/ReturnsAll", database.GetChatsParams{Search: " "}, allRootIDs}, + {"TabOnlySearch/ReturnsAll", database.GetChatsParams{Search: "\t\t"}, allRootIDs}, + {"EmptySearch/TitleQueryStillWorks", database.GetChatsParams{Search: "", TitleQuery: "pipeline alpha"}, []uuid.UUID{titleChat.ID}}, + {"EmptySearch/PRTitleQueryStillWorks", database.GetChatsParams{Search: "", PrTitleQuery: "authentication bug"}, []uuid.UUID{prTitleChat.ID}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + params := tt.params + params.OwnedOnly = true + params.ViewerID = user.ID + + rows, err := store.GetChats(ctx, params) + require.NoError(t, err) + + got := make([]uuid.UUID, 0, len(rows)) + for _, row := range rows { + got = append(got, row.Chat.ID) + } + + if tt.want == nil { + require.Empty(t, got) + } else { + require.ElementsMatch(t, tt.want, got) + } + }) + } +} + func TestChatHasUnread(t *testing.T) { t.Parallel() diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0a942b22491..fd6a136097e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8738,6 +8738,46 @@ WHERE ) ELSE true END + -- websearch_to_tsquery accepts quoted phrases, OR, and -negation; + -- the 'simple' config folds case and skips stemming. + AND CASE + WHEN btrim($16::text, E' \t\n\r') != '' THEN ( + -- Served by idx_chats_title_fts. + to_tsvector('simple', chats_expanded.title) @@ websearch_to_tsquery('simple', $16) + -- Served by idx_chat_diff_statuses_pr_title_fts. + OR EXISTS ( + SELECT 1 + FROM chat_diff_statuses cds + WHERE cds.chat_id = chats_expanded.id + AND to_tsvector('simple', cds.pull_request_title) @@ websearch_to_tsquery('simple', $16) + ) + -- The WHERE clause must repeat the partial predicate of + -- idx_chat_messages_search_tsv exactly so the planner can use it. + OR EXISTS ( + SELECT 1 + FROM chat_messages cm + WHERE cm.chat_id = chats_expanded.id + AND cm.search_tsv IS NOT NULL + AND cm.deleted = false + AND cm.visibility IN ('user', 'both') + AND cm.role IN ('user', 'assistant') + AND cm.search_tsv @@ websearch_to_tsquery('simple', $16) + ) + -- CASE forces the digits guard before the ::bigint cast; AND + -- operand order is not guaranteed. + OR CASE + WHEN $16 ~ '^[0-9]{1,18}$' THEN EXISTS ( + SELECT 1 + FROM chat_diff_statuses cds + WHERE cds.chat_id = chats_expanded.id + AND cds.pr_number IS NOT NULL + AND cds.pr_number::bigint = $16::bigint + ) + ELSE false + END + ) + ELSE true + END -- Paginate over root chats only. Children are fetched -- separately via GetChildChatsByParentIDs and embedded under -- each parent. Other callers that need the full set should @@ -8754,11 +8794,11 @@ ORDER BY -chats_expanded.pin_order DESC, chats_expanded.updated_at DESC, chats_expanded.id DESC -OFFSET $16 +OFFSET $17 LIMIT -- The chat list is unbounded and expected to grow large. -- Default to 50 to prevent accidental excessively large queries. - COALESCE(NULLIF($17 :: int, 0), 50) + COALESCE(NULLIF($18 :: int, 0), 50) ` type GetChatsParams struct { @@ -8777,6 +8817,7 @@ type GetChatsParams struct { PrNumber int32 `db:"pr_number" json:"pr_number"` RepoQuery string `db:"repo_query" json:"repo_query"` PrTitleQuery string `db:"pr_title_query" json:"pr_title_query"` + Search string `db:"search" json:"search"` OffsetOpt int32 `db:"offset_opt" json:"offset_opt"` LimitOpt int32 `db:"limit_opt" json:"limit_opt"` } @@ -8803,6 +8844,7 @@ func (q *sqlQuerier) GetChats(ctx context.Context, arg GetChatsParams) ([]GetCha arg.PrNumber, arg.RepoQuery, arg.PrTitleQuery, + arg.Search, arg.OffsetOpt, arg.LimitOpt, ) diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 8740afb75e3..7a087977571 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -670,6 +670,46 @@ WHERE ) ELSE true END + -- websearch_to_tsquery accepts quoted phrases, OR, and -negation; + -- the 'simple' config folds case and skips stemming. + AND CASE + WHEN btrim(@search::text, E' \t\n\r') != '' THEN ( + -- Served by idx_chats_title_fts. + to_tsvector('simple', chats_expanded.title) @@ websearch_to_tsquery('simple', @search) + -- Served by idx_chat_diff_statuses_pr_title_fts. + OR EXISTS ( + SELECT 1 + FROM chat_diff_statuses cds + WHERE cds.chat_id = chats_expanded.id + AND to_tsvector('simple', cds.pull_request_title) @@ websearch_to_tsquery('simple', @search) + ) + -- The WHERE clause must repeat the partial predicate of + -- idx_chat_messages_search_tsv exactly so the planner can use it. + OR EXISTS ( + SELECT 1 + FROM chat_messages cm + WHERE cm.chat_id = chats_expanded.id + AND cm.search_tsv IS NOT NULL + AND cm.deleted = false + AND cm.visibility IN ('user', 'both') + AND cm.role IN ('user', 'assistant') + AND cm.search_tsv @@ websearch_to_tsquery('simple', @search) + ) + -- CASE forces the digits guard before the ::bigint cast; AND + -- operand order is not guaranteed. + OR CASE + WHEN @search ~ '^[0-9]{1,18}$' THEN EXISTS ( + SELECT 1 + FROM chat_diff_statuses cds + WHERE cds.chat_id = chats_expanded.id + AND cds.pr_number IS NOT NULL + AND cds.pr_number::bigint = @search::bigint + ) + ELSE false + END + ) + ELSE true + END -- Paginate over root chats only. Children are fetched -- separately via GetChildChatsByParentIDs and embedded under -- each parent. Other callers that need the full set should From bf6195215481dad7c3d68d22393ce879dd46cb45 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 8 Jul 2026 11:54:27 +0000 Subject: [PATCH 04/12] feat(coderd/searchquery): parse search filter for chats - Adds `search:` to `searchquery.Chats` that accepts a single search parameter (multiple terms in quotes). - Mutually exclusive with `title`, `pr_title`, `pr`. - Validation: only non-empty values are supported. --- coderd/searchquery/search.go | 26 +++++++++ coderd/searchquery/search_test.go | 89 +++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/coderd/searchquery/search.go b/coderd/searchquery/search.go index 20291c8033a..f8e7dd64b6a 100644 --- a/coderd/searchquery/search.go +++ b/coderd/searchquery/search.go @@ -519,6 +519,8 @@ func Tasks(ctx context.Context, db database.Store, query string, actorID uuid.UU // ownership scope; created_by_me returns only chats the caller owns, // shared_with_me returns only chats shared with the caller, all returns // both) +// - search: full-text search over chat content; mutually exclusive +// with title, pr_title, and pr func Chats(query string) (database.GetChatsParams, []codersdk.ValidationError) { filter := database.GetChatsParams{ // Default to hiding archived chats and chats not owned by the caller. @@ -606,6 +608,30 @@ func Chats(query string) (database.GetChatsParams, []codersdk.ValidationError) { } } + if values.Has("search") { + parser.RequiredNotEmpty("search") + if search := parser.String(values, "", "search"); search != "" { + var conflicts []string + if filter.TitleQuery != "" { + conflicts = append(conflicts, `"title"`) + } + if filter.PrTitleQuery != "" { + conflicts = append(conflicts, `"pr_title"`) + } + if filter.PrNumber != 0 { + conflicts = append(conflicts, `"pr"`) + } + if len(conflicts) > 0 { + parser.Errors = append(parser.Errors, codersdk.ValidationError{ + Field: "search", + Detail: fmt.Sprintf(`"search" cannot be combined with %s`, strings.Join(conflicts, ", ")), + }) + } else { + filter.Search = search + } + } + } + parser.ErrorExcessParams(values) return filter, parser.Errors } diff --git a/coderd/searchquery/search_test.go b/coderd/searchquery/search_test.go index 021df7edc35..76deab73981 100644 --- a/coderd/searchquery/search_test.go +++ b/coderd/searchquery/search_test.go @@ -1238,6 +1238,8 @@ func TestSearchChats(t *testing.T) { Query string Expected database.GetChatsParams ExpectedErrorContains string + // When non-zero, asserts the exact number of validation errors. + ExpectedErrorCount int }{ { Name: "Empty", @@ -1597,6 +1599,90 @@ func TestSearchChats(t *testing.T) { Query: "some random words", ExpectedErrorContains: `unsupported search term: "some random words"`, }, + { + Name: "Search", + Query: "search:foo", + Expected: database.GetChatsParams{ + Archived: sql.NullBool{Bool: false, Valid: true}, + OwnedOnly: true, + Search: "foo", + }, + }, + { + Name: "SearchQuoted", + Query: `search:"foo bar"`, + Expected: database.GetChatsParams{ + Archived: sql.NullBool{Bool: false, Valid: true}, + OwnedOnly: true, + Search: "foo bar", + }, + }, + { + Name: "SearchWithStructuralFilters", + Query: `repo:coder/coder archived:true search:"foo bar"`, + Expected: database.GetChatsParams{ + Archived: sql.NullBool{Bool: true, Valid: true}, + OwnedOnly: true, + RepoQuery: "coder/coder", + Search: "foo bar", + }, + }, + { + Name: "SearchWithAllStructuralFilters", + Query: `search:foo archived:true repo:coder/coder diff_url:"https://github.com/coder/coder/pull/1" has_unread:true pr_status:open source:created_by_me`, + Expected: database.GetChatsParams{ + Archived: sql.NullBool{Bool: true, Valid: true}, + OwnedOnly: true, + RepoQuery: "coder/coder", + DiffURL: sql.NullString{String: "https://github.com/coder/coder/pull/1", Valid: true}, + HasUnread: sql.NullBool{Bool: true, Valid: true}, + PullRequestStatuses: []string{"open"}, + Search: "foo", + }, + }, + { + Name: "SearchRepeated", + Query: "search:foo search:bar", + ExpectedErrorContains: `search: Query param "search" provided more than once`, + ExpectedErrorCount: 1, + }, + { + Name: "SearchConflictsWithTitle", + Query: "search:foo title:bar", + ExpectedErrorContains: `search: "search" cannot be combined with "title"`, + }, + { + Name: "SearchConflictsWithPrTitle", + Query: "search:foo pr_title:bar", + ExpectedErrorContains: `search: "search" cannot be combined with "pr_title"`, + }, + { + Name: "SearchConflictsWithPr", + Query: "search:foo pr:12", + ExpectedErrorContains: `search: "search" cannot be combined with "pr"`, + }, + { + Name: "SearchConflictsOrderIndependent", + Query: "title:bar search:foo", + ExpectedErrorContains: `search: "search" cannot be combined with "title"`, + }, + { + Name: "SearchConflictsWithMultiple", + Query: "search:foo title:bar pr:12", + ExpectedErrorContains: `search: "search" cannot be combined with "title", "pr"`, + }, + { + // The tokenizer rejects trailing colons before search validation runs. + Name: "SearchBareKey", + Query: "search:", + ExpectedErrorContains: "cannot start or end with ':'", + }, + { + Name: "SearchEmptyQuoted", + Query: `search:""`, + ExpectedErrorContains: `search: Query param "search" is required and cannot be empty`, + ExpectedErrorCount: 1, + }, } for _, c := range testCases { @@ -1606,6 +1692,9 @@ func TestSearchChats(t *testing.T) { values, errs := searchquery.Chats(c.Query) if c.ExpectedErrorContains != "" { require.True(t, len(errs) > 0, "expect some errors") + if c.ExpectedErrorCount > 0 { + require.Len(t, errs, c.ExpectedErrorCount, "expected exact error count") + } var s strings.Builder for _, err := range errs { _, _ = s.WriteString(fmt.Sprintf("%s: %s\n", err.Field, err.Detail)) From a558b88d8131e105577cfe9bf989feda53a4123c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 8 Jul 2026 13:20:02 +0000 Subject: [PATCH 05/12] feat(coderd): wire chat search filter into chats API - Wires the `search` filter through to `GetChats` - Adds a preflight query to check if the tokenized query evaluates to an empty tsquery, returns an 400 error if so. - Adds corresponding API tests. --- coderd/apidoc/docs.go | 2 +- coderd/apidoc/swagger.json | 2 +- coderd/database/dbauthz/dbauthz.go | 8 ++ coderd/database/dbauthz/dbauthz_test.go | 4 + coderd/database/dbmetrics/querymetrics.go | 8 ++ coderd/database/dbmock/dbmock.go | 15 ++ coderd/database/querier.go | 3 + coderd/database/queries.sql.go | 13 ++ coderd/database/queries/chats.sql | 5 + coderd/exp_chats.go | 25 +++- coderd/exp_chats_test.go | 165 ++++++++++++++++++++++ docs/reference/api/chats.md | 8 +- 12 files changed, 251 insertions(+), 7 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index f018cf7bc6e..c5b618c2099 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -78,7 +78,7 @@ const docTemplate = `{ "parameters": [ { "type": "string", - "description": "Search query. Supports title:\u003csubstring\u003e (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status:\u003cdraft\\|open\\|merged\\|closed\u003e as repeated or comma-separated values, source:\u003ccreated_by_me\\|shared_with_me\u003e, diff_url:\u003curl\u003e (quote values containing colons), pr:\u003cnumber\u003e (exact PR number match), repo:\u003cowner/repo\u003e (case-insensitive substring match against git remote origin or URL), pr_title:\u003ctext\u003e (case-insensitive PR title substring). Bare terms are not supported; use title:\u003cvalue\u003e for title filtering.", + "description": "Search query. Supports title:\u003csubstring\u003e (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status:\u003cdraft\\|open\\|merged\\|closed\u003e as repeated or comma-separated values, source:\u003ccreated_by_me\\|shared_with_me\u003e, diff_url:\u003curl\u003e (quote values containing colons), pr:\u003cnumber\u003e (exact PR number match), repo:\u003cowner/repo\u003e (case-insensitive substring match against git remote origin or URL), pr_title:\u003ctext\u003e (case-insensitive PR title substring), search:\u003ctext\u003e (full-text search across chat titles, PR titles, PR numbers, and message bodies; quote multi-word values; cannot be combined with title, pr_title, or pr). Bare terms are not supported; use title:\u003cvalue\u003e or search:\u003cvalue\u003e.", "name": "q", "in": "query" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 699d7805c9c..82cebadf771 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -59,7 +59,7 @@ "parameters": [ { "type": "string", - "description": "Search query. Supports title:\u003csubstring\u003e (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status:\u003cdraft\\|open\\|merged\\|closed\u003e as repeated or comma-separated values, source:\u003ccreated_by_me\\|shared_with_me\u003e, diff_url:\u003curl\u003e (quote values containing colons), pr:\u003cnumber\u003e (exact PR number match), repo:\u003cowner/repo\u003e (case-insensitive substring match against git remote origin or URL), pr_title:\u003ctext\u003e (case-insensitive PR title substring). Bare terms are not supported; use title:\u003cvalue\u003e for title filtering.", + "description": "Search query. Supports title:\u003csubstring\u003e (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status:\u003cdraft\\|open\\|merged\\|closed\u003e as repeated or comma-separated values, source:\u003ccreated_by_me\\|shared_with_me\u003e, diff_url:\u003curl\u003e (quote values containing colons), pr:\u003cnumber\u003e (exact PR number match), repo:\u003cowner/repo\u003e (case-insensitive substring match against git remote origin or URL), pr_title:\u003ctext\u003e (case-insensitive PR title substring), search:\u003ctext\u003e (full-text search across chat titles, PR titles, PR numbers, and message bodies; quote multi-word values; cannot be combined with title, pr_title, or pr). Bare terms are not supported; use title:\u003cvalue\u003e or search:\u003cvalue\u003e.", "name": "q", "in": "query" }, diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 6b4ff3a0aee..62de9665dd3 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -1837,6 +1837,14 @@ func (q *querier) CalculateAIBridgeInterceptionsTelemetrySummary(ctx context.Con return q.db.CalculateAIBridgeInterceptionsTelemetrySummary(ctx, arg) } +func (q *querier) ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) { + // Pure function, no rows. Gates on chat read to match the listing caller. + if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceChat); err != nil { + return false, err + } + return q.db.ChatSearchQueryIsEmpty(ctx, search) +} + func (q *querier) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { empty := database.ClaimPrebuiltWorkspaceRow{} diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 0da18ab923f..f5c1cb05814 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -1016,6 +1016,10 @@ func (s *MethodTestSuite) TestChats() { dbm.EXPECT().BackfillChatMessagesSearchTsv(gomock.Any(), int32(100)).Return(int64(0), nil).AnyTimes() check.Args(int32(100)).Asserts(rbac.ResourceChat, policy.ActionUpdate) })) + s.Run("ChatSearchQueryIsEmpty", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { + dbm.EXPECT().ChatSearchQueryIsEmpty(gomock.Any(), "!!!").Return(true, nil).AnyTimes() + check.Args("!!!").Asserts(rbac.ResourceChat, policy.ActionRead) + })) s.Run("GetChatRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) { dbm.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(30), nil).AnyTimes() check.Args().Asserts() diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index d0cd769fcbc..e43a3adfe77 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -273,6 +273,14 @@ func (m queryMetricsStore) CalculateAIBridgeInterceptionsTelemetrySummary(ctx co return r0, r1 } +func (m queryMetricsStore) ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) { + start := time.Now() + r0, r1 := m.s.ChatSearchQueryIsEmpty(ctx, search) + m.queryLatencies.WithLabelValues("ChatSearchQueryIsEmpty").Observe(time.Since(start).Seconds()) + m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "ChatSearchQueryIsEmpty").Inc() + return r0, r1 +} + func (m queryMetricsStore) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { start := time.Now() r0, r1 := m.s.ClaimPrebuiltWorkspace(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 9fbabdc3e4f..28e94b51562 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -352,6 +352,21 @@ func (mr *MockStoreMockRecorder) CalculateAIBridgeInterceptionsTelemetrySummary( return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CalculateAIBridgeInterceptionsTelemetrySummary", reflect.TypeOf((*MockStore)(nil).CalculateAIBridgeInterceptionsTelemetrySummary), ctx, arg) } +// ChatSearchQueryIsEmpty mocks base method. +func (m *MockStore) ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChatSearchQueryIsEmpty", ctx, search) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ChatSearchQueryIsEmpty indicates an expected call of ChatSearchQueryIsEmpty. +func (mr *MockStoreMockRecorder) ChatSearchQueryIsEmpty(ctx, search any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChatSearchQueryIsEmpty", reflect.TypeOf((*MockStore)(nil).ChatSearchQueryIsEmpty), ctx, search) +} + // ClaimPrebuiltWorkspace mocks base method. func (m *MockStore) ClaimPrebuiltWorkspace(ctx context.Context, arg database.ClaimPrebuiltWorkspaceParams) (database.ClaimPrebuiltWorkspaceRow, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index f6534bd93b5..34ab3808ad0 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -89,6 +89,9 @@ type sqlcQuerier interface { // Calculates the telemetry summary for a given provider, model, and client // combination for telemetry reporting. CalculateAIBridgeInterceptionsTelemetrySummary(ctx context.Context, arg CalculateAIBridgeInterceptionsTelemetrySummaryParams) (CalculateAIBridgeInterceptionsTelemetrySummaryRow, error) + // Reports whether search text tokenizes to an empty tsquery (e.g. '!!!'). + // Used to reject input that would silently match nothing. + ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) ClaimPrebuiltWorkspace(ctx context.Context, arg ClaimPrebuiltWorkspaceParams) (ClaimPrebuiltWorkspaceRow, error) CleanTailnetCoordinators(ctx context.Context) error CleanTailnetLostPeers(ctx context.Context) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index fd6a136097e..07304c02f96 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6257,6 +6257,19 @@ func (q *sqlQuerier) BatchUpsertChatHeartbeats(ctx context.Context, arg BatchUps return err } +const chatSearchQueryIsEmpty = `-- name: ChatSearchQueryIsEmpty :one +SELECT numnode(websearch_to_tsquery('simple', $1::text)) = 0 AS is_empty +` + +// Reports whether search text tokenizes to an empty tsquery (e.g. '!!!'). +// Used to reject input that would silently match nothing. +func (q *sqlQuerier) ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) { + row := q.db.QueryRowContext(ctx, chatSearchQueryIsEmpty, search) + var is_empty bool + err := row.Scan(&is_empty) + return is_empty, err +} + const countChatQueuedMessages = `-- name: CountChatQueuedMessages :one SELECT COUNT(*)::bigint AS count FROM chat_queued_messages diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 7a087977571..093cecf566e 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -331,6 +331,11 @@ SET search_tsv = COALESCE( ''::tsvector) FROM batch WHERE cm.id = batch.id; +-- name: ChatSearchQueryIsEmpty :one +-- Reports whether search text tokenizes to an empty tsquery (e.g. '!!!'). +-- Used to reject input that would silently match nothing. +SELECT numnode(websearch_to_tsquery('simple', @search::text)) = 0 AS is_empty; + -- name: GetChatByID :one SELECT * FROM chats_expanded diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 1c35f334573..d01f7784b03 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -333,7 +333,7 @@ func (api *API) chatsByWorkspace(rw http.ResponseWriter, r *http.Request) { // @Security CoderSessionToken // @Tags Chats // @Produce json -// @Param q query string false "Search query. Supports title: (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status: as repeated or comma-separated values, source:, diff_url: (quote values containing colons), pr: (exact PR number match), repo: (case-insensitive substring match against git remote origin or URL), pr_title: (case-insensitive PR title substring). Bare terms are not supported; use title: for title filtering." +// @Param q query string false "Search query. Supports title: (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status: as repeated or comma-separated values, source:, diff_url: (quote values containing colons), pr: (exact PR number match), repo: (case-insensitive substring match against git remote origin or URL), pr_title: (case-insensitive PR title substring), search: (full-text search across chat titles, PR titles, PR numbers, and message bodies; quote multi-word values; cannot be combined with title, pr_title, or pr). Bare terms are not supported; use title: or search:." // @Param label query string false "Filter by label as key:value. Repeat for multiple (AND logic)." // @Success 200 {array} codersdk.Chat // @Router /api/experimental/chats [get] @@ -357,6 +357,28 @@ func (api *API) listChats(rw http.ResponseWriter, r *http.Request) { return } + // Reject text that tokenizes to nothing; it would silently match no rows. + if searchParams.Search != "" { + isEmpty, err := api.Database.ChatSearchQueryIsEmpty(ctx, searchParams.Search) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Failed to validate search query.", + Detail: err.Error(), + }) + return + } + if isEmpty { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Invalid chat search query.", + Validations: []codersdk.ValidationError{{ + Field: "search", + Detail: "Search query contains no searchable words.", + }}, + }) + return + } + } + var labelFilter pqtype.NullRawMessage if labelParams := r.URL.Query()["label"]; len(labelParams) > 0 { labelMap := make(map[string]string, len(labelParams)) @@ -416,6 +438,7 @@ func (api *API) listChats(rw http.ResponseWriter, r *http.Request) { PrNumber: searchParams.PrNumber, RepoQuery: searchParams.RepoQuery, PrTitleQuery: searchParams.PrTitleQuery, + Search: searchParams.Search, // #nosec G115 - Pagination offsets are small and fit in int32 OffsetOpt: int32(paginationParams.Offset), // #nosec G115 - Pagination limits are small and fit in int32 diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index a804ea70315..edf40d5ae0b 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -1926,6 +1926,171 @@ func TestListChatModels(t *testing.T) { }) } +func TestListChats_Search(t *testing.T) { + t.Parallel() + + setup := func(t *testing.T) (context.Context, *codersdk.ExperimentalClient, database.Store, codersdk.CreateFirstUserResponse, codersdk.ChatModelConfig) { + t.Helper() + ctx := testutil.Context(t, testutil.WaitLong) + client, db := newChatClientWithDatabase(t) + firstUser := coderdtest.CreateFirstUser(t, client.Client) + modelConfig := createChatModelConfig(t, client) + return ctx, client, db, firstUser, modelConfig + } + + createChat := func(t *testing.T, db database.Store, firstUser codersdk.CreateFirstUserResponse, modelConfigID uuid.UUID, title string) database.Chat { + t.Helper() + return dbgen.Chat(t, db, database.Chat{ + OrganizationID: firstUser.OrganizationID, + OwnerID: firstUser.UserID, + LastModelConfigID: modelConfigID, + Title: title, + Status: database.ChatStatusCompleted, + }) + } + + insertMessage := func(t *testing.T, db database.Store, firstUser codersdk.CreateFirstUserResponse, modelConfigID, chatID uuid.UUID, text string) { + t.Helper() + content, err := json.Marshal([]map[string]string{{"type": "text", "text": text}}) + require.NoError(t, err) + dbgen.ChatMessage(t, db, database.ChatMessage{ + ChatID: chatID, + CreatedBy: uuid.NullUUID{UUID: firstUser.UserID, Valid: true}, + ModelConfigID: uuid.NullUUID{UUID: modelConfigID, Valid: true}, + Role: database.ChatMessageRoleUser, + Visibility: database.ChatMessageVisibilityBoth, + Content: pqtype.NullRawMessage{RawMessage: content, Valid: true}, + }) + } + + backfillSearchTsv := func(ctx context.Context, t *testing.T, db database.Store) { + t.Helper() + _, err := db.BackfillChatMessagesSearchTsv(dbauthz.AsSystemRestricted(ctx), 1000) + require.NoError(t, err) + } + + chatIDs := func(chats []codersdk.Chat) map[uuid.UUID]struct{} { + ids := make(map[uuid.UUID]struct{}, len(chats)) + for _, chat := range chats { + ids[chat.ID] = struct{}{} + } + return ids + } + + t.Run("MatchesTitleAndMessageBody", func(t *testing.T) { + t.Parallel() + ctx, client, db, firstUser, modelConfig := setup(t) + + titleMatch := createChat(t, db, firstUser, modelConfig.ID, "kubernetes upgrade notes") + bodyMatch := createChat(t, db, firstUser, modelConfig.ID, "plain title") + insertMessage(t, db, firstUser, modelConfig.ID, bodyMatch.ID, "restart the kubernetes cluster") + noMatch := createChat(t, db, firstUser, modelConfig.ID, "unrelated chat") + insertMessage(t, db, firstUser, modelConfig.ID, noMatch.ID, "terraform apply failure") + backfillSearchTsv(ctx, t, db) + + chats, err := client.ListChats(ctx, &codersdk.ListChatsOptions{ + Query: `search:"kubernetes"`, + }) + require.NoError(t, err) + ids := chatIDs(chats) + require.Contains(t, ids, titleMatch.ID) + require.Contains(t, ids, bodyMatch.ID) + require.NotContains(t, ids, noMatch.ID) + }) + + t.Run("NoSearchableWordsReturns400", func(t *testing.T) { + t.Parallel() + ctx, client, _, _, _ := setup(t) + + _, err := client.ListChats(ctx, &codersdk.ListChatsOptions{ + Query: `search:"!!!"`, + }) + sdkErr := requireSDKError(t, err, http.StatusBadRequest) + require.Len(t, sdkErr.Validations, 1) + require.Equal(t, "search", sdkErr.Validations[0].Field) + require.Contains(t, sdkErr.Validations[0].Detail, "no searchable words") + }) + + t.Run("ComposesWithRepoFilterAndArchivedDefault", func(t *testing.T) { + t.Parallel() + ctx, client, db, firstUser, modelConfig := setup(t) + + linkRepo := func(chatID uuid.UUID, remote string) { + t.Helper() + _, err := db.UpsertChatDiffStatusReference( + dbauthz.AsSystemRestricted(ctx), + database.UpsertChatDiffStatusReferenceParams{ + ChatID: chatID, + GitBranch: "main", + GitRemoteOrigin: remote, + StaleAt: time.Now().UTC().Add(time.Hour), + }, + ) + require.NoError(t, err) + } + + bothMatch := createChat(t, db, firstUser, modelConfig.ID, "kubernetes in coder repo") + linkRepo(bothMatch.ID, "git@github.com:acme/widget.git") + searchOnly := createChat(t, db, firstUser, modelConfig.ID, "kubernetes elsewhere") + linkRepo(searchOnly.ID, "git@github.com:acme/other.git") + repoOnly := createChat(t, db, firstUser, modelConfig.ID, "plain title") + linkRepo(repoOnly.ID, "git@github.com:acme/widget.git") + // Matches via message body, not title, so composition also covers + // search_tsv. + bodyMatch := createChat(t, db, firstUser, modelConfig.ID, "quiet title") + linkRepo(bodyMatch.ID, "git@github.com:acme/widget.git") + insertMessage(t, db, firstUser, modelConfig.ID, bodyMatch.ID, "kubernetes rollout stuck") + bodyMatchWrongRepo := createChat(t, db, firstUser, modelConfig.ID, "quiet title two") + linkRepo(bodyMatchWrongRepo.ID, "git@github.com:acme/other.git") + insertMessage(t, db, firstUser, modelConfig.ID, bodyMatchWrongRepo.ID, "kubernetes rollout stuck") + archivedMatch := createChat(t, db, firstUser, modelConfig.ID, "kubernetes archived") + linkRepo(archivedMatch.ID, "git@github.com:acme/widget.git") + _, err := db.ArchiveChatByID(dbauthz.AsSystemRestricted(ctx), archivedMatch.ID) + require.NoError(t, err) + backfillSearchTsv(ctx, t, db) + + chats, err := client.ListChats(ctx, &codersdk.ListChatsOptions{ + Query: `repo:widget search:"kubernetes"`, + }) + require.NoError(t, err) + ids := chatIDs(chats) + require.Contains(t, ids, bothMatch.ID) + require.Contains(t, ids, bodyMatch.ID) + require.NotContains(t, ids, bodyMatchWrongRepo.ID) + require.NotContains(t, ids, searchOnly.ID) + require.NotContains(t, ids, repoOnly.ID) + // Archived chats stay hidden unless archived:true is requested. + require.NotContains(t, ids, archivedMatch.ID) + }) + + t.Run("NoSearchTermUnchanged", func(t *testing.T) { + t.Parallel() + ctx, client, db, firstUser, modelConfig := setup(t) + + chat := createChat(t, db, firstUser, modelConfig.ID, "kubernetes upgrade notes") + other := createChat(t, db, firstUser, modelConfig.ID, "unrelated chat") + + chats, err := client.ListChats(ctx, nil) + require.NoError(t, err) + ids := chatIDs(chats) + require.Contains(t, ids, chat.ID) + require.Contains(t, ids, other.ID) + }) + + t.Run("MutualExclusionWithTitleReturns400", func(t *testing.T) { + t.Parallel() + ctx, client, _, _, _ := setup(t) + + _, err := client.ListChats(ctx, &codersdk.ListChatsOptions{ + Query: `search:alpha title:beta`, + }) + sdkErr := requireSDKError(t, err, http.StatusBadRequest) + require.Len(t, sdkErr.Validations, 1) + require.Equal(t, "search", sdkErr.Validations[0].Field) + require.Contains(t, sdkErr.Validations[0].Detail, `"title"`) + }) +} + func TestWatchChats(t *testing.T) { t.Parallel() diff --git a/docs/reference/api/chats.md b/docs/reference/api/chats.md index 96db32dbeb3..edfd03c8ed3 100644 --- a/docs/reference/api/chats.md +++ b/docs/reference/api/chats.md @@ -19,10 +19,10 @@ Experimental: this endpoint is subject to change. ### Parameters -| Name | In | Type | Required | Description | -|---------|-------|--------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `q` | query | string | false | Search query. Supports title: (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status: as repeated or comma-separated values, source:, diff_url: (quote values containing colons), pr: (exact PR number match), repo: (case-insensitive substring match against git remote origin or URL), pr_title: (case-insensitive PR title substring). Bare terms are not supported; use title: for title filtering. | -| `label` | query | string | false | Filter by label as key:value. Repeat for multiple (AND logic). | +| Name | In | Type | Required | Description | +|---------|-------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `q` | query | string | false | Search query. Supports title: (case-insensitive, quote multi-word values), archived:bool, has_unread:bool, pr_status: as repeated or comma-separated values, source:, diff_url: (quote values containing colons), pr: (exact PR number match), repo: (case-insensitive substring match against git remote origin or URL), pr_title: (case-insensitive PR title substring), search: (full-text search across chat titles, PR titles, PR numbers, and message bodies; quote multi-word values; cannot be combined with title, pr_title, or pr). Bare terms are not supported; use title: or search:. | +| `label` | query | string | false | Filter by label as key:value. Repeat for multiple (AND logic). | ### Example responses From e2f4544a2680691da2a27d881f2457561815050c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 9 Jul 2026 16:31:12 +0100 Subject: [PATCH 06/12] fixup! feat(coderd): add chat search schema --- ..._search_schema.down.sql => 000542_chat_search_schema.down.sql} | 0 ...chat_search_schema.up.sql => 000542_chat_search_schema.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000541_chat_search_schema.down.sql => 000542_chat_search_schema.down.sql} (100%) rename coderd/database/migrations/{000541_chat_search_schema.up.sql => 000542_chat_search_schema.up.sql} (100%) diff --git a/coderd/database/migrations/000541_chat_search_schema.down.sql b/coderd/database/migrations/000542_chat_search_schema.down.sql similarity index 100% rename from coderd/database/migrations/000541_chat_search_schema.down.sql rename to coderd/database/migrations/000542_chat_search_schema.down.sql diff --git a/coderd/database/migrations/000541_chat_search_schema.up.sql b/coderd/database/migrations/000542_chat_search_schema.up.sql similarity index 100% rename from coderd/database/migrations/000541_chat_search_schema.up.sql rename to coderd/database/migrations/000542_chat_search_schema.up.sql From 21f44f733e9dfc6b3d87d3654e82f80e6cb5c99e Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 9 Jul 2026 16:32:02 +0100 Subject: [PATCH 07/12] fixup! feat(coderd): add chat search schema --- coderd/database/queries/chats.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 093cecf566e..91d2399f37a 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -325,7 +325,7 @@ WITH batch AS ( LIMIT @batch_size::int ) UPDATE chat_messages cm --- NULL means "pending", '' means "backfilled, no text". +-- NULL means "pending", empty tsvector means "backfilled, no text". SET search_tsv = COALESCE( to_tsvector('simple', chat_message_search_text(cm.content)), ''::tsvector) From 80a90717e809160607f91fb76e023be86bfee138 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 9 Jul 2026 16:33:43 +0100 Subject: [PATCH 08/12] fixup! feat(coderd): add chat search schema --- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 34ab3808ad0..77dc728637d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -74,7 +74,7 @@ type sqlcQuerier interface { // The WHERE clause must match the predicate of // idx_chat_messages_search_tsv_pending exactly so the partial index // serves this query. - // NULL means "pending", '' means "backfilled, no text". + // NULL means "pending", empty tsvector means "backfilled, no text". BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) BackoffChatDiffStatus(ctx context.Context, arg BackoffChatDiffStatusParams) error // Deletes heartbeat rows for the supplied (chat_id, runner_id) pairs. diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 07304c02f96..27eb2cf6207 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -6184,7 +6184,7 @@ FROM batch WHERE cm.id = batch.id // The WHERE clause must match the predicate of // idx_chat_messages_search_tsv_pending exactly so the partial index // serves this query. -// NULL means "pending", ” means "backfilled, no text". +// NULL means "pending", empty tsvector means "backfilled, no text". func (q *sqlQuerier) BackfillChatMessagesSearchTsv(ctx context.Context, batchSize int32) (int64, error) { result, err := q.db.ExecContext(ctx, backfillChatMessagesSearchTsv, batchSize) if err != nil { From 04e7b3c07a41ae9924265c3caa67824084fe4b68 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 13 Jul 2026 11:54:53 +0100 Subject: [PATCH 09/12] fixup! feat(coderd): add chat search schema --- coderd/database/migrations/migrate_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/coderd/database/migrations/migrate_test.go b/coderd/database/migrations/migrate_test.go index 8339a4a5570..ae5b2edc489 100644 --- a/coderd/database/migrations/migrate_test.go +++ b/coderd/database/migrations/migrate_test.go @@ -1869,7 +1869,7 @@ func TestMigration000498SoftDeleteStaleWorkspaceAgents(t *testing.T) { // under coderd/coderd_test.go; not retested here. } -func TestMigration000541ChatMessageSearchText(t *testing.T) { +func TestMigration000543ChatMessageSearchText(t *testing.T) { t.Parallel() if testing.Short() { t.SkipNow() @@ -1945,7 +1945,7 @@ const eligibilityPredicate = `deleted = false AND visibility IN ('user', 'both') AND role IN ('user', 'assistant')` -func TestMigration000541ChatSearchSchemaIndexes(t *testing.T) { +func TestMigration000543ChatSearchSchemaIndexes(t *testing.T) { t.Parallel() if testing.Short() { t.SkipNow() @@ -1984,7 +1984,7 @@ func TestMigration000541ChatSearchSchemaIndexes(t *testing.T) { } } -func TestMigration000541ChatSearchSchemaBehavior(t *testing.T) { +func TestMigration000543ChatSearchSchemaBehavior(t *testing.T) { t.Parallel() if testing.Short() { t.SkipNow() From 8b9c85f51d946cd10512d93c43c42497054efb6a Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Tue, 14 Jul 2026 19:12:34 +0100 Subject: [PATCH 10/12] address mafredri comments --- coderd/database/dbauthz/dbauthz.go | 5 ++-- coderd/database/dbpurge/dbpurge.go | 16 +++++----- coderd/database/dbpurge/dbpurge_test.go | 10 +------ coderd/database/dump.sql | 13 +++++++- .../000543_chat_search_schema.up.sql | 30 +++++++++++-------- coderd/database/models.go | 3 +- coderd/database/queries.sql.go | 12 ++++---- coderd/database/queries/chats.sql | 12 ++++---- 8 files changed, 56 insertions(+), 45 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 62de9665dd3..7d3cab4c218 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -697,8 +697,8 @@ var ( rbac.ResourceApiKey.Type: {policy.ActionDelete}, rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete}, rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionDelete}, - // Chat auto-archive sets archived=true on inactive chats; the - // search_tsv backfill also updates chat messages. + // Chat auto-archive sets archived=true on inactive chats and computes + // search_tsv tsvector for chat_messages. rbac.ResourceChat.Type: {policy.ActionRead, policy.ActionUpdate}, // Purge old boundary logs past the retention period. rbac.ResourceBoundaryLog.Type: {policy.ActionDelete}, @@ -1838,7 +1838,6 @@ func (q *querier) CalculateAIBridgeInterceptionsTelemetrySummary(ctx context.Con } func (q *querier) ChatSearchQueryIsEmpty(ctx context.Context, search string) (bool, error) { - // Pure function, no rows. Gates on chat read to match the listing caller. if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceChat); err != nil { return false, err } diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index ca658801397..ea29969be86 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -51,9 +51,10 @@ const ( // Chat debug run deletions can cascade into steps with large JSONB // payloads, so they use the same conservative batch size. chatDebugRunsBatchSize = 1000 - // 10k rows take ~800ms; capping at 5 batches bounds per-tick - // transaction growth at a few seconds. Larger backlogs drain - // across ticks. + // Chat search tsvector backfill is capped at 5 batches of 10k + // rows per tick. Benchmarks on a dogfood-class machine (EPYC 9454P) + // with containerized Postgres were measured to take ~800ms per batch. + // This is considered acceptable but may need dialing in later. chatSearchBackfillBatchSize = 10000 chatSearchBackfillMaxBatches = 5 ) @@ -100,7 +101,6 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder }, []string{"record_type"}) reg.MustRegister(recordsPurged) - // Separate counter: the backfill updates rows, not purges them. chatSearchRowsBackfilled := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "coderd", Subsystem: "dbpurge", @@ -335,9 +335,11 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. } } - // Safe to run incrementally: queue membership is per-row, content is - // immutable after insert, and soft-deleted rows leave the index - // automatically. + // Backfill search_tsv tsvector on chat_messages in batches. Doing this here because it's + // potentially too much for a regular migration, especially on larger deployments: + // - Each row with search_tsv = NULL is present in idx_chat_messages_search_tsv_pending. + // - Content of chat_messages is not changed after insert. + // - Rows that are soft-deleted are no longer part of the index. var backfilledChatSearchRows int64 for range i.chatSearchBackfillMaxBatches { n, err := tx.BackfillChatMessagesSearchTsv(ctx, i.chatSearchBackfillBatchSize) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index cd3fd0d3103..25e780ec3d2 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -2865,12 +2865,6 @@ func TestDeleteOldChatFiles(t *testing.T) { } } -// awaitDoTicks returns a function that blocks until the next purge tick -// completes. The first call waits for the initial tick fired by -// dbpurge.New; each subsequent call advances the mock clock by the purge -// interval and waits for that tick to finish. Unlike awaitDoTick, it does -// not trigger an extra trailing tick, so database state is stable between -// calls. tick may be called at most n times. func awaitDoTicks(ctx context.Context, t *testing.T, clk *quartz.Mock, n int) func() { t.Helper() completed := make(chan struct{}) @@ -2883,7 +2877,6 @@ func awaitDoTicks(ctx context.Context, t *testing.T, clk *quartz.Mock, n int) fu defer trapReset.Close() defer trapStop.Close() defer trapNow.Close() - // Initial tick: Now() trap. Completion: TickerReset trap. trapNow.MustWait(ctx).MustRelease(ctx) trapReset.MustWait(ctx).MustRelease(ctx) select { @@ -2902,7 +2895,6 @@ func awaitDoTicks(ctx context.Context, t *testing.T, clk *quartz.Mock, n int) fu return } w.MustWait(ctx) - // The purge loop stops the ticker, runs doTick, then resets. trapStop.MustWait(ctx).MustRelease(ctx) trapReset.MustWait(ctx).MustRelease(ctx) select { @@ -2978,7 +2970,7 @@ func TestBackfillChatMessagesSearchTsv(t *testing.T) { _, err := rawDB.ExecContext(ctx, "UPDATE chat_messages SET deleted = true WHERE id = $1", id) require.NoError(t, err) } - // Repeats the predicate of idx_chat_messages_search_tsv_pending. + // The WHERE clause below must match the predicate of idx_chat_messages_search_tsv_pending. countPending := func(ctx context.Context, t *testing.T, rawDB *sql.DB) int { t.Helper() var count int diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 3fdab7f5e7c..30e13aa295a 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -796,6 +796,8 @@ CREATE FUNCTION chat_message_search_text(content jsonb) RETURNS text ) END $$; +COMMENT ON FUNCTION chat_message_search_text(content jsonb) IS 'Extracts searchable content from chat_messages. Returns NULL for scalar JSON strings (content_version=0). Immutable as it is used in indexes.'; + CREATE FUNCTION check_workspace_agent_name_unique() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1412,6 +1414,8 @@ BEGIN END; $$; +COMMENT ON FUNCTION set_chat_message_revision_before() IS 'Component of chatd. Updates chat_snapshot_version when any fields of chat_messages change. Excludes changes to search_tsv as it is not relevant to chatd''s processing loop.'; + CREATE FUNCTION sync_chat_retry_state() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -1462,7 +1466,6 @@ BEGIN SELECT DISTINCT n.chat_id FROM chat_message_history_new_rows n JOIN chat_message_history_old_rows o ON o.id = n.id - -- jsonb-minus here: transition-table rows have no composite-copy idiom in pure SQL. WHERE (to_jsonb(o) - 'search_tsv') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv') ) AS affected WHERE c.id = affected.chat_id @@ -1474,6 +1477,8 @@ BEGIN END; $$; +COMMENT ON FUNCTION update_chat_history_after_message_update() IS 'Component of chatd. Updates history_version and generation_attempt on chats when chat_messages is updated. Excludes changes to search_tsv.'; + CREATE TABLE ai_gateway_keys ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, @@ -1969,6 +1974,8 @@ CREATE TABLE chat_messages ( COMMENT ON COLUMN chat_messages.reasoning_effort IS 'Stores the selected effort for the turn triggered by this message.'; +COMMENT ON COLUMN chat_messages.search_tsv IS 'Used for full text search. NULL initially, populated async via background job.'; + CREATE SEQUENCE chat_messages_id_seq START WITH 1 INCREMENT BY 1 @@ -4758,6 +4765,8 @@ CREATE INDEX idx_chat_messages_owner_spend ON chat_messages USING btree (chat_id CREATE INDEX idx_chat_messages_search_tsv ON chat_messages USING gin (search_tsv) WHERE ((search_tsv IS NOT NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); +COMMENT ON INDEX idx_chat_messages_search_tsv IS 'Partial index over chat_messages used for populating search_tsv in the background. Only defined over ''searchable'' rows of chat_messages where search_tsv is NULL.'; + CREATE INDEX idx_chat_messages_search_tsv_pending ON chat_messages USING btree (id DESC) WHERE ((search_tsv IS NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); CREATE INDEX idx_chat_messages_user_prompts ON chat_messages USING btree (chat_id, id DESC) WHERE ((deleted = false) AND (role = 'user'::chat_message_role) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility]))); @@ -4790,6 +4799,8 @@ CREATE INDEX idx_chats_root_chat_id ON chats USING btree (root_chat_id); CREATE INDEX idx_chats_title_fts ON chats USING gin (to_tsvector('simple'::regconfig, title)); +COMMENT ON INDEX idx_chats_title_fts IS 'Used for full text search. Defined over all rows of the chats table.'; + CREATE INDEX idx_chats_worker_acquisition_candidates ON chats USING btree (status, updated_at, id) WHERE (archived = false); CREATE INDEX idx_chats_workspace ON chats USING btree (workspace_id); diff --git a/coderd/database/migrations/000543_chat_search_schema.up.sql b/coderd/database/migrations/000543_chat_search_schema.up.sql index eaa4ec95f29..0101e4933f4 100644 --- a/coderd/database/migrations/000543_chat_search_schema.up.sql +++ b/coderd/database/migrations/000543_chat_search_schema.up.sql @@ -1,6 +1,3 @@ --- CASE guard: jsonb_array_elements raises on non-array input. Legacy --- content_version=0 rows store scalar JSON strings; excluded from search --- by design. IMMUTABLE for expression index use. CREATE FUNCTION chat_message_search_text(content jsonb) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ SELECT CASE WHEN jsonb_typeof(content) = 'array' THEN ( @@ -10,26 +7,32 @@ LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ ) END $$; +COMMENT ON FUNCTION chat_message_search_text IS 'Extracts searchable content from chat_messages. Returns NULL for scalar JSON strings (content_version=0). Immutable as it is used in indexes.'; + -- Populated by a background sweep, not at insert time. NULL means pending. ALTER TABLE chat_messages ADD COLUMN search_tsv tsvector; +COMMENT ON COLUMN chat_messages.search_tsv IS 'Used for full text search. NULL initially, populated async via background job.'; + CREATE INDEX idx_chat_messages_search_tsv ON chat_messages USING GIN (search_tsv) WHERE ((search_tsv IS NOT NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); --- Sweep uses this to find pending rows. Queries must repeat the full predicate. +COMMENT ON INDEX idx_chat_messages_search_tsv IS 'Partial index over chat_messages used for full text search. Only defined over ''searchable'' rows of chat_messages.'; + CREATE INDEX idx_chat_messages_search_tsv_pending ON chat_messages USING btree (id DESC) WHERE ((search_tsv IS NULL) AND (deleted = false) AND (visibility = ANY (ARRAY['user'::chat_message_visibility, 'both'::chat_message_visibility])) AND (role = ANY (ARRAY['user'::chat_message_role, 'assistant'::chat_message_role]))); -CREATE INDEX idx_chats_title_fts ON chats - USING GIN (to_tsvector('simple', title)); +COMMENT ON INDEX idx_chat_messages_search_tsv IS 'Partial index over chat_messages used for populating search_tsv in the background. Only defined over ''searchable'' rows of chat_messages where search_tsv is NULL.'; + +CREATE INDEX idx_chats_title_fts ON chats USING GIN (to_tsvector('simple', title)); -CREATE INDEX idx_chat_diff_statuses_pr_title_fts ON chat_diff_statuses - USING GIN (to_tsvector('simple', pull_request_title)); +COMMENT ON index idx_chats_title_fts IS 'Used for full text search. Defined over all rows of the chats table.'; + +CREATE INDEX idx_chat_diff_statuses_pr_title_fts ON chat_diff_statuses USING GIN (to_tsvector('simple', pull_request_title)); + +COMMENT ON index idx_chats_title_fts IS 'Used for full text search. Defined over all rows of the chats table.'; --- search_tsv is system-maintained; the backfill must not perturb message --- revision, chat history_version, generation_attempt, or retry state. --- Both triggers therefore ignore changes confined to search_tsv. CREATE OR REPLACE FUNCTION set_chat_message_revision_before() RETURNS trigger AS $$ DECLARE @@ -68,6 +71,8 @@ BEGIN END; $$ LANGUAGE plpgsql; +COMMENT ON FUNCTION set_chat_message_revision_before IS 'Component of chatd. Updates chat_snapshot_version when any fields of chat_messages change. Excludes changes to search_tsv as it is not relevant to chatd''s processing loop.'; + CREATE OR REPLACE FUNCTION update_chat_history_after_message_update() RETURNS trigger AS $$ BEGIN @@ -78,7 +83,6 @@ BEGIN SELECT DISTINCT n.chat_id FROM chat_message_history_new_rows n JOIN chat_message_history_old_rows o ON o.id = n.id - -- jsonb-minus here: transition-table rows have no composite-copy idiom in pure SQL. WHERE (to_jsonb(o) - 'search_tsv') IS DISTINCT FROM (to_jsonb(n) - 'search_tsv') ) AS affected WHERE c.id = affected.chat_id @@ -89,3 +93,5 @@ BEGIN RETURN NULL; END; $$ LANGUAGE plpgsql; + +COMMENT ON FUNCTION update_chat_history_after_message_update IS 'Component of chatd. Updates history_version and generation_attempt on chats when chat_messages is updated. Excludes changes to search_tsv.'; diff --git a/coderd/database/models.go b/coderd/database/models.go index ddfc2989835..2f78533d169 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -5132,7 +5132,8 @@ type ChatMessage struct { Revision int64 `db:"revision" json:"revision"` // Stores the selected effort for the turn triggered by this message. ReasoningEffort NullChatReasoningEffort `db:"reasoning_effort" json:"reasoning_effort"` - SearchTsv interface{} `db:"search_tsv" json:"search_tsv"` + // Used for full text search. NULL initially, populated async via background job. + SearchTsv interface{} `db:"search_tsv" json:"search_tsv"` } type ChatModelConfig struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 1134448a8d2..cfce7b047b5 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8802,7 +8802,7 @@ WHERE -- websearch_to_tsquery accepts quoted phrases, OR, and -negation; -- the 'simple' config folds case and skips stemming. AND CASE - WHEN btrim($16::text, E' \t\n\r') != '' THEN ( + WHEN $16::text != '' THEN ( -- Served by idx_chats_title_fts. to_tsvector('simple', chats_expanded.title) @@ websearch_to_tsquery('simple', $16) -- Served by idx_chat_diff_statuses_pr_title_fts. @@ -8812,8 +8812,9 @@ WHERE WHERE cds.chat_id = chats_expanded.id AND to_tsvector('simple', cds.pull_request_title) @@ websearch_to_tsquery('simple', $16) ) - -- The WHERE clause must repeat the partial predicate of - -- idx_chat_messages_search_tsv exactly so the planner can use it. + -- The WHERE clause must repeat the predicate of the partial index + -- idx_chat_messages_search_tsv so the planner can use it. Additional + -- filters should still be fine. OR EXISTS ( SELECT 1 FROM chat_messages cm @@ -8824,15 +8825,14 @@ WHERE AND cm.role IN ('user', 'assistant') AND cm.search_tsv @@ websearch_to_tsquery('simple', $16) ) - -- CASE forces the digits guard before the ::bigint cast; AND - -- operand order is not guaranteed. + -- Skip an explicit pr_number lookup unless the search is a valid bigint. OR CASE WHEN $16 ~ '^[0-9]{1,18}$' THEN EXISTS ( SELECT 1 FROM chat_diff_statuses cds WHERE cds.chat_id = chats_expanded.id AND cds.pr_number IS NOT NULL - AND cds.pr_number::bigint = $16::bigint + AND cds.pr_number = $16::bigint ) ELSE false END diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 60f113805bc..9fff81f392d 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -680,7 +680,7 @@ WHERE -- websearch_to_tsquery accepts quoted phrases, OR, and -negation; -- the 'simple' config folds case and skips stemming. AND CASE - WHEN btrim(@search::text, E' \t\n\r') != '' THEN ( + WHEN @search::text != '' THEN ( -- Served by idx_chats_title_fts. to_tsvector('simple', chats_expanded.title) @@ websearch_to_tsquery('simple', @search) -- Served by idx_chat_diff_statuses_pr_title_fts. @@ -690,8 +690,9 @@ WHERE WHERE cds.chat_id = chats_expanded.id AND to_tsvector('simple', cds.pull_request_title) @@ websearch_to_tsquery('simple', @search) ) - -- The WHERE clause must repeat the partial predicate of - -- idx_chat_messages_search_tsv exactly so the planner can use it. + -- The WHERE clause must repeat the predicate of the partial index + -- idx_chat_messages_search_tsv so the planner can use it. Additional + -- filters should still be fine. OR EXISTS ( SELECT 1 FROM chat_messages cm @@ -702,15 +703,14 @@ WHERE AND cm.role IN ('user', 'assistant') AND cm.search_tsv @@ websearch_to_tsquery('simple', @search) ) - -- CASE forces the digits guard before the ::bigint cast; AND - -- operand order is not guaranteed. + -- Skip an explicit pr_number lookup unless the search is a valid bigint. OR CASE WHEN @search ~ '^[0-9]{1,18}$' THEN EXISTS ( SELECT 1 FROM chat_diff_statuses cds WHERE cds.chat_id = chats_expanded.id AND cds.pr_number IS NOT NULL - AND cds.pr_number::bigint = @search::bigint + AND cds.pr_number = @search::bigint ) ELSE false END From 061e4da187716879904ea819358b3a85df4afd71 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 16 Jul 2026 13:47:58 +0100 Subject: [PATCH 11/12] add dbops comment --- coderd/database/dbpurge/dbpurge.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/coderd/database/dbpurge/dbpurge.go b/coderd/database/dbpurge/dbpurge.go index ea29969be86..7c284339d0e 100644 --- a/coderd/database/dbpurge/dbpurge.go +++ b/coderd/database/dbpurge/dbpurge.go @@ -340,6 +340,8 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time. // - Each row with search_tsv = NULL is present in idx_chat_messages_search_tsv_pending. // - Content of chat_messages is not changed after insert. // - Rows that are soft-deleted are no longer part of the index. + // NOTE: This should not remain in dbpurge and should be adjusted when the "DBOps" gets + // implemented. var backfilledChatSearchRows int64 for range i.chatSearchBackfillMaxBatches { n, err := tx.BackfillChatMessagesSearchTsv(ctx, i.chatSearchBackfillBatchSize) From 76916d840d27c3132e713440bc24a1ddc671a155 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 16 Jul 2026 14:05:55 +0100 Subject: [PATCH 12/12] fix migration numbers --- ..._search_schema.down.sql => 000545_chat_search_schema.down.sql} | 0 ...chat_search_schema.up.sql => 000545_chat_search_schema.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000544_chat_search_schema.down.sql => 000545_chat_search_schema.down.sql} (100%) rename coderd/database/migrations/{000544_chat_search_schema.up.sql => 000545_chat_search_schema.up.sql} (100%) diff --git a/coderd/database/migrations/000544_chat_search_schema.down.sql b/coderd/database/migrations/000545_chat_search_schema.down.sql similarity index 100% rename from coderd/database/migrations/000544_chat_search_schema.down.sql rename to coderd/database/migrations/000545_chat_search_schema.down.sql diff --git a/coderd/database/migrations/000544_chat_search_schema.up.sql b/coderd/database/migrations/000545_chat_search_schema.up.sql similarity index 100% rename from coderd/database/migrations/000544_chat_search_schema.up.sql rename to coderd/database/migrations/000545_chat_search_schema.up.sql