From aacd95edcb1066eb4c1aaff1c409ed29c2d75af7 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Sun, 29 Mar 2026 18:18:48 -0700 Subject: [PATCH 01/11] perf: cap count queries and emit native UUID comparisons for audit/connection logs When audit_logs and connection_logs tables grow to tens of millions of rows, they start causing two performance bottlenecks: 1. COUNT(*) queries become extremely slow due to full sequential scans. Solution: wrap count queries in a subquery with LIMIT 2001 so PostgreSQL stops scanning early. The frontend infers capping from count > 2000 and displays "of 2,000+". 2. Authorized queries (both COUNT and SELECT) are slowed down by RBAC authorization that emits text-based UUID comparisons like 'uuid' = COALESCE(organization_id::text, '') which prevents index usage. Solution: add a UUIDVarMatcher type that emits `organization_id = 'uuid'::uuid` instead, allowing PostgreSQL to use indexes on the organization_id column. Related to: https://linear.app/codercom/issue/PLAT-31/connectionaudit-log-performance-issue --- coderd/audit.go | 3 + coderd/database/modelqueries_internal_test.go | 8 + coderd/database/queries.sql.go | 391 +++++++++--------- coderd/database/queries/auditlogs.sql | 180 ++++---- coderd/database/queries/connectionlogs.sql | 213 +++++----- coderd/rbac/regosql/compile_test.go | 34 ++ coderd/rbac/regosql/configs.go | 4 +- coderd/rbac/regosql/sqltypes/uuid.go | 114 +++++ enterprise/coderd/connectionlog.go | 3 + .../PaginationWidget/PaginationAmount.tsx | 12 +- .../PaginationContainer.mocks.ts | 2 + .../PaginationWidget/PaginationContainer.tsx | 2 + .../PaginationWidget/PaginationWidgetBase.tsx | 7 +- site/src/hooks/usePaginatedQuery.test.ts | 59 +++ site/src/hooks/usePaginatedQuery.ts | 87 +++- 15 files changed, 722 insertions(+), 397 deletions(-) create mode 100644 coderd/rbac/regosql/sqltypes/uuid.go diff --git a/coderd/audit.go b/coderd/audit.go index f1fd7668f75..02dbd65b498 100644 --- a/coderd/audit.go +++ b/coderd/audit.go @@ -67,6 +67,9 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { } // Use the same filters to count the number of audit logs + // NOTE: The count query is capped with LIMIT N+1 to avoid a slow + // sequential scan on a large table. The frontend infers capping + // from count > N. count, err := api.Database.CountAuditLogs(ctx, countFilter) if dbauthz.IsNotAuthorizedError(err) { httpapi.Forbidden(rw) diff --git a/coderd/database/modelqueries_internal_test.go b/coderd/database/modelqueries_internal_test.go index 9e84324b72e..3f425b4347d 100644 --- a/coderd/database/modelqueries_internal_test.go +++ b/coderd/database/modelqueries_internal_test.go @@ -145,5 +145,13 @@ func extractWhereClause(query string) string { // Remove SQL comments whereClause = regexp.MustCompile(`(?m)--.*$`).ReplaceAllString(whereClause, "") + // Normalize indentation so subquery wrapping doesn't cause + // mismatches. + lines := strings.Split(whereClause, "\n") + for i, line := range lines { + lines[i] = strings.TrimLeft(line, " \t") + } + whereClause = strings.Join(lines, "\n") + return strings.TrimSpace(whereClause) } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5287e138143..5dc480e3bec 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2263,93 +2263,98 @@ func (q *sqlQuerier) UpdateAPIKeyByID(ctx context.Context, arg UpdateAPIKeyByIDP } const countAuditLogs = `-- name: CountAuditLogs :one -SELECT COUNT(*) -FROM audit_logs - LEFT JOIN users ON audit_logs.user_id = users.id - LEFT JOIN organizations ON audit_logs.organization_id = organizations.id - -- First join on workspaces to get the initial workspace create - -- to workspace build 1 id. This is because the first create is - -- is a different audit log than subsequent starts. - LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' - AND audit_logs.resource_id = workspaces.id - -- Get the reason from the build if the resource type - -- is a workspace_build - LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' - AND audit_logs.resource_id = wb_build.id - -- Get the reason from the build #1 if this is the first - -- workspace create. - LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' - AND audit_logs.action = 'create' - AND workspaces.id = wb_workspace.workspace_id - AND wb_workspace.build_number = 1 -WHERE - -- Filter resource_type - CASE - WHEN $1::text != '' THEN resource_type = $1::resource_type - ELSE true - END - -- Filter resource_id - AND CASE - WHEN $2::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = $2 - ELSE true - END - -- Filter organization_id - AND CASE - WHEN $3::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = $3 - ELSE true - END - -- Filter by resource_target - AND CASE - WHEN $4::text != '' THEN resource_target = $4 - ELSE true - END - -- Filter action - AND CASE - WHEN $5::text != '' THEN action = $5::audit_action - ELSE true - END - -- Filter by user_id - AND CASE - WHEN $6::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = $6 - ELSE true - END - -- Filter by username - AND CASE - WHEN $7::text != '' THEN user_id = ( - SELECT id - FROM users - WHERE lower(username) = lower($7) - AND deleted = false - ) - ELSE true - END - -- Filter by user_email - AND CASE - WHEN $8::text != '' THEN users.email = $8 - ELSE true - END - -- Filter by date_from - AND CASE - WHEN $9::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= $9 - ELSE true - END - -- Filter by date_to - AND CASE - WHEN $10::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= $10 - ELSE true - END - -- Filter by build_reason - AND CASE - WHEN $11::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = $11 - ELSE true - END - -- Filter request_id - AND CASE - WHEN $12::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = $12 - ELSE true - END - -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs - -- @authorize_filter +SELECT COUNT(*) FROM ( + SELECT 1 + FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 + WHERE + -- Filter resource_type + CASE + WHEN $1::text != '' THEN resource_type = $1::resource_type + ELSE true + END + -- Filter resource_id + AND CASE + WHEN $2::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = $2 + ELSE true + END + -- Filter organization_id + AND CASE + WHEN $3::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = $3 + ELSE true + END + -- Filter by resource_target + AND CASE + WHEN $4::text != '' THEN resource_target = $4 + ELSE true + END + -- Filter action + AND CASE + WHEN $5::text != '' THEN action = $5::audit_action + ELSE true + END + -- Filter by user_id + AND CASE + WHEN $6::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = $6 + ELSE true + END + -- Filter by username + AND CASE + WHEN $7::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower($7) + AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN $8::text != '' THEN users.email = $8 + ELSE true + END + -- Filter by date_from + AND CASE + WHEN $9::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= $9 + ELSE true + END + -- Filter by date_to + AND CASE + WHEN $10::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= $10 + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN $11::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = $11 + ELSE true + END + -- Filter request_id + AND CASE + WHEN $12::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = $12 + ELSE true + END + -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs + -- @authorize_filter + -- Avoid a full sequential scan on a large table: if count > 2000, + -- the frontend will show "of 2000+" + LIMIT 2001 +) AS limited_count ` type CountAuditLogsParams struct { @@ -7185,110 +7190,114 @@ func (q *sqlQuerier) UpsertChatUsageLimitUserOverride(ctx context.Context, arg U } const countConnectionLogs = `-- name: CountConnectionLogs :one -SELECT - COUNT(*) AS count -FROM - connection_logs -JOIN users AS workspace_owner ON - connection_logs.workspace_owner_id = workspace_owner.id -LEFT JOIN users ON - connection_logs.user_id = users.id -JOIN organizations ON - connection_logs.organization_id = organizations.id -WHERE - -- Filter organization_id - CASE - WHEN $1 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.organization_id = $1 - ELSE true - END - -- Filter by workspace owner username - AND CASE - WHEN $2 :: text != '' THEN - workspace_owner_id = ( - SELECT id FROM users - WHERE lower(username) = lower($2) AND deleted = false - ) - ELSE true - END - -- Filter by workspace_owner_id - AND CASE - WHEN $3 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - workspace_owner_id = $3 - ELSE true - END - -- Filter by workspace_owner_email - AND CASE - WHEN $4 :: text != '' THEN - workspace_owner_id = ( - SELECT id FROM users - WHERE email = $4 AND deleted = false - ) - ELSE true - END - -- Filter by type - AND CASE - WHEN $5 :: text != '' THEN - type = $5 :: connection_type - ELSE true - END - -- Filter by user_id - AND CASE - WHEN $6 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - user_id = $6 - ELSE true - END - -- Filter by username - AND CASE - WHEN $7 :: text != '' THEN - user_id = ( - SELECT id FROM users - WHERE lower(username) = lower($7) AND deleted = false - ) - ELSE true - END - -- Filter by user_email - AND CASE - WHEN $8 :: text != '' THEN - users.email = $8 - ELSE true - END - -- Filter by connected_after - AND CASE - WHEN $9 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - connect_time >= $9 - ELSE true - END - -- Filter by connected_before - AND CASE - WHEN $10 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - connect_time <= $10 - ELSE true - END - -- Filter by workspace_id - AND CASE - WHEN $11 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.workspace_id = $11 - ELSE true - END - -- Filter by connection_id - AND CASE - WHEN $12 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.connection_id = $12 - ELSE true - END - -- Filter by whether the session has a disconnect_time - AND CASE - WHEN $13 :: text != '' THEN - (($13 = 'ongoing' AND disconnect_time IS NULL) OR - ($13 = 'completed' AND disconnect_time IS NOT NULL)) AND - -- Exclude web events, since we don't know their close time. - "type" NOT IN ('workspace_app', 'port_forwarding') - ELSE true - END - -- Authorize Filter clause will be injected below in - -- CountAuthorizedConnectionLogs - -- @authorize_filter +SELECT COUNT(*) AS count FROM ( + SELECT 1 + FROM + connection_logs + JOIN users AS workspace_owner ON + connection_logs.workspace_owner_id = workspace_owner.id + LEFT JOIN users ON + connection_logs.user_id = users.id + JOIN organizations ON + connection_logs.organization_id = organizations.id + WHERE + -- Filter organization_id + CASE + WHEN $1 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.organization_id = $1 + ELSE true + END + -- Filter by workspace owner username + AND CASE + WHEN $2 :: text != '' THEN + workspace_owner_id = ( + SELECT id FROM users + WHERE lower(username) = lower($2) AND deleted = false + ) + ELSE true + END + -- Filter by workspace_owner_id + AND CASE + WHEN $3 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + workspace_owner_id = $3 + ELSE true + END + -- Filter by workspace_owner_email + AND CASE + WHEN $4 :: text != '' THEN + workspace_owner_id = ( + SELECT id FROM users + WHERE email = $4 AND deleted = false + ) + ELSE true + END + -- Filter by type + AND CASE + WHEN $5 :: text != '' THEN + type = $5 :: connection_type + ELSE true + END + -- Filter by user_id + AND CASE + WHEN $6 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + user_id = $6 + ELSE true + END + -- Filter by username + AND CASE + WHEN $7 :: text != '' THEN + user_id = ( + SELECT id FROM users + WHERE lower(username) = lower($7) AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN $8 :: text != '' THEN + users.email = $8 + ELSE true + END + -- Filter by connected_after + AND CASE + WHEN $9 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN + connect_time >= $9 + ELSE true + END + -- Filter by connected_before + AND CASE + WHEN $10 :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN + connect_time <= $10 + ELSE true + END + -- Filter by workspace_id + AND CASE + WHEN $11 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.workspace_id = $11 + ELSE true + END + -- Filter by connection_id + AND CASE + WHEN $12 :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.connection_id = $12 + ELSE true + END + -- Filter by whether the session has a disconnect_time + AND CASE + WHEN $13 :: text != '' THEN + (($13 = 'ongoing' AND disconnect_time IS NULL) OR + ($13 = 'completed' AND disconnect_time IS NOT NULL)) AND + -- Exclude web events, since we don't know their close time. + "type" NOT IN ('workspace_app', 'port_forwarding') + ELSE true + END + -- Authorize Filter clause will be injected below in + -- CountAuthorizedConnectionLogs + -- @authorize_filter + -- Avoid a full sequential scan on a large table: if count > 2000, + -- the frontend will show "of 2000+" + LIMIT 2001 +) AS limited_count ` type CountConnectionLogsParams struct { diff --git a/coderd/database/queries/auditlogs.sql b/coderd/database/queries/auditlogs.sql index a1c219e702a..0ff32e143be 100644 --- a/coderd/database/queries/auditlogs.sql +++ b/coderd/database/queries/auditlogs.sql @@ -149,94 +149,98 @@ VALUES ( RETURNING *; -- name: CountAuditLogs :one -SELECT COUNT(*) -FROM audit_logs - LEFT JOIN users ON audit_logs.user_id = users.id - LEFT JOIN organizations ON audit_logs.organization_id = organizations.id - -- First join on workspaces to get the initial workspace create - -- to workspace build 1 id. This is because the first create is - -- is a different audit log than subsequent starts. - LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' - AND audit_logs.resource_id = workspaces.id - -- Get the reason from the build if the resource type - -- is a workspace_build - LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' - AND audit_logs.resource_id = wb_build.id - -- Get the reason from the build #1 if this is the first - -- workspace create. - LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' - AND audit_logs.action = 'create' - AND workspaces.id = wb_workspace.workspace_id - AND wb_workspace.build_number = 1 -WHERE - -- Filter resource_type - CASE - WHEN @resource_type::text != '' THEN resource_type = @resource_type::resource_type - ELSE true - END - -- Filter resource_id - AND CASE - WHEN @resource_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = @resource_id - ELSE true - END - -- Filter organization_id - AND CASE - WHEN @organization_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = @organization_id - ELSE true - END - -- Filter by resource_target - AND CASE - WHEN @resource_target::text != '' THEN resource_target = @resource_target - ELSE true - END - -- Filter action - AND CASE - WHEN @action::text != '' THEN action = @action::audit_action - ELSE true - END - -- Filter by user_id - AND CASE - WHEN @user_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = @user_id - ELSE true - END - -- Filter by username - AND CASE - WHEN @username::text != '' THEN user_id = ( - SELECT id - FROM users - WHERE lower(username) = lower(@username) - AND deleted = false - ) - ELSE true - END - -- Filter by user_email - AND CASE - WHEN @email::text != '' THEN users.email = @email - ELSE true - END - -- Filter by date_from - AND CASE - WHEN @date_from::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= @date_from - ELSE true - END - -- Filter by date_to - AND CASE - WHEN @date_to::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= @date_to - ELSE true - END - -- Filter by build_reason - AND CASE - WHEN @build_reason::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = @build_reason - ELSE true - END - -- Filter request_id - AND CASE - WHEN @request_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = @request_id - ELSE true - END - -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs - -- @authorize_filter -; +SELECT COUNT(*) FROM ( + SELECT 1 + FROM audit_logs + LEFT JOIN users ON audit_logs.user_id = users.id + LEFT JOIN organizations ON audit_logs.organization_id = organizations.id + -- First join on workspaces to get the initial workspace create + -- to workspace build 1 id. This is because the first create is + -- is a different audit log than subsequent starts. + LEFT JOIN workspaces ON audit_logs.resource_type = 'workspace' + AND audit_logs.resource_id = workspaces.id + -- Get the reason from the build if the resource type + -- is a workspace_build + LEFT JOIN workspace_builds wb_build ON audit_logs.resource_type = 'workspace_build' + AND audit_logs.resource_id = wb_build.id + -- Get the reason from the build #1 if this is the first + -- workspace create. + LEFT JOIN workspace_builds wb_workspace ON audit_logs.resource_type = 'workspace' + AND audit_logs.action = 'create' + AND workspaces.id = wb_workspace.workspace_id + AND wb_workspace.build_number = 1 + WHERE + -- Filter resource_type + CASE + WHEN @resource_type::text != '' THEN resource_type = @resource_type::resource_type + ELSE true + END + -- Filter resource_id + AND CASE + WHEN @resource_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN resource_id = @resource_id + ELSE true + END + -- Filter organization_id + AND CASE + WHEN @organization_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.organization_id = @organization_id + ELSE true + END + -- Filter by resource_target + AND CASE + WHEN @resource_target::text != '' THEN resource_target = @resource_target + ELSE true + END + -- Filter action + AND CASE + WHEN @action::text != '' THEN action = @action::audit_action + ELSE true + END + -- Filter by user_id + AND CASE + WHEN @user_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN user_id = @user_id + ELSE true + END + -- Filter by username + AND CASE + WHEN @username::text != '' THEN user_id = ( + SELECT id + FROM users + WHERE lower(username) = lower(@username) + AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN @email::text != '' THEN users.email = @email + ELSE true + END + -- Filter by date_from + AND CASE + WHEN @date_from::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" >= @date_from + ELSE true + END + -- Filter by date_to + AND CASE + WHEN @date_to::timestamp with time zone != '0001-01-01 00:00:00Z' THEN "time" <= @date_to + ELSE true + END + -- Filter by build_reason + AND CASE + WHEN @build_reason::text != '' THEN COALESCE(wb_build.reason::text, wb_workspace.reason::text) = @build_reason + ELSE true + END + -- Filter request_id + AND CASE + WHEN @request_id::uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN audit_logs.request_id = @request_id + ELSE true + END + -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs + -- @authorize_filter + -- Avoid a full sequential scan on a large table: if count > 2000, + -- the frontend will show "of 2000+" + LIMIT 2001 +) AS limited_count; -- name: DeleteOldAuditLogConnectionEvents :exec DELETE FROM audit_logs diff --git a/coderd/database/queries/connectionlogs.sql b/coderd/database/queries/connectionlogs.sql index fc38d1af1ab..7eb8a5bc7e5 100644 --- a/coderd/database/queries/connectionlogs.sql +++ b/coderd/database/queries/connectionlogs.sql @@ -133,111 +133,114 @@ OFFSET @offset_opt; -- name: CountConnectionLogs :one -SELECT - COUNT(*) AS count -FROM - connection_logs -JOIN users AS workspace_owner ON - connection_logs.workspace_owner_id = workspace_owner.id -LEFT JOIN users ON - connection_logs.user_id = users.id -JOIN organizations ON - connection_logs.organization_id = organizations.id -WHERE - -- Filter organization_id - CASE - WHEN @organization_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.organization_id = @organization_id - ELSE true - END - -- Filter by workspace owner username - AND CASE - WHEN @workspace_owner :: text != '' THEN - workspace_owner_id = ( - SELECT id FROM users - WHERE lower(username) = lower(@workspace_owner) AND deleted = false - ) - ELSE true - END - -- Filter by workspace_owner_id - AND CASE - WHEN @workspace_owner_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - workspace_owner_id = @workspace_owner_id - ELSE true - END - -- Filter by workspace_owner_email - AND CASE - WHEN @workspace_owner_email :: text != '' THEN - workspace_owner_id = ( - SELECT id FROM users - WHERE email = @workspace_owner_email AND deleted = false - ) - ELSE true - END - -- Filter by type - AND CASE - WHEN @type :: text != '' THEN - type = @type :: connection_type - ELSE true - END - -- Filter by user_id - AND CASE - WHEN @user_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - user_id = @user_id - ELSE true - END - -- Filter by username - AND CASE - WHEN @username :: text != '' THEN - user_id = ( - SELECT id FROM users - WHERE lower(username) = lower(@username) AND deleted = false - ) - ELSE true - END - -- Filter by user_email - AND CASE - WHEN @user_email :: text != '' THEN - users.email = @user_email - ELSE true - END - -- Filter by connected_after - AND CASE - WHEN @connected_after :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - connect_time >= @connected_after - ELSE true - END - -- Filter by connected_before - AND CASE - WHEN @connected_before :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN - connect_time <= @connected_before - ELSE true - END - -- Filter by workspace_id - AND CASE - WHEN @workspace_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.workspace_id = @workspace_id - ELSE true - END - -- Filter by connection_id - AND CASE - WHEN @connection_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN - connection_logs.connection_id = @connection_id - ELSE true - END - -- Filter by whether the session has a disconnect_time - AND CASE - WHEN @status :: text != '' THEN - ((@status = 'ongoing' AND disconnect_time IS NULL) OR - (@status = 'completed' AND disconnect_time IS NOT NULL)) AND - -- Exclude web events, since we don't know their close time. - "type" NOT IN ('workspace_app', 'port_forwarding') - ELSE true - END - -- Authorize Filter clause will be injected below in - -- CountAuthorizedConnectionLogs - -- @authorize_filter -; +SELECT COUNT(*) AS count FROM ( + SELECT 1 + FROM + connection_logs + JOIN users AS workspace_owner ON + connection_logs.workspace_owner_id = workspace_owner.id + LEFT JOIN users ON + connection_logs.user_id = users.id + JOIN organizations ON + connection_logs.organization_id = organizations.id + WHERE + -- Filter organization_id + CASE + WHEN @organization_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.organization_id = @organization_id + ELSE true + END + -- Filter by workspace owner username + AND CASE + WHEN @workspace_owner :: text != '' THEN + workspace_owner_id = ( + SELECT id FROM users + WHERE lower(username) = lower(@workspace_owner) AND deleted = false + ) + ELSE true + END + -- Filter by workspace_owner_id + AND CASE + WHEN @workspace_owner_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + workspace_owner_id = @workspace_owner_id + ELSE true + END + -- Filter by workspace_owner_email + AND CASE + WHEN @workspace_owner_email :: text != '' THEN + workspace_owner_id = ( + SELECT id FROM users + WHERE email = @workspace_owner_email AND deleted = false + ) + ELSE true + END + -- Filter by type + AND CASE + WHEN @type :: text != '' THEN + type = @type :: connection_type + ELSE true + END + -- Filter by user_id + AND CASE + WHEN @user_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + user_id = @user_id + ELSE true + END + -- Filter by username + AND CASE + WHEN @username :: text != '' THEN + user_id = ( + SELECT id FROM users + WHERE lower(username) = lower(@username) AND deleted = false + ) + ELSE true + END + -- Filter by user_email + AND CASE + WHEN @user_email :: text != '' THEN + users.email = @user_email + ELSE true + END + -- Filter by connected_after + AND CASE + WHEN @connected_after :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN + connect_time >= @connected_after + ELSE true + END + -- Filter by connected_before + AND CASE + WHEN @connected_before :: timestamp with time zone != '0001-01-01 00:00:00Z' THEN + connect_time <= @connected_before + ELSE true + END + -- Filter by workspace_id + AND CASE + WHEN @workspace_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.workspace_id = @workspace_id + ELSE true + END + -- Filter by connection_id + AND CASE + WHEN @connection_id :: uuid != '00000000-0000-0000-0000-000000000000'::uuid THEN + connection_logs.connection_id = @connection_id + ELSE true + END + -- Filter by whether the session has a disconnect_time + AND CASE + WHEN @status :: text != '' THEN + ((@status = 'ongoing' AND disconnect_time IS NULL) OR + (@status = 'completed' AND disconnect_time IS NOT NULL)) AND + -- Exclude web events, since we don't know their close time. + "type" NOT IN ('workspace_app', 'port_forwarding') + ELSE true + END + -- Authorize Filter clause will be injected below in + -- CountAuthorizedConnectionLogs + -- @authorize_filter + -- Avoid a full sequential scan on a large table: if count > 2000, + -- the frontend will show "of 2000+" + LIMIT 2001 +) AS limited_count; -- name: DeleteOldConnectionLogs :execrows WITH old_logs AS ( diff --git a/coderd/rbac/regosql/compile_test.go b/coderd/rbac/regosql/compile_test.go index 9249e890ad4..63f9302d3ab 100644 --- a/coderd/rbac/regosql/compile_test.go +++ b/coderd/rbac/regosql/compile_test.go @@ -298,6 +298,40 @@ neq(input.object.owner, ""); ExpectedSQL: p("'' = 'org-id'"), VariableConverter: regosql.ChatConverter(), }, + { + Name: "AuditLogUUID", + Queries: []string{ + `"8c0b9bdc-a013-4b14-a49b-5747bc335708" = input.object.org_owner`, + `input.object.org_owner != ""`, + `neq(input.object.org_owner, "8c0b9bdc-a013-4b14-a49b-5747bc335708")`, + `input.object.org_owner in {"8c0b9bdc-a013-4b14-a49b-5747bc335708", "05f58202-4bfc-43ce-9ba4-5ff6e0174a71"}`, + `"read" in input.object.acl_group_list[input.object.org_owner]`, + }, + ExpectedSQL: p( + p("audit_logs.organization_id = '8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid") + " OR " + + p("audit_logs.organization_id IS NOT NULL") + " OR " + + p("audit_logs.organization_id != '8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid") + " OR " + + p("audit_logs.organization_id = ANY(ARRAY ['05f58202-4bfc-43ce-9ba4-5ff6e0174a71'::uuid,'8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid])") + " OR " + + "(false)"), + VariableConverter: regosql.AuditLogConverter(), + }, + { + Name: "ConnectionLogUUID", + Queries: []string{ + `"8c0b9bdc-a013-4b14-a49b-5747bc335708" = input.object.org_owner`, + `input.object.org_owner != ""`, + `neq(input.object.org_owner, "8c0b9bdc-a013-4b14-a49b-5747bc335708")`, + `input.object.org_owner in {"8c0b9bdc-a013-4b14-a49b-5747bc335708"}`, + `"read" in input.object.acl_group_list[input.object.org_owner]`, + }, + ExpectedSQL: p( + p("connection_logs.organization_id = '8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid") + " OR " + + p("connection_logs.organization_id IS NOT NULL") + " OR " + + p("connection_logs.organization_id != '8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid") + " OR " + + p("connection_logs.organization_id = ANY(ARRAY ['8c0b9bdc-a013-4b14-a49b-5747bc335708'::uuid])") + " OR " + + "(false)"), + VariableConverter: regosql.ConnectionLogConverter(), + }, } for _, tc := range testCases { diff --git a/coderd/rbac/regosql/configs.go b/coderd/rbac/regosql/configs.go index 4f156e8a26a..2066d934739 100644 --- a/coderd/rbac/regosql/configs.go +++ b/coderd/rbac/regosql/configs.go @@ -53,7 +53,7 @@ func WorkspaceConverter() *sqltypes.VariableConverter { func AuditLogConverter() *sqltypes.VariableConverter { matcher := sqltypes.NewVariableConverter().RegisterMatcher( resourceIDMatcher(), - sqltypes.StringVarMatcher("COALESCE(audit_logs.organization_id :: text, '')", []string{"input", "object", "org_owner"}), + sqltypes.UUIDVarMatcher("audit_logs.organization_id", []string{"input", "object", "org_owner"}), // Audit logs have no user owner, only owner by an organization. sqltypes.AlwaysFalse(userOwnerMatcher()), ) @@ -67,7 +67,7 @@ func AuditLogConverter() *sqltypes.VariableConverter { func ConnectionLogConverter() *sqltypes.VariableConverter { matcher := sqltypes.NewVariableConverter().RegisterMatcher( resourceIDMatcher(), - sqltypes.StringVarMatcher("COALESCE(connection_logs.organization_id :: text, '')", []string{"input", "object", "org_owner"}), + sqltypes.UUIDVarMatcher("connection_logs.organization_id", []string{"input", "object", "org_owner"}), // Connection logs have no user owner, only owner by an organization. sqltypes.AlwaysFalse(userOwnerMatcher()), ) diff --git a/coderd/rbac/regosql/sqltypes/uuid.go b/coderd/rbac/regosql/sqltypes/uuid.go new file mode 100644 index 00000000000..bcf95c8411a --- /dev/null +++ b/coderd/rbac/regosql/sqltypes/uuid.go @@ -0,0 +1,114 @@ +package sqltypes + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" + "golang.org/x/xerrors" +) + +var ( + _ VariableMatcher = astUUIDVar{} + _ Node = astUUIDVar{} + _ SupportsEquality = astUUIDVar{} +) + +// astUUIDVar is a variable that represents a UUID column. Unlike +// astStringVar it emits native UUID comparisons (column = 'val'::uuid) +// instead of text-based ones (COALESCE(column::text, ”) = 'val'). +// This allows PostgreSQL to use indexes on UUID columns. +type astUUIDVar struct { + Source RegoSource + FieldPath []string + ColumnString string +} + +func UUIDVarMatcher(sqlColumn string, regoPath []string) VariableMatcher { + return astUUIDVar{FieldPath: regoPath, ColumnString: sqlColumn} +} + +func (astUUIDVar) UseAs() Node { return astUUIDVar{} } + +func (u astUUIDVar) ConvertVariable(rego ast.Ref) (Node, bool) { + left, err := RegoVarPath(u.FieldPath, rego) + if err == nil && len(left) == 0 { + return astUUIDVar{ + Source: RegoSource(rego.String()), + FieldPath: u.FieldPath, + ColumnString: u.ColumnString, + }, true + } + + return nil, false +} + +func (u astUUIDVar) SQLString(_ *SQLGenerator) string { + return u.ColumnString +} + +// EqualsSQLString handles equality comparisons for UUID columns. +// Rego always produces string literals, so we accept AstString and +// cast the literal to ::uuid in the output SQL. This lets PG use +// native UUID indexes instead of falling back to text comparisons. +// nolint:revive +func (u astUUIDVar) EqualsSQLString(cfg *SQLGenerator, not bool, other Node) (string, error) { + switch other.UseAs().(type) { + case AstString: + // The other side is a rego string literal like + // "8c0b9bdc-a013-4b14-a49b-5747bc335708". Emit a comparison + // that casts the literal to uuid so PG can use indexes: + // column = 'val'::uuid + // instead of the text-based: + // 'val' = COALESCE(column::text, '') + s, ok := other.(AstString) + if !ok { + return "", xerrors.Errorf("expected AstString, got %T", other) + } + if s.Value == "" { + // Empty string in rego means "no value". Compare the + // column against NULL since UUID columns represent + // absent values as NULL, not empty strings. + op := "IS NULL" + if not { + op = "IS NOT NULL" + } + return fmt.Sprintf("%s %s", u.ColumnString, op), nil + } + return fmt.Sprintf("%s %s '%s'::uuid", + u.ColumnString, equalsOp(not), s.Value), nil + case astUUIDVar: + return basicSQLEquality(cfg, not, u, other), nil + default: + return "", xerrors.Errorf("unsupported equality: %T %s %T", + u, equalsOp(not), other) + } +} + +// ContainedInSQL implements SupportsContainedIn so that a UUID column +// can appear in membership checks like `col = ANY(ARRAY[...])`. The +// array elements are rego strings, so we cast each to ::uuid. +func (u astUUIDVar) ContainedInSQL(_ *SQLGenerator, haystack Node) (string, error) { + arr, ok := haystack.(ASTArray) + if !ok { + return "", xerrors.Errorf("unsupported containedIn: %T in %T", u, haystack) + } + + if len(arr.Value) == 0 { + return "false", nil + } + + // Build ARRAY['uuid1'::uuid, 'uuid2'::uuid, ...] + values := make([]string, 0, len(arr.Value)) + for _, v := range arr.Value { + s, ok := v.(AstString) + if !ok { + return "", xerrors.Errorf("expected AstString array element, got %T", v) + } + values = append(values, fmt.Sprintf("'%s'::uuid", s.Value)) + } + + return fmt.Sprintf("%s = ANY(ARRAY [%s])", + u.ColumnString, + strings.Join(values, ",")), nil +} diff --git a/enterprise/coderd/connectionlog.go b/enterprise/coderd/connectionlog.go index 05e3a40b2d7..6ad6cf03080 100644 --- a/enterprise/coderd/connectionlog.go +++ b/enterprise/coderd/connectionlog.go @@ -49,6 +49,9 @@ func (api *API) connectionLogs(rw http.ResponseWriter, r *http.Request) { // #nosec G115 - Safe conversion as pagination limit is expected to be within int32 range filter.LimitOpt = int32(page.Limit) + // NOTE: The count query is capped with LIMIT N+1 to avoid a slow + // sequential scan on a large table. The frontend infers capping + // from count > N. count, err := api.Database.CountConnectionLogs(ctx, countFilter) if dbauthz.IsNotAuthorizedError(err) { httpapi.Forbidden(rw) diff --git a/site/src/components/PaginationWidget/PaginationAmount.tsx b/site/src/components/PaginationWidget/PaginationAmount.tsx index 5e9f62b3af8..a8d53da276b 100644 --- a/site/src/components/PaginationWidget/PaginationAmount.tsx +++ b/site/src/components/PaginationWidget/PaginationAmount.tsx @@ -7,6 +7,7 @@ type PaginationHeaderProps = { limit: number; totalRecords: number | undefined; currentOffsetStart: number | undefined; + countIsCapped?: boolean; // Temporary escape hatch until Workspaces can be switched over to using // PaginationContainer @@ -18,6 +19,7 @@ export const PaginationAmount: FC = ({ limit, totalRecords, currentOffsetStart, + countIsCapped, className, }) => { const theme = useTheme(); @@ -52,10 +54,16 @@ export const PaginationAmount: FC = ({ {( currentOffsetStart + - Math.min(limit - 1, totalRecords - currentOffsetStart) + (countIsCapped + ? limit - 1 + : Math.min(limit - 1, totalRecords - currentOffsetStart)) ).toLocaleString()} {" "} - of {totalRecords.toLocaleString()}{" "} + of{" "} + + {totalRecords.toLocaleString()} + {countIsCapped && "+"} + {" "} {paginationUnitLabel} )} diff --git a/site/src/components/PaginationWidget/PaginationContainer.mocks.ts b/site/src/components/PaginationWidget/PaginationContainer.mocks.ts index e638e1e3db7..f90a6ccec97 100644 --- a/site/src/components/PaginationWidget/PaginationContainer.mocks.ts +++ b/site/src/components/PaginationWidget/PaginationContainer.mocks.ts @@ -18,6 +18,7 @@ export const mockPaginationResultBase: ResultBase = { limit: 25, hasNextPage: false, hasPreviousPage: false, + countIsCapped: false, goToPreviousPage: () => {}, goToNextPage: () => {}, goToFirstPage: () => {}, @@ -33,6 +34,7 @@ export const mockInitialRenderResult: PaginationResult = { hasPreviousPage: false, totalRecords: undefined, totalPages: undefined, + countIsCapped: false as const, }; export const mockSuccessResult: PaginationResult = { diff --git a/site/src/components/PaginationWidget/PaginationContainer.tsx b/site/src/components/PaginationWidget/PaginationContainer.tsx index b8ebd7d7b72..ce43d98e47c 100644 --- a/site/src/components/PaginationWidget/PaginationContainer.tsx +++ b/site/src/components/PaginationWidget/PaginationContainer.tsx @@ -27,12 +27,14 @@ export const PaginationContainer: FC = ({ totalRecords={query.totalRecords} currentOffsetStart={query.currentOffsetStart} paginationUnitLabel={paginationUnitLabel} + countIsCapped={query.countIsCapped} className="justify-end" /> {query.isSuccess && ( = ({ @@ -21,8 +25,9 @@ export const PaginationWidgetBase: FC = ({ onPageChange, hasPreviousPage, hasNextPage, + totalPages: totalPagesProp, }) => { - const totalPages = Math.ceil(totalRecords / pageSize); + const totalPages = totalPagesProp ?? Math.ceil(totalRecords / pageSize); if (totalPages < 2) { return null; diff --git a/site/src/hooks/usePaginatedQuery.test.ts b/site/src/hooks/usePaginatedQuery.test.ts index 060e44e07cf..a3d6831652a 100644 --- a/site/src/hooks/usePaginatedQuery.test.ts +++ b/site/src/hooks/usePaginatedQuery.test.ts @@ -258,6 +258,65 @@ describe(usePaginatedQuery.name, () => { }); }); + describe("Capped count behavior", () => { + const mockQueryKey = vi.fn(() => ["mock"]); + + // Returns count 2001 (capped) with items on pages up to page 84 + // (84 * 25 = 2100 items total). + const mockCappedQueryFn = vi.fn(({ pageNumber, limit }) => { + const totalItems = 2100; + const offset = (pageNumber - 1) * limit; + const itemsOnPage = Math.max(0, Math.min(limit, totalItems - offset)); + return Promise.resolve({ + data: new Array(itemsOnPage).fill(pageNumber), + count: 2001, + }); + }); + + it("Caps totalRecords at 2000 when count exceeds cap", async () => { + const { result } = await render({ + queryKey: mockQueryKey, + queryFn: mockCappedQueryFn, + }); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.totalRecords).toBe(2000); + }); + + it("hasNextPage is true when count is capped", async () => { + const { result } = await render( + { queryKey: mockQueryKey, queryFn: mockCappedQueryFn }, + "/?page=80", + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.hasNextPage).toBe(true); + }); + + it("Does not redirect to last page when count is capped and page is valid", async () => { + const { result } = await render( + { queryKey: mockQueryKey, queryFn: mockCappedQueryFn }, + "/?page=83", + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + // Should stay on page 83 — not redirect to page 80. + expect(result.current.currentPage).toBe(83); + }); + + it("Redirects to last known page when navigating beyond actual data", async () => { + const { result } = await render( + { queryKey: mockQueryKey, queryFn: mockCappedQueryFn }, + "/?page=999", + ); + + // Page 999 has no items. Should redirect to page 81 + // (ceil(2001 / 25) = 81), the last page guaranteed to + // have data. + await waitFor(() => expect(result.current.currentPage).toBe(81)); + }); + }); + describe("Passing in searchParams property", () => { const mockQueryKey = vi.fn(() => ["mock"]); const mockQueryFn = vi.fn(({ pageNumber, limit }) => diff --git a/site/src/hooks/usePaginatedQuery.ts b/site/src/hooks/usePaginatedQuery.ts index 200674d69c7..85adf0c4e69 100644 --- a/site/src/hooks/usePaginatedQuery.ts +++ b/site/src/hooks/usePaginatedQuery.ts @@ -19,6 +19,14 @@ const DEFAULT_RECORDS_PER_PAGE = 25; */ const PAGE_NUMBER_PARAMS_KEY = "page"; +/** + * Some count queries (audit logs, connection logs) use LIMIT 2001 + * in SQL to avoid slow sequential scans on large tables. When the + * response count exceeds this cap, the true total is unknown and + * the UI displays "2,000+". + */ +const COUNT_CAP = 2000; + /** * A more specialized version of UseQueryOptions built specifically for * paginated queries. @@ -144,16 +152,32 @@ export function usePaginatedQuery< placeholderData: keepPreviousData, }); - const totalRecords = query.data?.count; + // Audit and connection log count queries return at most 2001 + // (COUNT_CAP + 1). A value above COUNT_CAP means the true + // total is unknown, so we clamp to COUNT_CAP for display. + const rawCount = query.data?.count; + const countIsCapped = rawCount !== undefined && rawCount > COUNT_CAP; + const totalRecords = countIsCapped ? COUNT_CAP : rawCount; + // When the count is capped, ensure totalPages is at least + // currentPage so the pagination widget stays visible as the + // user navigates beyond the known range. const totalPages = - totalRecords !== undefined ? Math.ceil(totalRecords / limit) : undefined; - + totalRecords !== undefined + ? Math.max( + Math.ceil(totalRecords / limit), + countIsCapped ? currentPage : 0, + ) + : undefined; + + // When the count is capped, we don't know the true total so there + // is always potentially a next page. const hasNextPage = - totalRecords !== undefined && limit + currentPageOffset < totalRecords; + totalRecords !== undefined && + (countIsCapped || limit + currentPageOffset < totalRecords); const hasPreviousPage = totalRecords !== undefined && currentPage > 1 && - currentPageOffset - limit < totalRecords; + (countIsCapped || currentPageOffset - limit < totalRecords); const queryClient = useQueryClient(); const prefetchPage = useEffectEvent((newPage: number) => { @@ -224,10 +248,48 @@ export function usePaginatedQuery< }); useEffect(() => { - if (!query.isFetching && totalPages !== undefined) { + // When the count is capped we don't know the true total, so + // skip the page-validity check to allow navigating beyond the + // capped page range. + if (!countIsCapped && !query.isFetching && totalPages !== undefined) { void updatePageIfInvalid(totalPages); } - }, [updatePageIfInvalid, query.isFetching, totalPages]); + }, [updatePageIfInvalid, query.isFetching, totalPages, countIsCapped]); + + // When the count is capped and the user navigates to a page + // beyond the actual data, the response contains zero items. + // Redirect to the last page guaranteed to have data, which + // is page 81 for rawCount 2001. + const lastKnownPage = + rawCount !== undefined ? Math.ceil(rawCount / limit) : 1; + useEffect(() => { + if ( + !countIsCapped || + query.isFetching || + !query.isSuccess || + !query.data || + currentPage <= 1 + ) { + return; + } + const hasItems = Object.values(query.data).some( + (v) => Array.isArray(v) && v.length > 0, + ); + if (!hasItems) { + const updated = getParamsWithoutPage(searchParams); + updated.set(PAGE_NUMBER_PARAMS_KEY, String(lastKnownPage)); + setSearchParams(updated); + } + }, [ + countIsCapped, + query.isFetching, + query.isSuccess, + query.data, + currentPage, + lastKnownPage, + searchParams, + setSearchParams, + ]); const onPageChange = (newPage: number) => { // Page 1 is the only page that can be safely navigated to without knowing @@ -236,7 +298,12 @@ export function usePaginatedQuery< return; } - const cleanedInput = clamp(Math.trunc(newPage), 1, totalPages ?? 1); + // When the count is capped we allow navigating beyond the known + // page range, so only enforce a lower bound of 1. + const upperBound = countIsCapped + ? Number.MAX_SAFE_INTEGER + : (totalPages ?? 1); + const cleanedInput = clamp(Math.trunc(newPage), 1, upperBound); if (Number.isNaN(cleanedInput)) { return; } @@ -274,6 +341,7 @@ export function usePaginatedQuery< totalRecords: totalRecords as number, totalPages: totalPages as number, currentOffsetStart: currentPageOffset + 1, + countIsCapped, } : { isSuccess: false, @@ -282,6 +350,7 @@ export function usePaginatedQuery< totalRecords: undefined, totalPages: undefined, currentOffsetStart: undefined, + countIsCapped: false as const, }), }; @@ -323,6 +392,7 @@ export type PaginationResultInfo = { totalRecords: undefined; totalPages: undefined; currentOffsetStart: undefined; + countIsCapped: false; } | { isSuccess: true; @@ -331,6 +401,7 @@ export type PaginationResultInfo = { totalRecords: number; totalPages: number; currentOffsetStart: number; + countIsCapped: boolean; } ); From 6a32221560b66f67626a57a0ca66465f6b626f50 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 09:07:36 -0700 Subject: [PATCH 02/11] replace useEffect for empty-page detection with inline state derivation Also add a test for hasPreviousPage when count is capped. --- site/src/hooks/usePaginatedQuery.test.ts | 10 +++ site/src/hooks/usePaginatedQuery.ts | 102 ++++++++--------------- 2 files changed, 47 insertions(+), 65 deletions(-) diff --git a/site/src/hooks/usePaginatedQuery.test.ts b/site/src/hooks/usePaginatedQuery.test.ts index a3d6831652a..2c58c37268f 100644 --- a/site/src/hooks/usePaginatedQuery.test.ts +++ b/site/src/hooks/usePaginatedQuery.test.ts @@ -293,6 +293,16 @@ describe(usePaginatedQuery.name, () => { expect(result.current.hasNextPage).toBe(true); }); + it("hasPreviousPage is true when count is capped and page is beyond cap", async () => { + const { result } = await render( + { queryKey: mockQueryKey, queryFn: mockCappedQueryFn }, + "/?page=83", + ); + + await waitFor(() => expect(result.current.isSuccess).toBe(true)); + expect(result.current.hasPreviousPage).toBe(true); + }); + it("Does not redirect to last page when count is capped and page is valid", async () => { const { result } = await render( { queryKey: mockQueryKey, queryFn: mockCappedQueryFn }, diff --git a/site/src/hooks/usePaginatedQuery.ts b/site/src/hooks/usePaginatedQuery.ts index 85adf0c4e69..c899d9d8375 100644 --- a/site/src/hooks/usePaginatedQuery.ts +++ b/site/src/hooks/usePaginatedQuery.ts @@ -22,8 +22,8 @@ const PAGE_NUMBER_PARAMS_KEY = "page"; /** * Some count queries (audit logs, connection logs) use LIMIT 2001 * in SQL to avoid slow sequential scans on large tables. When the - * response count exceeds this cap, the true total is unknown and - * the UI displays "2,000+". + * response count exceeds this cap, we assume the true total is + * unknown and the UI displays "2,000+". */ const COUNT_CAP = 2000; @@ -152,32 +152,42 @@ export function usePaginatedQuery< placeholderData: keepPreviousData, }); - // Audit and connection log count queries return at most 2001 - // (COUNT_CAP + 1). A value above COUNT_CAP means the true - // total is unknown, so we clamp to COUNT_CAP for display. - const rawCount = query.data?.count; - const countIsCapped = rawCount !== undefined && rawCount > COUNT_CAP; - const totalRecords = countIsCapped ? COUNT_CAP : rawCount; - // When the count is capped, ensure totalPages is at least - // currentPage so the pagination widget stays visible as the - // user navigates beyond the known range. - const totalPages = - totalRecords !== undefined - ? Math.max( - Math.ceil(totalRecords / limit), - countIsCapped ? currentPage : 0, - ) - : undefined; - - // When the count is capped, we don't know the true total so there - // is always potentially a next page. + const count = query.data?.count; + const countIsCapped = count !== undefined && count > COUNT_CAP; + const totalRecords = countIsCapped ? COUNT_CAP : count; + let totalPages = totalRecords !== undefined + ? Math.max( + Math.ceil(totalRecords / limit), + // True count is not known; let them navigate forward + // until they hit an empty page (checked below). + countIsCapped ? currentPage : 0, + ) + : undefined; + + // When the true count is unknown, the user can navigate past + // all actual data. If that happens, we need to redirect (via + // updatePageIfInvalid) to the last page guaranteed to be not + // empty. + const pageIsEmpty = + query.data !== undefined && + !Object.values(query.data).some( + (v) => Array.isArray(v) && v.length > 0, + ); + if (pageIsEmpty) { + totalPages = count !== undefined + ? Math.ceil(count / limit) + : 1; + } + const hasNextPage = totalRecords !== undefined && - (countIsCapped || limit + currentPageOffset < totalRecords); + ((countIsCapped && !pageIsEmpty) || + limit + currentPageOffset < totalRecords); const hasPreviousPage = totalRecords !== undefined && currentPage > 1 && - (countIsCapped || currentPageOffset - limit < totalRecords); + ((countIsCapped && !pageIsEmpty) || + currentPageOffset - limit < totalRecords); const queryClient = useQueryClient(); const prefetchPage = useEffectEvent((newPage: number) => { @@ -248,48 +258,10 @@ export function usePaginatedQuery< }); useEffect(() => { - // When the count is capped we don't know the true total, so - // skip the page-validity check to allow navigating beyond the - // capped page range. - if (!countIsCapped && !query.isFetching && totalPages !== undefined) { + if (!query.isFetching && totalPages !== undefined && currentPage > totalPages) { void updatePageIfInvalid(totalPages); } - }, [updatePageIfInvalid, query.isFetching, totalPages, countIsCapped]); - - // When the count is capped and the user navigates to a page - // beyond the actual data, the response contains zero items. - // Redirect to the last page guaranteed to have data, which - // is page 81 for rawCount 2001. - const lastKnownPage = - rawCount !== undefined ? Math.ceil(rawCount / limit) : 1; - useEffect(() => { - if ( - !countIsCapped || - query.isFetching || - !query.isSuccess || - !query.data || - currentPage <= 1 - ) { - return; - } - const hasItems = Object.values(query.data).some( - (v) => Array.isArray(v) && v.length > 0, - ); - if (!hasItems) { - const updated = getParamsWithoutPage(searchParams); - updated.set(PAGE_NUMBER_PARAMS_KEY, String(lastKnownPage)); - setSearchParams(updated); - } - }, [ - countIsCapped, - query.isFetching, - query.isSuccess, - query.data, - currentPage, - lastKnownPage, - searchParams, - setSearchParams, - ]); + }, [updatePageIfInvalid, query.isFetching, totalPages, currentPage]); const onPageChange = (newPage: number) => { // Page 1 is the only page that can be safely navigated to without knowing @@ -298,8 +270,8 @@ export function usePaginatedQuery< return; } - // When the count is capped we allow navigating beyond the known - // page range, so only enforce a lower bound of 1. + // If the true count is unknown, we allow navigating past the + // known page range. const upperBound = countIsCapped ? Number.MAX_SAFE_INTEGER : (totalPages ?? 1); From a3e41530802ead149a8f98938bc3f9af25b4c81f Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 11:15:37 -0700 Subject: [PATCH 03/11] pnpm format --- site/src/hooks/usePaginatedQuery.ts | 31 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/site/src/hooks/usePaginatedQuery.ts b/site/src/hooks/usePaginatedQuery.ts index c899d9d8375..51d27136b2a 100644 --- a/site/src/hooks/usePaginatedQuery.ts +++ b/site/src/hooks/usePaginatedQuery.ts @@ -155,14 +155,15 @@ export function usePaginatedQuery< const count = query.data?.count; const countIsCapped = count !== undefined && count > COUNT_CAP; const totalRecords = countIsCapped ? COUNT_CAP : count; - let totalPages = totalRecords !== undefined - ? Math.max( - Math.ceil(totalRecords / limit), - // True count is not known; let them navigate forward - // until they hit an empty page (checked below). - countIsCapped ? currentPage : 0, - ) - : undefined; + let totalPages = + totalRecords !== undefined + ? Math.max( + Math.ceil(totalRecords / limit), + // True count is not known; let them navigate forward + // until they hit an empty page (checked below). + countIsCapped ? currentPage : 0, + ) + : undefined; // When the true count is unknown, the user can navigate past // all actual data. If that happens, we need to redirect (via @@ -170,13 +171,9 @@ export function usePaginatedQuery< // empty. const pageIsEmpty = query.data !== undefined && - !Object.values(query.data).some( - (v) => Array.isArray(v) && v.length > 0, - ); + !Object.values(query.data).some((v) => Array.isArray(v) && v.length > 0); if (pageIsEmpty) { - totalPages = count !== undefined - ? Math.ceil(count / limit) - : 1; + totalPages = count !== undefined ? Math.ceil(count / limit) : 1; } const hasNextPage = @@ -258,7 +255,11 @@ export function usePaginatedQuery< }); useEffect(() => { - if (!query.isFetching && totalPages !== undefined && currentPage > totalPages) { + if ( + !query.isFetching && + totalPages !== undefined && + currentPage > totalPages + ) { void updatePageIfInvalid(totalPages); } }, [updatePageIfInvalid, query.isFetching, totalPages, currentPage]); From fe8c9e45defc177bde62a3677eb5b20c2aaab6e8 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 13:00:38 -0700 Subject: [PATCH 04/11] fix: make count capping opt-in via count_cap response field The frontend used a hardcoded COUNT_CAP = 2000 constant to detect capped counts, which would incorrectly cap any paginated endpoint returning >2000 records. Instead, have the backend include count_cap in the response so the frontend only activates capping when the endpoint explicitly signals it. The SQL LIMIT is also parameterized so the cap value flows from a named Go constant through to the query. --- coderd/apidoc/docs.go | 6 ++++++ coderd/apidoc/swagger.json | 6 ++++++ coderd/audit.go | 11 ++++++---- coderd/database/modelqueries.go | 2 ++ coderd/database/queries.sql.go | 20 +++++++++++++------ coderd/database/queries/auditlogs.sql | 8 +++++--- coderd/database/queries/connectionlogs.sql | 8 +++++--- codersdk/audit.go | 1 + codersdk/connectionlog.go | 1 + docs/reference/api/audit.md | 3 ++- docs/reference/api/enterprise.md | 3 ++- docs/reference/api/schemas.md | 8 ++++++-- enterprise/coderd/connectionlog.go | 10 +++++++--- site/src/api/typesGenerated.ts | 2 ++ site/src/hooks/usePaginatedQuery.test.ts | 1 + site/src/hooks/usePaginatedQuery.ts | 18 ++++++++--------- site/src/pages/AuditPage/AuditPage.test.tsx | 9 ++++++++- .../ConnectionLogPage.test.tsx | 2 ++ 18 files changed, 85 insertions(+), 34 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 78481a71f4c..b0f7b43ee1a 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -14135,6 +14135,9 @@ const docTemplate = `{ }, "count": { "type": "integer" + }, + "count_cap": { + "type": "integer" } } }, @@ -14456,6 +14459,9 @@ const docTemplate = `{ }, "count": { "type": "integer" + }, + "count_cap": { + "type": "integer" } } }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index b6c8ae74906..8449ac8b233 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -12699,6 +12699,9 @@ }, "count": { "type": "integer" + }, + "count_cap": { + "type": "integer" } } }, @@ -12999,6 +13002,9 @@ }, "count": { "type": "integer" + }, + "count_cap": { + "type": "integer" } } }, diff --git a/coderd/audit.go b/coderd/audit.go index 02dbd65b498..34c0d4e14a8 100644 --- a/coderd/audit.go +++ b/coderd/audit.go @@ -26,6 +26,10 @@ import ( "github.com/coder/coder/v2/codersdk" ) +// Limit the count query to avoid a slow sequential scan due to joins +// on a large table. +const auditLogCountCap = 2000 + // @Summary Get audit logs // @ID get-audit-logs // @Security CoderSessionToken @@ -66,10 +70,7 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { countFilter.Username = "" } - // Use the same filters to count the number of audit logs - // NOTE: The count query is capped with LIMIT N+1 to avoid a slow - // sequential scan on a large table. The frontend infers capping - // from count > N. + countFilter.CountCap = auditLogCountCap count, err := api.Database.CountAuditLogs(ctx, countFilter) if dbauthz.IsNotAuthorizedError(err) { httpapi.Forbidden(rw) @@ -84,6 +85,7 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, codersdk.AuditLogResponse{ AuditLogs: []codersdk.AuditLog{}, Count: 0, + CountCap: auditLogCountCap, }) return } @@ -101,6 +103,7 @@ func (api *API) auditLogs(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, codersdk.AuditLogResponse{ AuditLogs: api.convertAuditLogs(ctx, dblogs), Count: count, + CountCap: auditLogCountCap, }) } diff --git a/coderd/database/modelqueries.go b/coderd/database/modelqueries.go index 2b92947a145..4219d263476 100644 --- a/coderd/database/modelqueries.go +++ b/coderd/database/modelqueries.go @@ -584,6 +584,7 @@ func (q *sqlQuerier) CountAuthorizedAuditLogs(ctx context.Context, arg CountAudi arg.DateTo, arg.BuildReason, arg.RequestID, + arg.CountCap, ) if err != nil { return 0, err @@ -720,6 +721,7 @@ func (q *sqlQuerier) CountAuthorizedConnectionLogs(ctx context.Context, arg Coun arg.WorkspaceID, arg.ConnectionID, arg.Status, + arg.CountCap, ) if err != nil { return 0, err diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5dc480e3bec..5f82575e96a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2351,9 +2351,11 @@ SELECT COUNT(*) FROM ( END -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs -- @authorize_filter - -- Avoid a full sequential scan on a large table: if count > 2000, - -- the frontend will show "of 2000+" - LIMIT 2001 + -- Avoid a slow scan on a large table with joins. The caller + -- passes the count cap and we add 1 so the frontend can detect + -- capping and show "... of N+". A cap of 0 means no limit (NULLIF + -- -> NULL + 1 = NULL). + LIMIT NULLIF($13::int, 0) + 1 ) AS limited_count ` @@ -2370,6 +2372,7 @@ type CountAuditLogsParams struct { DateTo time.Time `db:"date_to" json:"date_to"` BuildReason string `db:"build_reason" json:"build_reason"` RequestID uuid.UUID `db:"request_id" json:"request_id"` + CountCap int32 `db:"count_cap" json:"count_cap"` } func (q *sqlQuerier) CountAuditLogs(ctx context.Context, arg CountAuditLogsParams) (int64, error) { @@ -2386,6 +2389,7 @@ func (q *sqlQuerier) CountAuditLogs(ctx context.Context, arg CountAuditLogsParam arg.DateTo, arg.BuildReason, arg.RequestID, + arg.CountCap, ) var count int64 err := row.Scan(&count) @@ -7294,9 +7298,11 @@ SELECT COUNT(*) AS count FROM ( -- Authorize Filter clause will be injected below in -- CountAuthorizedConnectionLogs -- @authorize_filter - -- Avoid a full sequential scan on a large table: if count > 2000, - -- the frontend will show "of 2000+" - LIMIT 2001 + -- Avoid a slow scan on a large table with joins. The caller + -- passes the count cap and we add 1 so the frontend can detect + -- capping and show "... of N+". A cap of 0 means no limit (NULLIF + -- -> NULL + 1 = NULL). + LIMIT NULLIF($14::int, 0) + 1 ) AS limited_count ` @@ -7314,6 +7320,7 @@ type CountConnectionLogsParams struct { WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` ConnectionID uuid.UUID `db:"connection_id" json:"connection_id"` Status string `db:"status" json:"status"` + CountCap int32 `db:"count_cap" json:"count_cap"` } func (q *sqlQuerier) CountConnectionLogs(ctx context.Context, arg CountConnectionLogsParams) (int64, error) { @@ -7331,6 +7338,7 @@ func (q *sqlQuerier) CountConnectionLogs(ctx context.Context, arg CountConnectio arg.WorkspaceID, arg.ConnectionID, arg.Status, + arg.CountCap, ) var count int64 err := row.Scan(&count) diff --git a/coderd/database/queries/auditlogs.sql b/coderd/database/queries/auditlogs.sql index 0ff32e143be..116c863a59f 100644 --- a/coderd/database/queries/auditlogs.sql +++ b/coderd/database/queries/auditlogs.sql @@ -237,9 +237,11 @@ SELECT COUNT(*) FROM ( END -- Authorize Filter clause will be injected below in CountAuthorizedAuditLogs -- @authorize_filter - -- Avoid a full sequential scan on a large table: if count > 2000, - -- the frontend will show "of 2000+" - LIMIT 2001 + -- Avoid a slow scan on a large table with joins. The caller + -- passes the count cap and we add 1 so the frontend can detect + -- capping and show "... of N+". A cap of 0 means no limit (NULLIF + -- -> NULL + 1 = NULL). + LIMIT NULLIF(@count_cap::int, 0) + 1 ) AS limited_count; -- name: DeleteOldAuditLogConnectionEvents :exec diff --git a/coderd/database/queries/connectionlogs.sql b/coderd/database/queries/connectionlogs.sql index 7eb8a5bc7e5..d5a97fe998c 100644 --- a/coderd/database/queries/connectionlogs.sql +++ b/coderd/database/queries/connectionlogs.sql @@ -237,9 +237,11 @@ SELECT COUNT(*) AS count FROM ( -- Authorize Filter clause will be injected below in -- CountAuthorizedConnectionLogs -- @authorize_filter - -- Avoid a full sequential scan on a large table: if count > 2000, - -- the frontend will show "of 2000+" - LIMIT 2001 + -- Avoid a slow scan on a large table with joins. The caller + -- passes the count cap and we add 1 so the frontend can detect + -- capping and show "... of N+". A cap of 0 means no limit (NULLIF + -- -> NULL + 1 = NULL). + LIMIT NULLIF(@count_cap::int, 0) + 1 ) AS limited_count; -- name: DeleteOldConnectionLogs :execrows diff --git a/codersdk/audit.go b/codersdk/audit.go index 5018982c6c6..ac0b4e908fb 100644 --- a/codersdk/audit.go +++ b/codersdk/audit.go @@ -212,6 +212,7 @@ type AuditLogsRequest struct { type AuditLogResponse struct { AuditLogs []AuditLog `json:"audit_logs"` Count int64 `json:"count"` + CountCap int64 `json:"count_cap"` } type CreateTestAuditLogRequest struct { diff --git a/codersdk/connectionlog.go b/codersdk/connectionlog.go index 3e2acec6df6..61e1ccbb307 100644 --- a/codersdk/connectionlog.go +++ b/codersdk/connectionlog.go @@ -96,6 +96,7 @@ type ConnectionLogsRequest struct { type ConnectionLogResponse struct { ConnectionLogs []ConnectionLog `json:"connection_logs"` Count int64 `json:"count"` + CountCap int64 `json:"count_cap"` } func (c *Client) ConnectionLogs(ctx context.Context, req ConnectionLogsRequest) (ConnectionLogResponse, error) { diff --git a/docs/reference/api/audit.md b/docs/reference/api/audit.md index bfdc1a259eb..8ae32c1295d 100644 --- a/docs/reference/api/audit.md +++ b/docs/reference/api/audit.md @@ -90,7 +90,8 @@ curl -X GET http://coder-server:8080/api/v2/audit?limit=0 \ "user_agent": "string" } ], - "count": 0 + "count": 0, + "count_cap": 0 } ``` diff --git a/docs/reference/api/enterprise.md b/docs/reference/api/enterprise.md index 7b16911c5c9..439de03cd33 100644 --- a/docs/reference/api/enterprise.md +++ b/docs/reference/api/enterprise.md @@ -291,7 +291,8 @@ curl -X GET http://coder-server:8080/api/v2/connectionlog?limit=0 \ "workspace_owner_username": "string" } ], - "count": 0 + "count": 0, + "count_cap": 0 } ``` diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 1b9d686c2a2..e2c9e5167bd 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -1703,7 +1703,8 @@ "user_agent": "string" } ], - "count": 0 + "count": 0, + "count_cap": 0 } ``` @@ -1713,6 +1714,7 @@ |--------------|-------------------------------------------------|----------|--------------|-------------| | `audit_logs` | array of [codersdk.AuditLog](#codersdkauditlog) | false | | | | `count` | integer | false | | | +| `count_cap` | integer | false | | | ## codersdk.AuthMethod @@ -2136,7 +2138,8 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in "workspace_owner_username": "string" } ], - "count": 0 + "count": 0, + "count_cap": 0 } ``` @@ -2146,6 +2149,7 @@ AuthorizationObject can represent a "set" of objects, such as: all workspaces in |-------------------|-----------------------------------------------------------|----------|--------------|-------------| | `connection_logs` | array of [codersdk.ConnectionLog](#codersdkconnectionlog) | false | | | | `count` | integer | false | | | +| `count_cap` | integer | false | | | ## codersdk.ConnectionLogSSHInfo diff --git a/enterprise/coderd/connectionlog.go b/enterprise/coderd/connectionlog.go index 6ad6cf03080..d98f2874593 100644 --- a/enterprise/coderd/connectionlog.go +++ b/enterprise/coderd/connectionlog.go @@ -16,6 +16,10 @@ import ( "github.com/coder/coder/v2/codersdk" ) +// Limit the count query to avoid a slow sequential scan due to joins +// on a large table. +const connectionLogCountCap = 2000 + // @Summary Get connection logs // @ID get-connection-logs // @Security CoderSessionToken @@ -49,9 +53,7 @@ func (api *API) connectionLogs(rw http.ResponseWriter, r *http.Request) { // #nosec G115 - Safe conversion as pagination limit is expected to be within int32 range filter.LimitOpt = int32(page.Limit) - // NOTE: The count query is capped with LIMIT N+1 to avoid a slow - // sequential scan on a large table. The frontend infers capping - // from count > N. + countFilter.CountCap = connectionLogCountCap count, err := api.Database.CountConnectionLogs(ctx, countFilter) if dbauthz.IsNotAuthorizedError(err) { httpapi.Forbidden(rw) @@ -66,6 +68,7 @@ func (api *API) connectionLogs(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, codersdk.ConnectionLogResponse{ ConnectionLogs: []codersdk.ConnectionLog{}, Count: 0, + CountCap: connectionLogCountCap, }) return } @@ -83,6 +86,7 @@ func (api *API) connectionLogs(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, codersdk.ConnectionLogResponse{ ConnectionLogs: convertConnectionLogs(dblogs), Count: count, + CountCap: connectionLogCountCap, }) } diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 61941fb84b6..0d2de51c774 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -907,6 +907,7 @@ export interface AuditLog { export interface AuditLogResponse { readonly audit_logs: readonly AuditLog[]; readonly count: number; + readonly count_cap: number; } // From codersdk/audit.go @@ -2234,6 +2235,7 @@ export interface ConnectionLog { export interface ConnectionLogResponse { readonly connection_logs: readonly ConnectionLog[]; readonly count: number; + readonly count_cap: number; } // From codersdk/connectionlog.go diff --git a/site/src/hooks/usePaginatedQuery.test.ts b/site/src/hooks/usePaginatedQuery.test.ts index 2c58c37268f..0ef4f1dc129 100644 --- a/site/src/hooks/usePaginatedQuery.test.ts +++ b/site/src/hooks/usePaginatedQuery.test.ts @@ -270,6 +270,7 @@ describe(usePaginatedQuery.name, () => { return Promise.resolve({ data: new Array(itemsOnPage).fill(pageNumber), count: 2001, + count_cap: 2000, }); }); diff --git a/site/src/hooks/usePaginatedQuery.ts b/site/src/hooks/usePaginatedQuery.ts index 51d27136b2a..7407d6b9cd7 100644 --- a/site/src/hooks/usePaginatedQuery.ts +++ b/site/src/hooks/usePaginatedQuery.ts @@ -19,14 +19,6 @@ const DEFAULT_RECORDS_PER_PAGE = 25; */ const PAGE_NUMBER_PARAMS_KEY = "page"; -/** - * Some count queries (audit logs, connection logs) use LIMIT 2001 - * in SQL to avoid slow sequential scans on large tables. When the - * response count exceeds this cap, we assume the true total is - * unknown and the UI displays "2,000+". - */ -const COUNT_CAP = 2000; - /** * A more specialized version of UseQueryOptions built specifically for * paginated queries. @@ -153,8 +145,13 @@ export function usePaginatedQuery< }); const count = query.data?.count; - const countIsCapped = count !== undefined && count > COUNT_CAP; - const totalRecords = countIsCapped ? COUNT_CAP : count; + const countCap = query.data?.count_cap; + const countIsCapped = + countCap !== undefined && + countCap > 0 && + count !== undefined && + count > countCap; + const totalRecords = countIsCapped ? countCap : count; let totalPages = totalRecords !== undefined ? Math.max( @@ -461,6 +458,7 @@ type QueryPageParamsWithPayload = QueryPageParams & { */ export type PaginatedData = { count: number; + count_cap?: number; }; /** diff --git a/site/src/pages/AuditPage/AuditPage.test.tsx b/site/src/pages/AuditPage/AuditPage.test.tsx index 75d91e7acbc..eaee77bca7a 100644 --- a/site/src/pages/AuditPage/AuditPage.test.tsx +++ b/site/src/pages/AuditPage/AuditPage.test.tsx @@ -71,6 +71,7 @@ describe("AuditPage", () => { const getAuditLogsSpy = vi.spyOn(API, "getAuditLogs").mockResolvedValue({ audit_logs: [MockAuditLog, MockAuditLog2], count: 2, + count_cap: 0, }); // When @@ -90,6 +91,7 @@ describe("AuditPage", () => { vi.spyOn(API, "getAuditLogs").mockResolvedValue({ audit_logs: [MockAuditLog], count: 1, + count_cap: 0, }); await renderPage(); @@ -114,6 +116,7 @@ describe("AuditPage", () => { vi.spyOn(API, "getAuditLogs").mockResolvedValue({ audit_logs: [MockAuditLog], count: 1, + count_cap: 0, }); await renderPage(); @@ -142,7 +145,11 @@ describe("AuditPage", () => { it("filters by URL", async () => { const getAuditLogsSpy = vi .spyOn(API, "getAuditLogs") - .mockResolvedValue({ audit_logs: [MockAuditLog], count: 1 }); + .mockResolvedValue({ + audit_logs: [MockAuditLog], + count: 1, + count_cap: 0, + }); const query = "resource_type:workspace action:create"; await renderPage({ filter: query }); diff --git a/site/src/pages/ConnectionLogPage/ConnectionLogPage.test.tsx b/site/src/pages/ConnectionLogPage/ConnectionLogPage.test.tsx index c555839ab9d..3fff5aa8966 100644 --- a/site/src/pages/ConnectionLogPage/ConnectionLogPage.test.tsx +++ b/site/src/pages/ConnectionLogPage/ConnectionLogPage.test.tsx @@ -69,6 +69,7 @@ describe("ConnectionLogPage", () => { MockDisconnectedSSHConnectionLog, ], count: 2, + count_cap: 0, }); // When @@ -95,6 +96,7 @@ describe("ConnectionLogPage", () => { .mockResolvedValue({ connection_logs: [MockConnectedSSHConnectionLog], count: 1, + count_cap: 0, }); const query = "type:ssh status:ongoing"; From cc5fdfd1259543ed3d284de57d07e992dde150f4 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 15:48:46 -0700 Subject: [PATCH 05/11] make lint and make fmt --- coderd/searchquery/search.go | 3 ++- site/src/pages/AuditPage/AuditPage.test.tsx | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/coderd/searchquery/search.go b/coderd/searchquery/search.go index 330c2e6eb44..260ba792fc5 100644 --- a/coderd/searchquery/search.go +++ b/coderd/searchquery/search.go @@ -66,7 +66,7 @@ func AuditLogs(ctx context.Context, db database.Store, query string) (database.G } // Prepare the count filter, which uses the same parameters as the GetAuditLogsOffsetParams. - // nolint:exhaustruct // UserID is not obtained from the query parameters. + // nolint:exhaustruct // UserID and CountCap are not obtained from the query parameters. countFilter := database.CountAuditLogsParams{ RequestID: filter.RequestID, ResourceID: filter.ResourceID, @@ -123,6 +123,7 @@ func ConnectionLogs(ctx context.Context, db database.Store, query string, apiKey } // This MUST be kept in sync with the above + // nolint:exhaustruct // CountCap is not obtained from the query parameters. countFilter := database.CountConnectionLogsParams{ OrganizationID: filter.OrganizationID, WorkspaceOwner: filter.WorkspaceOwner, diff --git a/site/src/pages/AuditPage/AuditPage.test.tsx b/site/src/pages/AuditPage/AuditPage.test.tsx index eaee77bca7a..46930546830 100644 --- a/site/src/pages/AuditPage/AuditPage.test.tsx +++ b/site/src/pages/AuditPage/AuditPage.test.tsx @@ -143,13 +143,11 @@ describe("AuditPage", () => { describe("Filtering", () => { it("filters by URL", async () => { - const getAuditLogsSpy = vi - .spyOn(API, "getAuditLogs") - .mockResolvedValue({ - audit_logs: [MockAuditLog], - count: 1, - count_cap: 0, - }); + const getAuditLogsSpy = vi.spyOn(API, "getAuditLogs").mockResolvedValue({ + audit_logs: [MockAuditLog], + count: 1, + count_cap: 0, + }); const query = "resource_type:workspace action:create"; await renderPage({ filter: query }); From 064304d0c960cb2d91f395a6d3c844504906254c Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 16:18:37 -0700 Subject: [PATCH 06/11] add comments --- coderd/audit.go | 2 +- enterprise/coderd/connectionlog.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/audit.go b/coderd/audit.go index 34c0d4e14a8..e54abe0d48e 100644 --- a/coderd/audit.go +++ b/coderd/audit.go @@ -27,7 +27,7 @@ import ( ) // Limit the count query to avoid a slow sequential scan due to joins -// on a large table. +// on a large table (set to 0 to disable capping). const auditLogCountCap = 2000 // @Summary Get audit logs diff --git a/enterprise/coderd/connectionlog.go b/enterprise/coderd/connectionlog.go index d98f2874593..70984138eec 100644 --- a/enterprise/coderd/connectionlog.go +++ b/enterprise/coderd/connectionlog.go @@ -17,7 +17,7 @@ import ( ) // Limit the count query to avoid a slow sequential scan due to joins -// on a large table. +// on a large table (set to 0 to disable capping). const connectionLogCountCap = 2000 // @Summary Get connection logs From f07dc60e6e95003ad29748176951b23cbf790f6b Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 17:58:05 -0700 Subject: [PATCH 07/11] guard against null query data in usePaginatedQuery pageIsEmpty check (fixes GroupPage story crash) --- site/src/hooks/usePaginatedQuery.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/src/hooks/usePaginatedQuery.ts b/site/src/hooks/usePaginatedQuery.ts index 7407d6b9cd7..1ad03272a72 100644 --- a/site/src/hooks/usePaginatedQuery.ts +++ b/site/src/hooks/usePaginatedQuery.ts @@ -167,7 +167,7 @@ export function usePaginatedQuery< // updatePageIfInvalid) to the last page guaranteed to be not // empty. const pageIsEmpty = - query.data !== undefined && + query.data != null && !Object.values(query.data).some((v) => Array.isArray(v) && v.length > 0); if (pageIsEmpty) { totalPages = count !== undefined ? Math.ceil(count / limit) : 1; From 9c342b04eab43f679aa0b2cd3409b95be0555c53 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 18:29:27 -0700 Subject: [PATCH 08/11] add storybook stories for capped pagination states --- .../PaginationContainer.mocks.ts | 2 +- .../PaginationContainer.stories.tsx | 53 ++++++++++++++++++- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/site/src/components/PaginationWidget/PaginationContainer.mocks.ts b/site/src/components/PaginationWidget/PaginationContainer.mocks.ts index f90a6ccec97..7466529af55 100644 --- a/site/src/components/PaginationWidget/PaginationContainer.mocks.ts +++ b/site/src/components/PaginationWidget/PaginationContainer.mocks.ts @@ -34,7 +34,7 @@ export const mockInitialRenderResult: PaginationResult = { hasPreviousPage: false, totalRecords: undefined, totalPages: undefined, - countIsCapped: false as const, + countIsCapped: false, }; export const mockSuccessResult: PaginationResult = { diff --git a/site/src/components/PaginationWidget/PaginationContainer.stories.tsx b/site/src/components/PaginationWidget/PaginationContainer.stories.tsx index 23ea7700b7c..1cf3904b3a1 100644 --- a/site/src/components/PaginationWidget/PaginationContainer.stories.tsx +++ b/site/src/components/PaginationWidget/PaginationContainer.stories.tsx @@ -94,7 +94,7 @@ export const FirstPageWithTonsOfData: Story = { currentPage: 2, currentOffsetStart: 1000, totalRecords: 123_456, - totalPages: 1235, + totalPages: 4939, hasPreviousPage: false, hasNextPage: true, isPlaceholderData: false, @@ -135,3 +135,54 @@ export const SecondPageWithData: Story = { children:
New data for page 2
, }, }; + +export const CappedCountFirstPage: Story = { + args: { + query: { + ...mockPaginationResultBase, + isSuccess: true, + currentPage: 1, + currentOffsetStart: 1, + totalRecords: 2000, + totalPages: 80, + hasPreviousPage: false, + hasNextPage: true, + isPlaceholderData: false, + countIsCapped: true, + }, + }, +}; + +export const CappedCountMiddlePage: Story = { + args: { + query: { + ...mockPaginationResultBase, + isSuccess: true, + currentPage: 3, + currentOffsetStart: 51, + totalRecords: 2000, + totalPages: 80, + hasPreviousPage: true, + hasNextPage: true, + isPlaceholderData: false, + countIsCapped: true, + }, + }, +}; + +export const CappedCountBeyondKnownPages: Story = { + args: { + query: { + ...mockPaginationResultBase, + isSuccess: true, + currentPage: 85, + currentOffsetStart: 2101, + totalRecords: 2000, + totalPages: 85, + hasPreviousPage: true, + hasNextPage: true, + isPlaceholderData: false, + countIsCapped: true, + }, + }, +}; From ead4b4a08cbd2e77ac957152496d2518248e279d Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Wed, 1 Apr 2026 21:17:35 -0700 Subject: [PATCH 09/11] add integration test for capped count pagination --- site/src/pages/AuditPage/AuditPage.test.tsx | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/site/src/pages/AuditPage/AuditPage.test.tsx b/site/src/pages/AuditPage/AuditPage.test.tsx index 46930546830..241ef3196db 100644 --- a/site/src/pages/AuditPage/AuditPage.test.tsx +++ b/site/src/pages/AuditPage/AuditPage.test.tsx @@ -178,4 +178,29 @@ describe("AuditPage", () => { ); }); }); + + describe("Capped count", () => { + it("shows capped count indicator and navigates to next page with correct offset", async () => { + vi.spyOn(API, "getAuditLogs").mockResolvedValue({ + audit_logs: [MockAuditLog, MockAuditLog2], + count: 5000, + count_cap: 2000, + }); + + const user = userEvent.setup(); + await renderPage(); + + await screen.findByText(/2,000\+/); + + await user.click(screen.getByRole("button", { name: /next page/i })); + + await waitFor(() => + expect(API.getAuditLogs).toHaveBeenLastCalledWith<[AuditLogsRequest]>({ + limit: DEFAULT_RECORDS_PER_PAGE, + offset: DEFAULT_RECORDS_PER_PAGE, + q: "", + }), + ); + }); + }); }); From 78d9b26614f0d065fa1d42b7f860f4a8d89a84f7 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Thu, 2 Apr 2026 09:01:31 -0700 Subject: [PATCH 10/11] add a comment and fix a test value --- site/src/hooks/usePaginatedQuery.test.ts | 2 ++ site/src/pages/AuditPage/AuditPage.test.tsx | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/site/src/hooks/usePaginatedQuery.test.ts b/site/src/hooks/usePaginatedQuery.test.ts index 0ef4f1dc129..46044f495f3 100644 --- a/site/src/hooks/usePaginatedQuery.test.ts +++ b/site/src/hooks/usePaginatedQuery.test.ts @@ -266,6 +266,8 @@ describe(usePaginatedQuery.name, () => { const mockCappedQueryFn = vi.fn(({ pageNumber, limit }) => { const totalItems = 2100; const offset = (pageNumber - 1) * limit; + // Returns 0 items when the requested page is past the end, simulating + // an empty server response. const itemsOnPage = Math.max(0, Math.min(limit, totalItems - offset)); return Promise.resolve({ data: new Array(itemsOnPage).fill(pageNumber), diff --git a/site/src/pages/AuditPage/AuditPage.test.tsx b/site/src/pages/AuditPage/AuditPage.test.tsx index 241ef3196db..fb0d82a3f3c 100644 --- a/site/src/pages/AuditPage/AuditPage.test.tsx +++ b/site/src/pages/AuditPage/AuditPage.test.tsx @@ -183,7 +183,7 @@ describe("AuditPage", () => { it("shows capped count indicator and navigates to next page with correct offset", async () => { vi.spyOn(API, "getAuditLogs").mockResolvedValue({ audit_logs: [MockAuditLog, MockAuditLog2], - count: 5000, + count: 2001, count_cap: 2000, }); From c85f9a0b987cbfdf75539770c2c4c4265ee94425 Mon Sep 17 00:00:00 2001 From: George Katsitadze Date: Thu, 2 Apr 2026 10:21:11 -0700 Subject: [PATCH 11/11] count cap comments --- coderd/audit.go | 3 ++- coderd/database/queries.sql.go | 10 ++++++---- coderd/database/queries/auditlogs.sql | 5 +++++ coderd/database/queries/connectionlogs.sql | 5 +---- enterprise/coderd/connectionlog.go | 3 +-- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/coderd/audit.go b/coderd/audit.go index e54abe0d48e..3d8aed30052 100644 --- a/coderd/audit.go +++ b/coderd/audit.go @@ -27,7 +27,8 @@ import ( ) // Limit the count query to avoid a slow sequential scan due to joins -// on a large table (set to 0 to disable capping). +// on a large table. Set to 0 to disable capping (but also see the note +// in the SQL query). const auditLogCountCap = 2000 // @Summary Get audit logs diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5f82575e96a..4d3e876e426 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2355,6 +2355,11 @@ SELECT COUNT(*) FROM ( -- passes the count cap and we add 1 so the frontend can detect -- capping and show "... of N+". A cap of 0 means no limit (NULLIF -- -> NULL + 1 = NULL). + -- NOTE: Parameterizing this so that we can easily change from, + -- e.g., 2000 to 5000. However, use literal NULL (or no LIMIT) + -- here if disabling the capping on a large table permanently. + -- This way the PG planner can plan parallel execution for + -- potential large wins. LIMIT NULLIF($13::int, 0) + 1 ) AS limited_count ` @@ -7298,10 +7303,7 @@ SELECT COUNT(*) AS count FROM ( -- Authorize Filter clause will be injected below in -- CountAuthorizedConnectionLogs -- @authorize_filter - -- Avoid a slow scan on a large table with joins. The caller - -- passes the count cap and we add 1 so the frontend can detect - -- capping and show "... of N+". A cap of 0 means no limit (NULLIF - -- -> NULL + 1 = NULL). + -- NOTE: See the CountAuditLogs LIMIT note. LIMIT NULLIF($14::int, 0) + 1 ) AS limited_count ` diff --git a/coderd/database/queries/auditlogs.sql b/coderd/database/queries/auditlogs.sql index 116c863a59f..5a2f9a31e8d 100644 --- a/coderd/database/queries/auditlogs.sql +++ b/coderd/database/queries/auditlogs.sql @@ -241,6 +241,11 @@ SELECT COUNT(*) FROM ( -- passes the count cap and we add 1 so the frontend can detect -- capping and show "... of N+". A cap of 0 means no limit (NULLIF -- -> NULL + 1 = NULL). + -- NOTE: Parameterizing this so that we can easily change from, + -- e.g., 2000 to 5000. However, use literal NULL (or no LIMIT) + -- here if disabling the capping on a large table permanently. + -- This way the PG planner can plan parallel execution for + -- potential large wins. LIMIT NULLIF(@count_cap::int, 0) + 1 ) AS limited_count; diff --git a/coderd/database/queries/connectionlogs.sql b/coderd/database/queries/connectionlogs.sql index d5a97fe998c..6eb9b14ec56 100644 --- a/coderd/database/queries/connectionlogs.sql +++ b/coderd/database/queries/connectionlogs.sql @@ -237,10 +237,7 @@ SELECT COUNT(*) AS count FROM ( -- Authorize Filter clause will be injected below in -- CountAuthorizedConnectionLogs -- @authorize_filter - -- Avoid a slow scan on a large table with joins. The caller - -- passes the count cap and we add 1 so the frontend can detect - -- capping and show "... of N+". A cap of 0 means no limit (NULLIF - -- -> NULL + 1 = NULL). + -- NOTE: See the CountAuditLogs LIMIT note. LIMIT NULLIF(@count_cap::int, 0) + 1 ) AS limited_count; diff --git a/enterprise/coderd/connectionlog.go b/enterprise/coderd/connectionlog.go index 70984138eec..c37e2ce497d 100644 --- a/enterprise/coderd/connectionlog.go +++ b/enterprise/coderd/connectionlog.go @@ -16,8 +16,7 @@ import ( "github.com/coder/coder/v2/codersdk" ) -// Limit the count query to avoid a slow sequential scan due to joins -// on a large table (set to 0 to disable capping). +// NOTE: See the auditLogCountCap note. const connectionLogCountCap = 2000 // @Summary Get connection logs