Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: Handle pagination cases where after_id does not exist #1947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 2, 2022
Merged

feat: Handle pagination cases where after_id does not exist #1947

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
132784e
feat: Handle pagination cases where after_id does not exist
Emyrk Jun 1, 2022
0e4a3c0
import ordering
Emyrk Jun 1, 2022
303feb1
feat: Use InTx to ensure db calls are consistent
Emyrk Jun 1, 2022
cffcd71
fix: Use correct db inside InTx
Emyrk Jun 1, 2022
be23937
Ruleguard is acting strange... will move to another PR
Emyrk Jun 1, 2022
1c6f546
Merge remote-tracking branch 'origin/main' into stevenmasley/paginati…
Emyrk Jun 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions coderd/database/databasefake/databasefake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,17 +231,19 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams
users = tmp
}

if len(params.Status) > 0 {
usersFilteredByStatus := make([]database.User, 0, len(users))
for i, user := range users {
for _, status := range params.Status {
if user.Status == status {
usersFilteredByStatus = append(usersFilteredByStatus, users[i])
}
if len(params.Status) == 0 {
params.Status = []database.UserStatus{database.UserStatusActive}
}
Comment on lines +234 to +236
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

postgres defaults to an "active" filter.


usersFilteredByStatus := make([]database.User, 0, len(users))
for i, user := range users {
for _, status := range params.Status {
if user.Status == status {
usersFilteredByStatus = append(usersFilteredByStatus, users[i])
}
}
users = usersFilteredByStatus
}
users = usersFilteredByStatus

if params.OffsetOpt > 0 {
if int(params.OffsetOpt) > len(users)-1 {
Expand Down
17 changes: 17 additions & 0 deletions coderd/templateversions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,23 @@ func (api *API) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque
return
}

if paginationParams.AfterID != uuid.Nil {
// See if the record exists first. If the record does not exist, the pagination
// query will not work.
_, err := api.Database.GetTemplateVersionByID(r.Context(), paginationParams.AfterID)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this almost entirely solves the problem, but not 100%. It handles the case where the AfterID is bogus (never existed), and where the row was deleted before the current request started, but there's still a possibility that the row is deleted between GetTemplateVersionByID and GetTemplateVersionsByTemplateID.

(Or alternatively, if the DB is using something like RDS with read-replicas, the row could appear to be deleted if the two queries hit different replicas, which would break the assumption of sequential consistency. Not sure if that's something we plan to support.)

I guess even if that race happened, the consequence would just be falling back to the current behavior of returning an empty resultset, so this is still a big improvement.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yea, I was trying to do it in postgres to prevent that tiny window @dwahler, but I don't think you can raise an exception in a query. It has to be a postgres procedure, which is a headache.

I think this is good enough.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am dumb, @dwahler I'll put both db calls in a transaction.

if err != nil && xerrors.Is(err, sql.ErrNoRows) {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("record at \"after_id\" (%q) does not exists", paginationParams.AfterID.String()),
})
return
} else if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get template version at after_id: %s", err),
})
return
}
}

apiVersion := []codersdk.TemplateVersion{}
versions, err := api.Database.GetTemplateVersionsByTemplateID(r.Context(), database.GetTemplateVersionsByTemplateIDParams{
TemplateID: template.ID,
Expand Down
21 changes: 16 additions & 5 deletions coderd/templateversions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,9 +694,10 @@ func TestPaginatedTemplateVersions(t *testing.T) {
pagination codersdk.Pagination
}
tests := []struct {
name string
args args
want []codersdk.TemplateVersion
name string
args args
want []codersdk.TemplateVersion
expectedError string
}{
{
name: "Single result",
Expand Down Expand Up @@ -728,6 +729,11 @@ func TestPaginatedTemplateVersions(t *testing.T) {
args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 2, Offset: 10}},
want: []codersdk.TemplateVersion{},
},
{
name: "After_id does not exist",
args: args{ctx: ctx, pagination: codersdk.Pagination{AfterID: uuid.New()}},
expectedError: "does not exist",
},
}
for _, tt := range tests {
tt := tt
Expand All @@ -737,8 +743,13 @@ func TestPaginatedTemplateVersions(t *testing.T) {
TemplateID: template.ID,
Pagination: tt.args.pagination,
})
assert.NoError(t, err)
assert.Equal(t, tt.want, got)
if tt.expectedError != "" {
require.Error(t, err)
require.ErrorContains(t, err, tt.expectedError)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.want, got)
}
})
}
}
Expand Down
44 changes: 44 additions & 0 deletions coderd/users_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -735,6 +735,50 @@ func TestWorkspacesByUser(t *testing.T) {
})
}

// TestSuspendedPagination is when the after_id is a suspended record.
// The database query should still return the correct page, as the after_id
// is in a subquery that finds the record regardless of its status.
// This is mainly to confirm the db fake has the same behavior.
func TestSuspendedPagination(t *testing.T) {
t.Parallel()
ctx := context.Background()
client := coderdtest.New(t, &coderdtest.Options{APIRateLimit: -1})
coderdtest.CreateFirstUser(t, client)
me, err := client.User(context.Background(), codersdk.Me)
require.NoError(t, err)
orgID := me.OrganizationIDs[0]

total := 10
users := make([]codersdk.User, 0, total)
// Create users
for i := 0; i < total; i++ {
email := fmt.Sprintf("%[email protected]", i)
username := fmt.Sprintf("user%d", i)
user, err := client.CreateUser(context.Background(), codersdk.CreateUserRequest{
Email: email,
Username: username,
Password: "password",
OrganizationID: orgID,
})
require.NoError(t, err)
users = append(users, user)
}
sortUsers(users)
deletedUser := users[2]
expected := users[3:8]
_, err = client.UpdateUserStatus(ctx, deletedUser.ID.String(), codersdk.UserStatusSuspended)
require.NoError(t, err, "suspend user")

page, err := client.Users(ctx, codersdk.UsersRequest{
Pagination: codersdk.Pagination{
Limit: len(expected),
AfterID: deletedUser.ID,
},
})
require.NoError(t, err)
require.Equal(t, expected, page, "expected page")
}

// TestPaginatedUsers creates a list of users, then tries to paginate through
// them using different page sizes.
func TestPaginatedUsers(t *testing.T) {
Expand Down
18 changes: 18 additions & 0 deletions coderd/workspacebuilds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,24 @@ func (api *API) workspaceBuilds(rw http.ResponseWriter, r *http.Request) {
if !ok {
return
}

if paginationParams.AfterID != uuid.Nil {
// See if the record exists first. If the record does not exist, the pagination
// query will not work.
_, err := api.Database.GetWorkspaceBuildByID(r.Context(), paginationParams.AfterID)
if err != nil && xerrors.Is(err, sql.ErrNoRows) {
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
Message: fmt.Sprintf("record at \"after_id\" (%q) does not exist", paginationParams.AfterID.String()),
})
return
} else if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
Message: fmt.Sprintf("get workspace build at after_id: %s", err),
})
return
}
}

req := database.GetWorkspaceBuildByWorkspaceIDParams{
WorkspaceID: workspace.ID,
AfterID: paginationParams.AfterID,
Expand Down
26 changes: 26 additions & 0 deletions coderd/workspacebuilds_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"testing"
"time"

"github.com/google/uuid"

"github.com/stretchr/testify/require"

"github.com/coder/coder/coderd/coderdtest"
Expand Down Expand Up @@ -44,6 +46,30 @@ func TestWorkspaceBuilds(t *testing.T) {
require.NoError(t, err)
})

t.Run("PaginateNonExistentRow", func(t *testing.T) {
t.Parallel()
ctx := context.Background()

client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)

_, err := client.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{
WorkspaceID: workspace.ID,
Pagination: codersdk.Pagination{
AfterID: uuid.New(),
},
})
var apiError *codersdk.Error
require.ErrorAs(t, err, &apiError)
require.Equal(t, http.StatusBadRequest, apiError.StatusCode())
require.Contains(t, apiError.Message, "does not exist")
})

t.Run("PaginateLimitOffset", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
Expand Down