From 475dcf98d5746bc9b58f2ea2e12564ae522aaffb Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 10:02:43 +0000 Subject: [PATCH 01/21] feat: Implement pagination for template versions --- coderd/database/databasefake/databasefake.go | 45 ++++++++- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 45 ++++++++- coderd/database/queries/templateversions.sql | 31 +++++- coderd/httpapi/httpapi.go | 42 +++++++++ coderd/templates.go | 13 ++- coderd/templates_test.go | 99 +++++++++++++++++++- codersdk/pagination.go | 45 +++++++++ codersdk/templates.go | 11 ++- site/src/api/typesGenerated.ts | 13 +++ 10 files changed, 335 insertions(+), 11 deletions(-) create mode 100644 codersdk/pagination.go diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 850f5605a4ea4..0690a7f6957b7 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -621,20 +621,59 @@ func (q *fakeQuerier) GetTemplateByOrganizationAndName(_ context.Context, arg da return database.Template{}, sql.ErrNoRows } -func (q *fakeQuerier) GetTemplateVersionsByTemplateID(_ context.Context, templateID uuid.UUID) ([]database.TemplateVersion, error) { +func (q *fakeQuerier) GetTemplateVersionsByTemplateID(_ context.Context, arg database.GetTemplateVersionsByTemplateIDParams) (version []database.TemplateVersion, err error) { q.mutex.RLock() defer q.mutex.RUnlock() - version := make([]database.TemplateVersion, 0) for _, templateVersion := range q.templateVersions { - if templateVersion.TemplateID.UUID.String() != templateID.String() { + if templateVersion.TemplateID.UUID.String() != arg.TemplateID.String() { continue } version = append(version, templateVersion) } + + // Database orders by created_at + sort.Slice(version, func(i, j int) bool { + if version[i].CreatedAt.Equal(version[j].CreatedAt) { + // Technically the postgres database also orders by uuid. So match + // that behavior + return version[i].ID.String() < version[j].ID.String() + } + return version[i].CreatedAt.Before(version[j].CreatedAt) + }) + + if arg.AfterID != uuid.Nil { + found := false + for i, v := range version { + if v.ID == arg.AfterID { + version = version[i+1:] + found = true + break + } + } + if !found { + return nil, sql.ErrNoRows + } + } + + if arg.OffsetOpt > 0 { + if int(arg.OffsetOpt) > len(version)-1 { + return nil, sql.ErrNoRows + } + version = version[arg.OffsetOpt:] + } + + if arg.LimitOpt > 0 { + if int(arg.LimitOpt) > len(version) { + arg.LimitOpt = int32(len(version)) + } + version = version[:arg.LimitOpt] + } + if len(version) == 0 { return nil, sql.ErrNoRows } + return version, nil } diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 216db6dcd93fd..4182d15284a18 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -47,7 +47,7 @@ type querier interface { GetTemplateVersionByID(ctx context.Context, id uuid.UUID) (TemplateVersion, error) GetTemplateVersionByJobID(ctx context.Context, jobID uuid.UUID) (TemplateVersion, error) GetTemplateVersionByTemplateIDAndName(ctx context.Context, arg GetTemplateVersionByTemplateIDAndNameParams) (TemplateVersion, error) - GetTemplateVersionsByTemplateID(ctx context.Context, dollar_1 uuid.UUID) ([]TemplateVersion, error) + GetTemplateVersionsByTemplateID(ctx context.Context, arg GetTemplateVersionsByTemplateIDParams) ([]TemplateVersion, error) GetTemplatesByIDs(ctx context.Context, ids []uuid.UUID) ([]Template, error) GetTemplatesByOrganization(ctx context.Context, arg GetTemplatesByOrganizationParams) ([]Template, error) GetUserByEmailOrUsername(ctx context.Context, arg GetUserByEmailOrUsernameParams) (User, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 8392ea278318a..0dbcbed868628 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1908,10 +1908,51 @@ FROM template_versions WHERE template_id = $1 :: uuid + AND CASE + -- This allows using the last element on a page as effectively a cursor. + -- This is an important option for scripts that need to paginate without + -- duplicating or missing data. + WHEN $2 :: uuid != '00000000-00000000-00000000-00000000' THEN ( + -- The pagination cursor is the last user of the previous page. + -- The query is ordered by the created_at field, so select all + -- users after the cursor. We also want to include any users + -- that share the created_at (super rare). + created_at >= ( + SELECT + created_at + FROM + template_versions + WHERE + id = $2 + ) + -- Omit the cursor from the final. + AND id != $2 + ) + ELSE true + END +ORDER BY + -- Deterministic and consistent ordering of all users, even if they share + -- a timestamp. This is to ensure consistent pagination. + (created_at, id) ASC OFFSET $3 +LIMIT + -- A null limit means "no limit", so -1 means return all + NULLIF($4 :: int, -1) ` -func (q *sqlQuerier) GetTemplateVersionsByTemplateID(ctx context.Context, dollar_1 uuid.UUID) ([]TemplateVersion, error) { - rows, err := q.db.QueryContext(ctx, getTemplateVersionsByTemplateID, dollar_1) +type GetTemplateVersionsByTemplateIDParams struct { + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + AfterID uuid.UUID `db:"after_id" json:"after_id"` + OffsetOpt int32 `db:"offset_opt" json:"offset_opt"` + LimitOpt int32 `db:"limit_opt" json:"limit_opt"` +} + +func (q *sqlQuerier) GetTemplateVersionsByTemplateID(ctx context.Context, arg GetTemplateVersionsByTemplateIDParams) ([]TemplateVersion, error) { + rows, err := q.db.QueryContext(ctx, getTemplateVersionsByTemplateID, + arg.TemplateID, + arg.AfterID, + arg.OffsetOpt, + arg.LimitOpt, + ) if err != nil { return nil, err } diff --git a/coderd/database/queries/templateversions.sql b/coderd/database/queries/templateversions.sql index dccdc0a54bc28..ad98952ebd535 100644 --- a/coderd/database/queries/templateversions.sql +++ b/coderd/database/queries/templateversions.sql @@ -4,7 +4,36 @@ SELECT FROM template_versions WHERE - template_id = $1 :: uuid; + template_id = @template_id :: uuid + AND CASE + -- This allows using the last element on a page as effectively a cursor. + -- This is an important option for scripts that need to paginate without + -- duplicating or missing data. + WHEN @after_id :: uuid != '00000000-00000000-00000000-00000000' THEN ( + -- The pagination cursor is the last user of the previous page. + -- The query is ordered by the created_at field, so select all + -- users after the cursor. We also want to include any users + -- that share the created_at (super rare). + created_at >= ( + SELECT + created_at + FROM + template_versions + WHERE + id = @after_id + ) + -- Omit the cursor from the final. + AND id != @after_id + ) + ELSE true + END +ORDER BY + -- Deterministic and consistent ordering of all users, even if they share + -- a timestamp. This is to ensure consistent pagination. + (created_at, id) ASC OFFSET @offset_opt +LIMIT + -- A null limit means "no limit", so -1 means return all + NULLIF(@limit_opt :: int, -1); -- name: GetTemplateVersionByJobID :one SELECT diff --git a/coderd/httpapi/httpapi.go b/coderd/httpapi/httpapi.go index 331ec527e4328..ba9800285cba2 100644 --- a/coderd/httpapi/httpapi.go +++ b/coderd/httpapi/httpapi.go @@ -8,9 +8,12 @@ import ( "net/http" "reflect" "regexp" + "strconv" "strings" "github.com/go-playground/validator/v10" + "github.com/google/uuid" + "golang.org/x/xerrors" ) var ( @@ -133,3 +136,42 @@ func WebsocketCloseSprintf(format string, vars ...any) string { return msg } + +type Pagination struct { + AfterID uuid.UUID + Limit int + Offset int +} + +func ParsePagination(r *http.Request) (p Pagination, err error) { + var ( + afterID = uuid.Nil + limit = -1 // Default to no limit and return all results. + offset = 0 + ) + + if s := r.URL.Query().Get("after_id"); s != "" { + afterID, err = uuid.Parse(r.URL.Query().Get("after_id")) + if err != nil { + return p, xerrors.Errorf("after_id must be a valid uuid: %w", err.Error()) + } + } + if s := r.URL.Query().Get("limit"); s != "" { + limit, err = strconv.Atoi(s) + if err != nil { + return p, xerrors.Errorf("limit must be an integer: %w", err.Error()) + } + } + if s := r.URL.Query().Get("offset"); s != "" { + offset, err = strconv.Atoi(s) + if err != nil { + return p, xerrors.Errorf("offset must be an integer: %w", err.Error()) + } + } + + return Pagination{ + AfterID: afterID, + Limit: limit, + Offset: offset, + }, nil +} diff --git a/coderd/templates.go b/coderd/templates.go index fe087c9f1b362..3eabcf16616be 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -75,7 +75,18 @@ func (api *api) deleteTemplate(rw http.ResponseWriter, r *http.Request) { func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Request) { template := httpmw.TemplateParam(r) - versions, err := api.Database.GetTemplateVersionsByTemplateID(r.Context(), template.ID) + paginationParams, err := httpapi.ParsePagination(r) + if err != nil { + httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) + return + } + + versions, err := api.Database.GetTemplateVersionsByTemplateID(r.Context(), database.GetTemplateVersionsByTemplateIDParams{ + TemplateID: template.ID, + AfterID: paginationParams.AfterID, + LimitOpt: int32(paginationParams.Limit), + OffsetOpt: int32(paginationParams.Offset), + }) if errors.Is(err, sql.ErrNoRows) { err = nil } diff --git a/coderd/templates_test.go b/coderd/templates_test.go index d7b0e67a659b2..a804f7008dc90 100644 --- a/coderd/templates_test.go +++ b/coderd/templates_test.go @@ -6,10 +6,13 @@ import ( "testing" "github.com/google/uuid" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/coderd/database" "github.com/coder/coder/codersdk" + "github.com/coder/coder/provisioner/echo" ) func TestTemplate(t *testing.T) { @@ -63,7 +66,9 @@ func TestTemplateVersionsByTemplate(t *testing.T) { user := coderdtest.CreateFirstUser(t, client) version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) - versions, err := client.TemplateVersionsByTemplate(context.Background(), template.ID) + versions, err := client.TemplateVersionsByTemplate(context.Background(), codersdk.TemplateVersionsByTemplateRequest{ + TemplateID: template.ID, + }) require.NoError(t, err) require.Len(t, versions, 1) }) @@ -137,3 +142,95 @@ func TestPatchActiveTemplateVersion(t *testing.T) { require.NoError(t, err) }) } + +// TestPaginatedTemplateVersions creates a list of template versions and paginate. +func TestPaginatedTemplateVersions(t *testing.T) { + t.Parallel() + ctx := context.Background() + + client := coderdtest.New(t, &coderdtest.Options{APIRateLimit: -1}) + // Prepare database. + user := coderdtest.CreateFirstUser(t, client) + coderdtest.NewProvisionerDaemon(t, client) + version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil) + _ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) + + // Populate database with template versions. + var templateVersions []codersdk.TemplateVersion + total := 9 + for i := 0; i < total; i++ { + data, err := echo.Tar(nil) + require.NoError(t, err) + file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data) + require.NoError(t, err) + templateVersion, err := client.CreateTemplateVersion(ctx, user.OrganizationID, codersdk.CreateTemplateVersionRequest{ + TemplateID: template.ID, + StorageSource: file.Hash, + StorageMethod: database.ProvisionerStorageMethodFile, + Provisioner: database.ProvisionerTypeEcho, + }) + require.NoError(t, err) + + _ = coderdtest.AwaitTemplateVersionJob(t, client, templateVersion.ID) + } + + templateVersions, err := client.TemplateVersionsByTemplate(ctx, + codersdk.TemplateVersionsByTemplateRequest{ + TemplateID: template.ID, + }, + ) + require.NoError(t, err) + require.Len(t, templateVersions, 10, "wrong number of template versions created") + + type args struct { + ctx context.Context + pagination codersdk.Pagination + } + tests := []struct { + name string + args args + want []codersdk.TemplateVersion + }{ + { + name: "Single result", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 1}}, + want: templateVersions[:1], + }, + { + name: "Single result, second page", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 1, Offset: 1}}, + want: templateVersions[1:2], + }, + { + name: "Last two results", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 2, Offset: 8}}, + want: templateVersions[8:10], + }, + { + name: "AfterID returns next two results", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 2, AfterID: templateVersions[1].ID}}, + want: templateVersions[2:4], + }, + { + name: "No result after last AfterID", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 2, AfterID: templateVersions[9].ID}}, + want: []codersdk.TemplateVersion{}, + }, + { + name: "No result after last Offset", + args: args{ctx: ctx, pagination: codersdk.Pagination{Limit: 2, Offset: 10}}, + want: []codersdk.TemplateVersion{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := client.TemplateVersionsByTemplate(tt.args.ctx, codersdk.TemplateVersionsByTemplateRequest{ + TemplateID: template.ID, + Pagination: tt.args.pagination, + }) + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/codersdk/pagination.go b/codersdk/pagination.go new file mode 100644 index 0000000000000..b5da71f65156e --- /dev/null +++ b/codersdk/pagination.go @@ -0,0 +1,45 @@ +package codersdk + +import ( + "net/http" + "strconv" + + "github.com/google/uuid" +) + +// Pagination sets pagination options for the endpoints that support it. +type Pagination struct { + // AfterID returns all or up to Limit results after the given + // UUID. This option can be used with or as an alternative to + // Offset for better performance. To use it as an alternative, + // set AfterID to the last UUID returned by the previous + // request. + AfterID uuid.UUID `json:"after_id"` + // Limit sets the maximum number of users to be returned + // in a single page. If the limit is <= 0, there is no limit + // and all users are returned. + Limit int `json:"limit"` + // Offset is used to indicate which page to return. An offset of 0 + // returns the first 'limit' number of users. + // To get the next page, use offset=*. + // Offset is 0 indexed, so the first record sits at offset 0. + Offset int `json:"offset"` +} + +// asRequestOption returns a function that can be used in (*Client).request. +// It modifies the request query parameters. +func (p Pagination) asRequestOption() func(*http.Request) { + return func(r *http.Request) { + q := r.URL.Query() + if p.AfterID != uuid.Nil { + q.Set("after_id", p.AfterID.String()) + } + if p.Limit > 0 { + q.Set("limit", strconv.Itoa(p.Limit)) + } + if p.Offset > 0 { + q.Set("offset", strconv.Itoa(p.Offset)) + } + r.URL.RawQuery = q.Encode() + } +} diff --git a/codersdk/templates.go b/codersdk/templates.go index 0f8355a47ff6c..d1f3361c85671 100644 --- a/codersdk/templates.go +++ b/codersdk/templates.go @@ -69,9 +69,16 @@ func (c *Client) UpdateActiveTemplateVersion(ctx context.Context, template uuid. return nil } +// TemplateVersionsByTemplateRequest defines the request parameters for +// TemplateVersionsByTemplate. +type TemplateVersionsByTemplateRequest struct { + TemplateID uuid.UUID `json:"template_id" validate:"required"` + Pagination +} + // TemplateVersionsByTemplate lists versions associated with a template. -func (c *Client) TemplateVersionsByTemplate(ctx context.Context, template uuid.UUID) ([]TemplateVersion, error) { - res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/templates/%s/versions", template), nil) +func (c *Client) TemplateVersionsByTemplate(ctx context.Context, req TemplateVersionsByTemplateRequest) ([]TemplateVersion, error) { + res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/templates/%s/versions", req.TemplateID), nil, req.Pagination.asRequestOption()) if err != nil { return nil, err } diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 846211a109a61..cf8a898700de4 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -147,6 +147,13 @@ export interface OrganizationMember { readonly roles: string[] } +// From codersdk/pagination.go:11:6 +export interface Pagination { + readonly after_id: string + readonly limit: number + readonly offset: number +} + // From codersdk/parameters.go:26:6 export interface Parameter { readonly id: string @@ -256,6 +263,12 @@ export interface TemplateVersionParameterSchema { readonly validation_value_type: string } +// From codersdk/templates.go:74:6 +export interface TemplateVersionsByTemplateRequest { + readonly template_id: string + readonly Pagination: Pagination +} + // From codersdk/templates.go:28:6 export interface UpdateActiveTemplateVersion { readonly id: string From 01a52d1aba63bbe9b4a0425c4b6134b33e4939d0 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 10:05:07 +0000 Subject: [PATCH 02/21] Use codersdk.Pagination in users endpoint --- coderd/database/databasefake/databasefake.go | 11 +++-- coderd/users.go | 45 +++----------------- coderd/users_test.go | 22 +++++++--- codersdk/users.go | 36 +++++----------- 4 files changed, 39 insertions(+), 75 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 0690a7f6957b7..fe1d019511986 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -633,24 +633,27 @@ func (q *fakeQuerier) GetTemplateVersionsByTemplateID(_ context.Context, arg dat } // Database orders by created_at - sort.Slice(version, func(i, j int) bool { - if version[i].CreatedAt.Equal(version[j].CreatedAt) { + slices.SortFunc(version, func(a, b database.TemplateVersion) bool { + if a.CreatedAt.Equal(b.CreatedAt) { // Technically the postgres database also orders by uuid. So match // that behavior - return version[i].ID.String() < version[j].ID.String() + return a.ID.String() < b.ID.String() } - return version[i].CreatedAt.Before(version[j].CreatedAt) + return a.CreatedAt.Before(b.CreatedAt) }) if arg.AfterID != uuid.Nil { found := false for i, v := range version { if v.ID == arg.AfterID { + // We want to return all users after index i. version = version[i+1:] found = true break } } + + // If no users after the time, then we return an empty list. if !found { return nil, sql.ErrNoRows } diff --git a/coderd/users.go b/coderd/users.go index 2d984c8ff9a07..2aa75c7446892 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "net/http" - "strconv" "time" "github.com/go-chi/chi/v5" @@ -106,52 +105,20 @@ func (api *api) postFirstUser(rw http.ResponseWriter, r *http.Request) { func (api *api) users(rw http.ResponseWriter, r *http.Request) { var ( - afterArg = r.URL.Query().Get("after_user") - limitArg = r.URL.Query().Get("limit") - offsetArg = r.URL.Query().Get("offset") searchName = r.URL.Query().Get("search") statusFilter = r.URL.Query().Get("status") ) - // createdAfter is a user uuid. - createdAfter := uuid.Nil - if afterArg != "" { - after, err := uuid.Parse(afterArg) - if err != nil { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ - Message: fmt.Sprintf("after_user must be a valid uuid: %s", err.Error()), - }) - return - } - createdAfter = after - } - - // Default to no limit and return all users. - pageLimit := -1 - if limitArg != "" { - limit, err := strconv.Atoi(limitArg) - if err != nil { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ - Message: fmt.Sprintf("limit must be an integer: %s", err.Error()), - }) - return - } - pageLimit = limit - } - - // The default for empty string is 0. - offset, err := strconv.ParseInt(offsetArg, 10, 64) - if offsetArg != "" && err != nil { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ - Message: fmt.Sprintf("offset must be an integer: %s", err.Error()), - }) + paginationParams, err := httpapi.ParsePagination(r) + if err != nil { + httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) return } users, err := api.Database.GetUsers(r.Context(), database.GetUsersParams{ - AfterUser: createdAfter, - OffsetOpt: int32(offset), - LimitOpt: int32(pageLimit), + AfterUser: paginationParams.AfterID, + OffsetOpt: int32(paginationParams.Offset), + LimitOpt: int32(paginationParams.Limit), Search: searchName, Status: statusFilter, }) diff --git a/coderd/users_test.go b/coderd/users_test.go index f075e670c4052..56d50cb0b4925 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -795,7 +795,9 @@ func assertPagination(ctx context.Context, t *testing.T, client *codersdk.Client // Check the first page page, err := client.Users(ctx, opt(codersdk.UsersRequest{ - Limit: limit, + Pagination: codersdk.Pagination{ + Limit: limit, + }, })) require.NoError(t, err, "first page") require.Equalf(t, page, allUsers[:limit], "first page, limit=%d", limit) @@ -811,15 +813,19 @@ func assertPagination(ctx context.Context, t *testing.T, client *codersdk.Client // This is using a cursor, and only works if all users created_at // is unique. page, err = client.Users(ctx, opt(codersdk.UsersRequest{ - Limit: limit, - AfterUser: afterCursor, + Pagination: codersdk.Pagination{ + Limit: limit, + AfterID: afterCursor, + }, })) require.NoError(t, err, "next cursor page") // Also check page by offset offsetPage, err := client.Users(ctx, opt(codersdk.UsersRequest{ - Limit: limit, - Offset: count, + Pagination: codersdk.Pagination{ + Limit: limit, + Offset: count, + }, })) require.NoError(t, err, "next offset page") @@ -834,8 +840,10 @@ func assertPagination(ctx context.Context, t *testing.T, client *codersdk.Client // Also check the before prevPage, err := client.Users(ctx, opt(codersdk.UsersRequest{ - Offset: count - limit, - Limit: limit, + Pagination: codersdk.Pagination{ + Offset: count - limit, + Limit: limit, + }, })) require.NoError(t, err, "prev page") require.Equal(t, allUsers[count-limit:count], prevPage, "prev users") diff --git a/codersdk/users.go b/codersdk/users.go index 1747a300cf947..4388b0cfd4957 100644 --- a/codersdk/users.go +++ b/codersdk/users.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "net/http" - "strconv" "time" "github.com/google/uuid" @@ -22,19 +21,10 @@ const ( ) type UsersRequest struct { - AfterUser uuid.UUID `json:"after_user"` - Search string `json:"search"` - // Limit sets the maximum number of users to be returned - // in a single page. If the limit is <= 0, there is no limit - // and all users are returned. - Limit int `json:"limit"` - // Offset is used to indicate which page to return. An offset of 0 - // returns the first 'limit' number of users. - // To get the next page, use offset=*. - // Offset is 0 indexed, so the first record sits at offset 0. - Offset int `json:"offset"` + Search string `json:"search"` // Filter users by status Status string `json:"status"` + Pagination } // User represents a user in Coder. @@ -317,19 +307,15 @@ func (c *Client) userByIdentifier(ctx context.Context, ident string) (User, erro // Users returns all users according to the request parameters. If no parameters are set, // the default behavior is to return all users in a single page. func (c *Client) Users(ctx context.Context, req UsersRequest) ([]User, error) { - res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/users"), nil, func(r *http.Request) { - q := r.URL.Query() - if req.AfterUser != uuid.Nil { - q.Set("after_user", req.AfterUser.String()) - } - if req.Limit > 0 { - q.Set("limit", strconv.Itoa(req.Limit)) - } - q.Set("offset", strconv.Itoa(req.Offset)) - q.Set("search", req.Search) - q.Set("status", req.Status) - r.URL.RawQuery = q.Encode() - }) + res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/users"), nil, + req.Pagination.asRequestOption(), + func(r *http.Request) { + q := r.URL.Query() + q.Set("search", req.Search) + q.Set("status", req.Status) + r.URL.RawQuery = q.Encode() + }, + ) if err != nil { return []User{}, err } From d1478a93fd41d032a59808bbdd8e1e7404b7cc48 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 10:11:24 +0000 Subject: [PATCH 03/21] Avoid user sort side-effect in databasefake --- coderd/database/databasefake/databasefake.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index fe1d019511986..bbda6d59e3ecd 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -172,7 +172,10 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams q.mutex.RLock() defer q.mutex.RUnlock() - users := q.users + // Avoid side-effect of sorting. + users := make([]database.User, len(q.users)) + copy(users, q.users) + // Database orders by created_at sort.Slice(users, func(i, j int) bool { if users[i].CreatedAt.Equal(users[j].CreatedAt) { @@ -239,10 +242,7 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams users = users[:params.LimitOpt] } - tmp := make([]database.User, len(users)) - copy(tmp, users) - - return tmp, nil + return users, nil } func (q *fakeQuerier) GetAllUserRoles(_ context.Context, userID uuid.UUID) (database.GetAllUserRolesRow, error) { From 7a36610db45fe74dbde8b813dcee142df21c577a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 11:25:30 +0000 Subject: [PATCH 04/21] Return sql.ErrNoRows for GetUsers in databasefake --- coderd/database/databasefake/databasefake.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index bbda6d59e3ecd..7ad1d7b1890c3 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -202,7 +202,7 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams // If no users after the time, then we return an empty list. if !found { - return []database.User{}, nil + return nil, sql.ErrNoRows } } @@ -230,7 +230,7 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams if params.OffsetOpt > 0 { if int(params.OffsetOpt) > len(users)-1 { - return []database.User{}, nil + return nil, sql.ErrNoRows } users = users[params.OffsetOpt:] } From 61c60c6a834593bb06181e87462809b73a6bbb04 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 11:30:06 +0000 Subject: [PATCH 05/21] Sync codepaths in databasefake --- coderd/database/databasefake/databasefake.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 7ad1d7b1890c3..e04f076d56769 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -177,23 +177,20 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams copy(users, q.users) // Database orders by created_at - sort.Slice(users, func(i, j int) bool { - if users[i].CreatedAt.Equal(users[j].CreatedAt) { + slices.SortFunc(users, func(a, b database.User) bool { + if a.CreatedAt.Equal(b.CreatedAt) { // Technically the postgres database also orders by uuid. So match // that behavior - return users[i].ID.String() < users[j].ID.String() + return a.ID.String() < b.ID.String() } - return users[i].CreatedAt.Before(users[j].CreatedAt) + return a.CreatedAt.Before(b.CreatedAt) }) if params.AfterUser != uuid.Nil { found := false - for i := range users { - if users[i].ID == params.AfterUser { + for i, v := range users { + if v.ID == params.AfterUser { // We want to return all users after index i. - if i+1 >= len(users) { - return []database.User{}, nil - } users = users[i+1:] found = true break From 62fa273f8b56348a446b0258dc9d10e80763464a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 11:31:06 +0000 Subject: [PATCH 06/21] Fix test with better handling of sql.ErrNoRows in coderd --- coderd/templates.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/coderd/templates.go b/coderd/templates.go index 3eabcf16616be..72f2984fa2ce8 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -81,6 +81,7 @@ func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque return } + apiVersion := []codersdk.TemplateVersion{} versions, err := api.Database.GetTemplateVersionsByTemplateID(r.Context(), database.GetTemplateVersionsByTemplateIDParams{ TemplateID: template.ID, AfterID: paginationParams.AfterID, @@ -88,7 +89,8 @@ func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque OffsetOpt: int32(paginationParams.Offset), }) if errors.Is(err, sql.ErrNoRows) { - err = nil + httpapi.Write(rw, http.StatusOK, apiVersion) + return } if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ @@ -112,7 +114,6 @@ func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque jobByID[job.ID.String()] = job } - apiVersion := make([]codersdk.TemplateVersion, 0) for _, version := range versions { job, exists := jobByID[version.JobID.String()] if !exists { From 878d6331eff28faf69395dcc5a59151ce09224a8 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 11:31:18 +0000 Subject: [PATCH 07/21] Remove an unused require.NoError --- coderd/users_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/coderd/users_test.go b/coderd/users_test.go index 56d50cb0b4925..99d1849ca6cf8 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -722,8 +722,6 @@ func TestPaginatedUsers(t *testing.T) { allUsers = append(allUsers, me) specialUsers := make([]codersdk.User, 0) - require.NoError(t, err) - // When 100 users exist total := 100 // Create users From 5c840fd4994e307a7649af1b625e229b54b37825 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 11:58:11 +0000 Subject: [PATCH 08/21] Return empty list for sql.ErrNoRows in coderd users endpoint --- coderd/users.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coderd/users.go b/coderd/users.go index 2aa75c7446892..5b6832dc14771 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -122,6 +122,10 @@ func (api *api) users(rw http.ResponseWriter, r *http.Request) { Search: searchName, Status: statusFilter, }) + if errors.Is(err, sql.ErrNoRows) { + httpapi.Write(rw, http.StatusOK, []codersdk.User{}) + return + } if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ Message: err.Error(), From 07e590ca70ec56f60948b5f758283e49357419e1 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 12:25:21 +0000 Subject: [PATCH 09/21] Add t.Parallel() to sub tests --- coderd/templates_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/templates_test.go b/coderd/templates_test.go index a804f7008dc90..4ce449a7ba96b 100644 --- a/coderd/templates_test.go +++ b/coderd/templates_test.go @@ -225,6 +225,7 @@ func TestPaginatedTemplateVersions(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + t.Parallel() got, err := client.TemplateVersionsByTemplate(tt.args.ctx, codersdk.TemplateVersionsByTemplateRequest{ TemplateID: template.ID, Pagination: tt.args.pagination, From be8ba6c257283a060e414d8053f7df03f300ab51 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 12:26:15 +0000 Subject: [PATCH 10/21] Reinit tt due to parallel test --- coderd/templates_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/coderd/templates_test.go b/coderd/templates_test.go index 4ce449a7ba96b..777e863c1c748 100644 --- a/coderd/templates_test.go +++ b/coderd/templates_test.go @@ -224,6 +224,7 @@ func TestPaginatedTemplateVersions(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() got, err := client.TemplateVersionsByTemplate(tt.args.ctx, codersdk.TemplateVersionsByTemplateRequest{ From 5ab9ad58cb9dc667bfbfcadd5500eb31110c0e0b Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 12:28:41 +0000 Subject: [PATCH 11/21] Remove unused variable --- coderd/templates_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/coderd/templates_test.go b/coderd/templates_test.go index 777e863c1c748..f0082408f3a23 100644 --- a/coderd/templates_test.go +++ b/coderd/templates_test.go @@ -157,7 +157,6 @@ func TestPaginatedTemplateVersions(t *testing.T) { template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) // Populate database with template versions. - var templateVersions []codersdk.TemplateVersion total := 9 for i := 0; i < total; i++ { data, err := echo.Tar(nil) From e2a37414063a5c5bfd962058677ad9b9048e991d Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 14:59:24 +0000 Subject: [PATCH 12/21] Fix copy pasta in query comments --- coderd/database/queries.sql.go | 6 +++--- coderd/database/queries/templateversions.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0dbcbed868628..b73f6d65b16d1 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1913,9 +1913,9 @@ WHERE -- This is an important option for scripts that need to paginate without -- duplicating or missing data. WHEN $2 :: uuid != '00000000-00000000-00000000-00000000' THEN ( - -- The pagination cursor is the last user of the previous page. + -- The pagination cursor is the last ID of the previous page. -- The query is ordered by the created_at field, so select all - -- users after the cursor. We also want to include any users + -- rows after the cursor. We also want to include any rows -- that share the created_at (super rare). created_at >= ( SELECT @@ -1931,7 +1931,7 @@ WHERE ELSE true END ORDER BY - -- Deterministic and consistent ordering of all users, even if they share + -- Deterministic and consistent ordering of all rows, even if they share -- a timestamp. This is to ensure consistent pagination. (created_at, id) ASC OFFSET $3 LIMIT diff --git a/coderd/database/queries/templateversions.sql b/coderd/database/queries/templateversions.sql index ad98952ebd535..78fca24a5bf3e 100644 --- a/coderd/database/queries/templateversions.sql +++ b/coderd/database/queries/templateversions.sql @@ -10,9 +10,9 @@ WHERE -- This is an important option for scripts that need to paginate without -- duplicating or missing data. WHEN @after_id :: uuid != '00000000-00000000-00000000-00000000' THEN ( - -- The pagination cursor is the last user of the previous page. + -- The pagination cursor is the last ID of the previous page. -- The query is ordered by the created_at field, so select all - -- users after the cursor. We also want to include any users + -- rows after the cursor. We also want to include any rows -- that share the created_at (super rare). created_at >= ( SELECT @@ -28,7 +28,7 @@ WHERE ELSE true END ORDER BY - -- Deterministic and consistent ordering of all users, even if they share + -- Deterministic and consistent ordering of all rows, even if they share -- a timestamp. This is to ensure consistent pagination. (created_at, id) ASC OFFSET @offset_opt LIMIT From 61410a52d99c30794f94be8eef63264700149a03 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 15:19:50 +0000 Subject: [PATCH 13/21] Move ParsePagination from httpapi to coderd and unexport, return codersdk sturct --- coderd/httpapi/httpapi.go | 42 ------------------------------------- coderd/pagination.go | 44 +++++++++++++++++++++++++++++++++++++++ coderd/templates.go | 2 +- coderd/users.go | 2 +- 4 files changed, 46 insertions(+), 44 deletions(-) create mode 100644 coderd/pagination.go diff --git a/coderd/httpapi/httpapi.go b/coderd/httpapi/httpapi.go index ba9800285cba2..331ec527e4328 100644 --- a/coderd/httpapi/httpapi.go +++ b/coderd/httpapi/httpapi.go @@ -8,12 +8,9 @@ import ( "net/http" "reflect" "regexp" - "strconv" "strings" "github.com/go-playground/validator/v10" - "github.com/google/uuid" - "golang.org/x/xerrors" ) var ( @@ -136,42 +133,3 @@ func WebsocketCloseSprintf(format string, vars ...any) string { return msg } - -type Pagination struct { - AfterID uuid.UUID - Limit int - Offset int -} - -func ParsePagination(r *http.Request) (p Pagination, err error) { - var ( - afterID = uuid.Nil - limit = -1 // Default to no limit and return all results. - offset = 0 - ) - - if s := r.URL.Query().Get("after_id"); s != "" { - afterID, err = uuid.Parse(r.URL.Query().Get("after_id")) - if err != nil { - return p, xerrors.Errorf("after_id must be a valid uuid: %w", err.Error()) - } - } - if s := r.URL.Query().Get("limit"); s != "" { - limit, err = strconv.Atoi(s) - if err != nil { - return p, xerrors.Errorf("limit must be an integer: %w", err.Error()) - } - } - if s := r.URL.Query().Get("offset"); s != "" { - offset, err = strconv.Atoi(s) - if err != nil { - return p, xerrors.Errorf("offset must be an integer: %w", err.Error()) - } - } - - return Pagination{ - AfterID: afterID, - Limit: limit, - Offset: offset, - }, nil -} diff --git a/coderd/pagination.go b/coderd/pagination.go new file mode 100644 index 0000000000000..8ad62137e3de5 --- /dev/null +++ b/coderd/pagination.go @@ -0,0 +1,44 @@ +package coderd + +import ( + "net/http" + "strconv" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "github.com/coder/coder/codersdk" +) + +func parsePagination(r *http.Request) (p codersdk.Pagination, err error) { + var ( + afterID = uuid.Nil + limit = -1 // Default to no limit and return all results. + offset = 0 + ) + + if s := r.URL.Query().Get("after_id"); s != "" { + afterID, err = uuid.Parse(r.URL.Query().Get("after_id")) + if err != nil { + return p, xerrors.Errorf("after_id must be a valid uuid: %w", err.Error()) + } + } + if s := r.URL.Query().Get("limit"); s != "" { + limit, err = strconv.Atoi(s) + if err != nil { + return p, xerrors.Errorf("limit must be an integer: %w", err.Error()) + } + } + if s := r.URL.Query().Get("offset"); s != "" { + offset, err = strconv.Atoi(s) + if err != nil { + return p, xerrors.Errorf("offset must be an integer: %w", err.Error()) + } + } + + return codersdk.Pagination{ + AfterID: afterID, + Limit: limit, + Offset: offset, + }, nil +} diff --git a/coderd/templates.go b/coderd/templates.go index 72f2984fa2ce8..3bbc5fcf8b78b 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -75,7 +75,7 @@ func (api *api) deleteTemplate(rw http.ResponseWriter, r *http.Request) { func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Request) { template := httpmw.TemplateParam(r) - paginationParams, err := httpapi.ParsePagination(r) + paginationParams, err := parsePagination(r) if err != nil { httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) return diff --git a/coderd/users.go b/coderd/users.go index 5b6832dc14771..87e95957f27f4 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -109,7 +109,7 @@ func (api *api) users(rw http.ResponseWriter, r *http.Request) { statusFilter = r.URL.Query().Get("status") ) - paginationParams, err := httpapi.ParsePagination(r) + paginationParams, err := parsePagination(r) if err != nil { httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) return From 8a67746c152f673ce6cc5546b89a75eaa14a3ad5 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 15:30:56 +0000 Subject: [PATCH 14/21] codersdk: Create requestOption type --- codersdk/client.go | 4 +++- codersdk/pagination.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/codersdk/client.go b/codersdk/client.go index d13927513ae38..1654c141e6827 100644 --- a/codersdk/client.go +++ b/codersdk/client.go @@ -33,9 +33,11 @@ type Client struct { URL *url.URL } +type requestOption func(*http.Request) + // request performs an HTTP request with the body provided. // The caller is responsible for closing the response body. -func (c *Client) request(ctx context.Context, method, path string, body interface{}, opts ...func(r *http.Request)) (*http.Response, error) { +func (c *Client) request(ctx context.Context, method, path string, body interface{}, opts ...requestOption) (*http.Response, error) { serverURL, err := c.URL.Parse(path) if err != nil { return nil, xerrors.Errorf("parse url: %w", err) diff --git a/codersdk/pagination.go b/codersdk/pagination.go index b5da71f65156e..81d15f1083023 100644 --- a/codersdk/pagination.go +++ b/codersdk/pagination.go @@ -28,7 +28,7 @@ type Pagination struct { // asRequestOption returns a function that can be used in (*Client).request. // It modifies the request query parameters. -func (p Pagination) asRequestOption() func(*http.Request) { +func (p Pagination) asRequestOption() requestOption { return func(r *http.Request) { q := r.URL.Query() if p.AfterID != uuid.Nil { From 55028d23f33c2355dcbdaf10abd99410588a1b1a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 15:31:39 +0000 Subject: [PATCH 15/21] codersdk: Add test for Pagination.asRequestOption --- codersdk/pagination_test.go | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 codersdk/pagination_test.go diff --git a/codersdk/pagination_test.go b/codersdk/pagination_test.go new file mode 100644 index 0000000000000..cf1d45870c741 --- /dev/null +++ b/codersdk/pagination_test.go @@ -0,0 +1,54 @@ +package codersdk + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestPagination_asRequestOption(t *testing.T) { + uuid1 := uuid.New() + type fields struct { + AfterID uuid.UUID + Limit int + Offset int + } + tests := []struct { + name string + fields fields + want url.Values + }{ + { + name: "Test AfterID is set", + fields: fields{AfterID: uuid1}, + want: url.Values{"after_id": []string{uuid1.String()}}, + }, + { + name: "Test Limit is set", + fields: fields{Limit: 10}, + want: url.Values{"limit": []string{"10"}}, + }, + { + name: "Test Offset is set", + fields: fields{Offset: 10}, + want: url.Values{"offset": []string{"10"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := Pagination{ + AfterID: tt.fields.AfterID, + Limit: tt.fields.Limit, + Offset: tt.fields.Offset, + } + req := httptest.NewRequest(http.MethodGet, "/", nil) + p.asRequestOption()(req) + got := req.URL.Query() + assert.Equal(t, tt.want, got) + }) + } +} From ea2371e9f94dbd1c32b32c02c5e0537071e795d8 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 15:56:34 +0000 Subject: [PATCH 16/21] coderd: Handle http response errors in parsePagination --- coderd/pagination.go | 25 +++++++++++++++++++------ coderd/templates.go | 5 ++--- coderd/users.go | 5 ++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/coderd/pagination.go b/coderd/pagination.go index 8ad62137e3de5..2fbc2a0df6c70 100644 --- a/coderd/pagination.go +++ b/coderd/pagination.go @@ -1,38 +1,51 @@ package coderd import ( + "fmt" "net/http" "strconv" "github.com/google/uuid" - "golang.org/x/xerrors" + "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/codersdk" ) -func parsePagination(r *http.Request) (p codersdk.Pagination, err error) { +// parsePagination extracts pagination query params from the http request. +// If an error is encountered, the error is written to w and ok is set to false. +func parsePagination(w http.ResponseWriter, r *http.Request) (p codersdk.Pagination, ok bool) { var ( afterID = uuid.Nil limit = -1 // Default to no limit and return all results. offset = 0 ) + var err error if s := r.URL.Query().Get("after_id"); s != "" { afterID, err = uuid.Parse(r.URL.Query().Get("after_id")) if err != nil { - return p, xerrors.Errorf("after_id must be a valid uuid: %w", err.Error()) + httpapi.Write(w, http.StatusBadRequest, httpapi.Response{ + Message: fmt.Sprintf("after_id must be a valid uuid: %s", err.Error()), + }) + return p, false } } if s := r.URL.Query().Get("limit"); s != "" { limit, err = strconv.Atoi(s) if err != nil { - return p, xerrors.Errorf("limit must be an integer: %w", err.Error()) + httpapi.Write(w, http.StatusBadRequest, httpapi.Response{ + Message: fmt.Sprintf("limit must be an integer: %s", err.Error()), + }) + return p, false } } if s := r.URL.Query().Get("offset"); s != "" { offset, err = strconv.Atoi(s) if err != nil { - return p, xerrors.Errorf("offset must be an integer: %w", err.Error()) + httpapi.Write(w, http.StatusBadRequest, httpapi.Response{ + Message: fmt.Sprintf("offset must be an integer: %s", err.Error()), + }) + return p, false } } @@ -40,5 +53,5 @@ func parsePagination(r *http.Request) (p codersdk.Pagination, err error) { AfterID: afterID, Limit: limit, Offset: offset, - }, nil + }, true } diff --git a/coderd/templates.go b/coderd/templates.go index 3bbc5fcf8b78b..153c8aba9c122 100644 --- a/coderd/templates.go +++ b/coderd/templates.go @@ -75,9 +75,8 @@ func (api *api) deleteTemplate(rw http.ResponseWriter, r *http.Request) { func (api *api) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Request) { template := httpmw.TemplateParam(r) - paginationParams, err := parsePagination(r) - if err != nil { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) + paginationParams, ok := parsePagination(rw, r) + if !ok { return } diff --git a/coderd/users.go b/coderd/users.go index 87e95957f27f4..db7b8cb27e50e 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -109,9 +109,8 @@ func (api *api) users(rw http.ResponseWriter, r *http.Request) { statusFilter = r.URL.Query().Get("status") ) - paginationParams, err := parsePagination(r) - if err != nil { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{Message: fmt.Sprintf("parse pagination request: %s", err.Error())}) + paginationParams, ok := parsePagination(rw, r) + if !ok { return } From 66af4ece0bc625a3d5b9f7be256bfa524450eea2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 16:21:59 +0000 Subject: [PATCH 17/21] codersdk: Use parallel test --- codersdk/pagination_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/codersdk/pagination_test.go b/codersdk/pagination_test.go index cf1d45870c741..53a3fcaebceb4 100644 --- a/codersdk/pagination_test.go +++ b/codersdk/pagination_test.go @@ -1,3 +1,4 @@ +//nolint:testpackage package codersdk import ( @@ -11,6 +12,8 @@ import ( ) func TestPagination_asRequestOption(t *testing.T) { + t.Parallel() + uuid1 := uuid.New() type fields struct { AfterID uuid.UUID @@ -39,7 +42,10 @@ func TestPagination_asRequestOption(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() + p := Pagination{ AfterID: tt.fields.AfterID, Limit: tt.fields.Limit, From 067f912a8fab01f08f48a9a99218b2e58470c9c1 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 18:24:28 +0000 Subject: [PATCH 18/21] Fix created_at edge case for pagination cursor in queries --- coderd/database/databasefake/databasefake.go | 4 +- coderd/database/queries.sql.go | 54 +++++++++----------- coderd/database/queries/templateversions.sql | 23 ++++----- coderd/database/queries/users.sql | 29 +++++------ coderd/users.go | 2 +- 5 files changed, 50 insertions(+), 62 deletions(-) diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index e04f076d56769..f1ec50bbbbe2f 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -186,10 +186,10 @@ func (q *fakeQuerier) GetUsers(_ context.Context, params database.GetUsersParams return a.CreatedAt.Before(b.CreatedAt) }) - if params.AfterUser != uuid.Nil { + if params.AfterID != uuid.Nil { found := false for i, v := range users { - if v.ID == params.AfterUser { + if v.ID == params.AfterID { // We want to return all users after index i. users = users[i+1:] found = true diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b73f6d65b16d1..229cb8e80fcba 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -1915,20 +1915,17 @@ WHERE WHEN $2 :: uuid != '00000000-00000000-00000000-00000000' THEN ( -- The pagination cursor is the last ID of the previous page. -- The query is ordered by the created_at field, so select all - -- rows after the cursor. We also want to include any rows - -- that share the created_at (super rare). - created_at >= ( - SELECT - created_at - FROM - template_versions - WHERE - id = $2 - ) - -- Omit the cursor from the final. - AND id != $2 + -- rows after the cursor. + (created_at, id) > ( + SELECT + created_at, id + FROM + template_versions + WHERE + id = $2 ) - ELSE true + ) + ELSE true END ORDER BY -- Deterministic and consistent ordering of all rows, even if they share @@ -2166,22 +2163,19 @@ WHERE -- This is an important option for scripts that need to paginate without -- duplicating or missing data. WHEN $1 :: uuid != '00000000-00000000-00000000-00000000' THEN ( - -- The pagination cursor is the last user of the previous page. - -- The query is ordered by the created_at field, so select all - -- users after the cursor. We also want to include any users - -- that share the created_at (super rare). - created_at >= ( - SELECT - created_at - FROM - users - WHERE - id = $1 - ) - -- Omit the cursor from the final. - AND id != $1 + -- The pagination cursor is the last ID of the previous page. + -- The query is ordered by the created_at field, so select all + -- rows after the cursor. + (created_at, id) > ( + SELECT + created_at, id + FROM + template_versions + WHERE + id = $1 ) - ELSE true + ) + ELSE true END -- Start filters -- Filter by name, email or username @@ -2212,7 +2206,7 @@ LIMIT ` type GetUsersParams struct { - AfterUser uuid.UUID `db:"after_user" json:"after_user"` + AfterID uuid.UUID `db:"after_id" json:"after_id"` Search string `db:"search" json:"search"` Status string `db:"status" json:"status"` OffsetOpt int32 `db:"offset_opt" json:"offset_opt"` @@ -2221,7 +2215,7 @@ type GetUsersParams struct { func (q *sqlQuerier) GetUsers(ctx context.Context, arg GetUsersParams) ([]User, error) { rows, err := q.db.QueryContext(ctx, getUsers, - arg.AfterUser, + arg.AfterID, arg.Search, arg.Status, arg.OffsetOpt, diff --git a/coderd/database/queries/templateversions.sql b/coderd/database/queries/templateversions.sql index 78fca24a5bf3e..b6a3550d5f2bd 100644 --- a/coderd/database/queries/templateversions.sql +++ b/coderd/database/queries/templateversions.sql @@ -12,20 +12,17 @@ WHERE WHEN @after_id :: uuid != '00000000-00000000-00000000-00000000' THEN ( -- The pagination cursor is the last ID of the previous page. -- The query is ordered by the created_at field, so select all - -- rows after the cursor. We also want to include any rows - -- that share the created_at (super rare). - created_at >= ( - SELECT - created_at - FROM - template_versions - WHERE - id = @after_id - ) - -- Omit the cursor from the final. - AND id != @after_id + -- rows after the cursor. + (created_at, id) > ( + SELECT + created_at, id + FROM + template_versions + WHERE + id = @after_id ) - ELSE true + ) + ELSE true END ORDER BY -- Deterministic and consistent ordering of all rows, even if they share diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index d0c3d4c3e7b66..7d7138b11c0c0 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -77,23 +77,20 @@ WHERE -- This allows using the last element on a page as effectively a cursor. -- This is an important option for scripts that need to paginate without -- duplicating or missing data. - WHEN @after_user :: uuid != '00000000-00000000-00000000-00000000' THEN ( - -- The pagination cursor is the last user of the previous page. - -- The query is ordered by the created_at field, so select all - -- users after the cursor. We also want to include any users - -- that share the created_at (super rare). - created_at >= ( - SELECT - created_at - FROM - users - WHERE - id = @after_user - ) - -- Omit the cursor from the final. - AND id != @after_user + WHEN @after_id :: uuid != '00000000-00000000-00000000-00000000' THEN ( + -- The pagination cursor is the last ID of the previous page. + -- The query is ordered by the created_at field, so select all + -- rows after the cursor. + (created_at, id) > ( + SELECT + created_at, id + FROM + template_versions + WHERE + id = @after_id ) - ELSE true + ) + ELSE true END -- Start filters -- Filter by name, email or username diff --git a/coderd/users.go b/coderd/users.go index db7b8cb27e50e..c85a23f3f72bd 100644 --- a/coderd/users.go +++ b/coderd/users.go @@ -115,7 +115,7 @@ func (api *api) users(rw http.ResponseWriter, r *http.Request) { } users, err := api.Database.GetUsers(r.Context(), database.GetUsersParams{ - AfterUser: paginationParams.AfterID, + AfterID: paginationParams.AfterID, OffsetOpt: int32(paginationParams.Offset), LimitOpt: int32(paginationParams.Limit), Search: searchName, From b8be7a8622740e226e5867812f95cd841bc07c88 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Thu, 5 May 2022 19:08:21 +0000 Subject: [PATCH 19/21] Fix copy paste issue --- coderd/database/queries.sql.go | 2 +- coderd/database/queries/users.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 229cb8e80fcba..362bae614c15e 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2170,7 +2170,7 @@ WHERE SELECT created_at, id FROM - template_versions + users WHERE id = $1 ) diff --git a/coderd/database/queries/users.sql b/coderd/database/queries/users.sql index 7d7138b11c0c0..c3d72b36378ab 100644 --- a/coderd/database/queries/users.sql +++ b/coderd/database/queries/users.sql @@ -85,7 +85,7 @@ WHERE SELECT created_at, id FROM - template_versions + users WHERE id = @after_id ) From 13fc406b9edd861f4ea7c089979d0eac621fbf55 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 10 May 2022 07:30:19 +0000 Subject: [PATCH 20/21] Run make gen --- site/src/api/typesGenerated.ts | 36 ++++++++++++++++------------------ 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index cf8a898700de4..d689bf1b49080 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -12,7 +12,7 @@ export interface AgentGitSSHKey { readonly private_key: string } -// From codersdk/users.go:110:6 +// From codersdk/users.go:100:6 export interface AuthMethods { readonly password: boolean readonly github: boolean @@ -30,7 +30,7 @@ export interface BuildInfoResponse { readonly version: string } -// From codersdk/users.go:51:6 +// From codersdk/users.go:41:6 export interface CreateFirstUserRequest { readonly email: string readonly username: string @@ -38,13 +38,13 @@ export interface CreateFirstUserRequest { readonly organization: string } -// From codersdk/users.go:59:6 +// From codersdk/users.go:49:6 export interface CreateFirstUserResponse { readonly user_id: string readonly organization_id: string } -// From codersdk/users.go:105:6 +// From codersdk/users.go:95:6 export interface CreateOrganizationRequest { readonly name: string } @@ -77,7 +77,7 @@ export interface CreateTemplateVersionRequest { readonly parameter_values: CreateParameterRequest[] } -// From codersdk/users.go:64:6 +// From codersdk/users.go:54:6 export interface CreateUserRequest { readonly email: string readonly username: string @@ -101,7 +101,7 @@ export interface CreateWorkspaceRequest { readonly parameter_values: CreateParameterRequest[] } -// From codersdk/users.go:101:6 +// From codersdk/users.go:91:6 export interface GenerateAPIKeyResponse { readonly key: string } @@ -119,13 +119,13 @@ export interface GoogleInstanceIdentityToken { readonly json_web_token: string } -// From codersdk/users.go:90:6 +// From codersdk/users.go:80:6 export interface LoginWithPasswordRequest { readonly email: string readonly password: string } -// From codersdk/users.go:96:6 +// From codersdk/users.go:86:6 export interface LoginWithPasswordResponse { readonly session_token: string } @@ -202,7 +202,7 @@ export interface ProvisionerJobLog { readonly output: string } -// From codersdk/roles.go:13:6 +// From codersdk/roles.go:12:6 export interface Role { readonly name: string readonly display_name: string @@ -274,17 +274,17 @@ export interface UpdateActiveTemplateVersion { readonly id: string } -// From codersdk/users.go:80:6 +// From codersdk/users.go:70:6 export interface UpdateRoles { readonly roles: string[] } -// From codersdk/users.go:76:6 +// From codersdk/users.go:66:6 export interface UpdateUserPasswordRequest { readonly password: string } -// From codersdk/users.go:71:6 +// From codersdk/users.go:61:6 export interface UpdateUserProfileRequest { readonly email: string readonly username: string @@ -305,7 +305,7 @@ export interface UploadResponse { readonly hash: string } -// From codersdk/users.go:41:6 +// From codersdk/users.go:31:6 export interface User { readonly id: string readonly email: string @@ -316,19 +316,17 @@ export interface User { readonly roles: Role[] } -// From codersdk/users.go:84:6 +// From codersdk/users.go:74:6 export interface UserRoles { readonly roles: string[] readonly organization_roles: Record } -// From codersdk/users.go:24:6 +// From codersdk/users.go:23:6 export interface UsersRequest { - readonly after_user: string readonly search: string - readonly limit: number - readonly offset: number readonly status: string + readonly Pagination: Pagination } // From codersdk/workspaces.go:18:6 @@ -426,7 +424,7 @@ export type ParameterScope = "organization" | "template" | "user" | "workspace" // From codersdk/provisionerdaemons.go:26:6 export type ProvisionerJobStatus = "canceled" | "canceling" | "failed" | "pending" | "running" | "succeeded" -// From codersdk/users.go:17:6 +// From codersdk/users.go:16:6 export type UserStatus = "active" | "suspended" // From codersdk/workspaceresources.go:15:6 From aab400ce94e9b6e3da3b2f954987d46a80b62eee Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 10 May 2022 10:36:19 +0300 Subject: [PATCH 21/21] feat: Add support for json omitempty and embedded structs in apitypings (#1318) * feat: Add support for json omitempty to apitypings * feat: Express embedded structs via extends in TypeScript * Handle unembedding via json field in apitypings * Add scripts/apitypings/main.go to Makefile --- Makefile | 2 +- codersdk/pagination.go | 6 +++--- scripts/apitypings/main.go | 31 ++++++++++++++++++++++++++++--- site/src/api/typesGenerated.ts | 24 +++++++++++------------- 4 files changed, 43 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index ffc08fe329732..9ea8aa90a13b0 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ site/out/index.html: $(shell find ./site -not -path './site/node_modules/*' -typ # Restores GITKEEP files! git checkout HEAD site/out -site/src/api/typesGenerated.ts: $(shell find codersdk -type f -name '*.go') +site/src/api/typesGenerated.ts: scripts/apitypings/main.go $(shell find codersdk -type f -name '*.go') go run scripts/apitypings/main.go > site/src/api/typesGenerated.ts cd site && yarn run format:types diff --git a/codersdk/pagination.go b/codersdk/pagination.go index 81d15f1083023..a4adee6b6e567 100644 --- a/codersdk/pagination.go +++ b/codersdk/pagination.go @@ -14,16 +14,16 @@ type Pagination struct { // Offset for better performance. To use it as an alternative, // set AfterID to the last UUID returned by the previous // request. - AfterID uuid.UUID `json:"after_id"` + AfterID uuid.UUID `json:"after_id,omitempty"` // Limit sets the maximum number of users to be returned // in a single page. If the limit is <= 0, there is no limit // and all users are returned. - Limit int `json:"limit"` + Limit int `json:"limit,omitempty"` // Offset is used to indicate which page to return. An offset of 0 // returns the first 'limit' number of users. // To get the next page, use offset=*. // Offset is 0 indexed, so the first record sits at offset 0. - Offset int `json:"offset"` + Offset int `json:"offset,omitempty"` } // asRequestOption returns a function that can be used in (*Client).request. diff --git a/scripts/apitypings/main.go b/scripts/apitypings/main.go index 529b7ae0f42f2..0891439e723e3 100644 --- a/scripts/apitypings/main.go +++ b/scripts/apitypings/main.go @@ -237,10 +237,31 @@ func (g *Generator) posLine(obj types.Object) string { func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, error) { var s strings.Builder _, _ = s.WriteString(g.posLine(obj)) + _, _ = s.WriteString(fmt.Sprintf("export interface %s ", obj.Name())) - _, _ = s.WriteString(fmt.Sprintf("export interface %s {\n", obj.Name())) + // Handle named embedded structs in the codersdk package via extension. + var extends []string + extendedFields := make(map[int]bool) + for i := 0; i < st.NumFields(); i++ { + field := st.Field(i) + tag := reflect.StructTag(st.Tag(i)) + // Adding a json struct tag causes the json package to consider + // the field unembedded. + if field.Embedded() && tag.Get("json") == "" && field.Pkg().Name() == "codersdk" { + extendedFields[i] = true + extends = append(extends, field.Name()) + } + } + if len(extends) > 0 { + _, _ = s.WriteString(fmt.Sprintf("extends %s ", strings.Join(extends, ", "))) + } + + _, _ = s.WriteString("{\n") // For each field in the struct, we print 1 line of the typescript interface for i := 0; i < st.NumFields(); i++ { + if extendedFields[i] { + continue + } field := st.Field(i) tag := reflect.StructTag(st.Tag(i)) @@ -251,6 +272,10 @@ func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, err if jsonName == "" { jsonName = field.Name() } + jsonOptional := false + if len(arr) > 1 && arr[1] == "omitempty" { + jsonOptional = true + } var tsType TypescriptType // If a `typescript:"string"` exists, we take this, and do not try to infer. @@ -273,7 +298,7 @@ func (g *Generator) buildStruct(obj types.Object, st *types.Struct) (string, err _, _ = s.WriteRune('\n') } optional := "" - if tsType.Optional { + if jsonOptional || tsType.Optional { optional = "?" } _, _ = s.WriteString(fmt.Sprintf("%sreadonly %s%s: %s\n", indent, jsonName, optional, tsType.ValueType)) @@ -322,7 +347,7 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { return TypescriptType{ ValueType: "any", AboveTypeLine: fmt.Sprintf("%s\n%s", - indentedComment("Embedded struct, please fix by naming it"), + indentedComment("Embedded anonymous struct, please fix by naming it"), indentedComment("eslint-disable-next-line @typescript-eslint/no-explicit-any"), ), }, nil diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index d689bf1b49080..e30c94e8663e6 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -91,7 +91,7 @@ export interface CreateWorkspaceBuildRequest { // This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition") readonly transition: string readonly dry_run: boolean - readonly state: string + readonly state?: string } // From codersdk/organizations.go:52:6 @@ -149,9 +149,9 @@ export interface OrganizationMember { // From codersdk/pagination.go:11:6 export interface Pagination { - readonly after_id: string - readonly limit: number - readonly offset: number + readonly after_id?: string + readonly limit?: number + readonly offset?: number } // From codersdk/parameters.go:26:6 @@ -185,7 +185,7 @@ export interface ProvisionerJob { readonly created_at: string readonly started_at?: string readonly completed_at?: string - readonly error: string + readonly error?: string readonly status: ProvisionerJobStatus readonly worker_id?: string } @@ -264,9 +264,8 @@ export interface TemplateVersionParameterSchema { } // From codersdk/templates.go:74:6 -export interface TemplateVersionsByTemplateRequest { +export interface TemplateVersionsByTemplateRequest extends Pagination { readonly template_id: string - readonly Pagination: Pagination } // From codersdk/templates.go:28:6 @@ -323,10 +322,9 @@ export interface UserRoles { } // From codersdk/users.go:23:6 -export interface UsersRequest { +export interface UsersRequest extends Pagination { readonly search: string readonly status: string - readonly Pagination: Pagination } // From codersdk/workspaces.go:18:6 @@ -355,12 +353,12 @@ export interface WorkspaceAgent { readonly status: WorkspaceAgentStatus readonly name: string readonly resource_id: string - readonly instance_id: string + readonly instance_id?: string readonly architecture: string readonly environment_variables: Record readonly operating_system: string - readonly startup_script: string - readonly directory: string + readonly startup_script?: string + readonly directory?: string } // From codersdk/workspaceagents.go:47:6 @@ -415,7 +413,7 @@ export interface WorkspaceResource { readonly workspace_transition: string readonly type: string readonly name: string - readonly agents: WorkspaceAgent[] + readonly agents?: WorkspaceAgent[] } // From codersdk/parameters.go:16:6