Thanks to visit codestin.com
Credit goes to github.com

Skip to content

WIP: Using joins to reduce DB round trips #6356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 12 commits into from
Closed
5 changes: 2 additions & 3 deletions coderd/autobuild/executor/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,18 +102,17 @@ func (e *Executor) runOnce(t time.Time) Stats {
// NOTE: If a workspace build is created with a given TTL and then the user either
// changes or unsets the TTL, the deadline for the workspace build will not
// have changed. This behavior is as expected per #2229.
workspaceRows, err := e.db.GetWorkspaces(e.ctx, database.GetWorkspacesParams{
workspaces, err := e.db.GetWorkspaces(e.ctx, database.GetWorkspacesParams{
Deleted: false,
})
if err != nil {
e.log.Error(e.ctx, "get workspaces for autostart or autostop", slog.Error(err))
return stats
}
workspaces := database.ConvertWorkspaceRows(workspaceRows)

var eligibleWorkspaceIDs []uuid.UUID
for _, ws := range workspaces {
if isEligibleForAutoStartStop(ws) {
if isEligibleForAutoStartStop(ws.Workspace) {
eligibleWorkspaceIDs = append(eligibleWorkspaceIDs, ws.ID)
}
}
Expand Down
80 changes: 80 additions & 0 deletions coderd/database/bindvars.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
package database

import (
"database/sql/driver"
"fmt"
"reflect"
"regexp"
"strings"

"github.com/google/uuid"
"github.com/lib/pq"

"github.com/jmoiron/sqlx/reflectx"

"github.com/coder/coder/coderd/util/slice"
)

var nameRegex = regexp.MustCompile(`@([a-zA-Z0-9_]+)`)

// dbmapper grabs struct 'db' tags.
var dbmapper = reflectx.NewMapper("db")
var sqlValuer = reflect.TypeOf((*driver.Valuer)(nil)).Elem()

// bindNamed is an implementation that improves on the SQLx implementation. This
// adjusts the query to use "$#" syntax for arguments instead of "@argument". The
// returned args are the values of the struct fields that match the names in the
// correct order and indexing.
//
// 1. SQLx does not reuse arguments, so "@arg, @arg" will result in two arguments
// "$1, $2" instead of "$1, $1".
// 2. SQLx does not handle uuid arrays.
// 3. SQLx only supports ":name" style arguments and breaks "::" type casting.
func bindNamed(query string, arg interface{}) (newQuery string, args []interface{}, err error) {
// We do not need to implement a sql parser to extract and replace the variable names.
// All names follow a simple regex.
names := nameRegex.FindAllString(query, -1)
// Get all unique names
names = slice.Unique(names)

// Replace all names with the correct index
for i, name := range names {
rpl := fmt.Sprintf("$%d", i+1)
query = strings.ReplaceAll(query, name, rpl)
// Remove the "@" prefix to match to the "db" struct tag.
names[i] = strings.TrimPrefix(name, "@")
}

arglist := make([]interface{}, 0, len(names))

// This comes straight from SQLx's implementation to get the values
// of the struct fields.
v := reflect.ValueOf(arg)
for v = reflect.ValueOf(arg); v.Kind() == reflect.Ptr; {
v = v.Elem()
}

err = dbmapper.TraversalsByNameFunc(v.Type(), names, func(i int, t []int) error {
if len(t) == 0 {
return fmt.Errorf("could not find name %s in %#v", names[i], arg)
}

val := reflectx.FieldByIndexesReadOnly(v, t)

// Handle some custom types to make arguments easier to use.
switch val.Interface().(type) {
// Feel free to add more types here as needed.
case []uuid.UUID:
arglist = append(arglist, pq.Array(val.Interface()))
default:
arglist = append(arglist, val.Interface())
}

return nil
})
if err != nil {
return "", nil, err
}

return query, arglist, nil
}
10 changes: 5 additions & 5 deletions coderd/database/dbauthz/querier.go
Original file line number Diff line number Diff line change
Expand Up @@ -1165,12 +1165,12 @@ func (q *querier) UpdateUserRoles(ctx context.Context, arg database.UpdateUserRo
return q.db.UpdateUserRoles(ctx, arg)
}

func (q *querier) GetAuthorizedWorkspaces(ctx context.Context, arg database.GetWorkspacesParams, _ rbac.PreparedAuthorized) ([]database.GetWorkspacesRow, error) {
func (q *querier) GetAuthorizedWorkspaces(ctx context.Context, arg database.GetWorkspacesParams, prep rbac.PreparedAuthorized) ([]database.WorkspaceWithData, error) {
// TODO Delete this function, all GetWorkspaces should be authorized. For now just call GetWorkspaces on the authz querier.
return q.GetWorkspaces(ctx, arg)
return q.db.GetAuthorizedWorkspaces(ctx, arg, prep)
}

func (q *querier) GetWorkspaces(ctx context.Context, arg database.GetWorkspacesParams) ([]database.GetWorkspacesRow, error) {
func (q *querier) GetWorkspaces(ctx context.Context, arg database.GetWorkspacesParams) ([]database.WorkspaceWithData, error) {
prep, err := prepareSQLFilter(ctx, q.auth, rbac.ActionRead, rbac.ResourceWorkspace.Type)
if err != nil {
return nil, xerrors.Errorf("(dev error) prepare sql filter: %w", err)
Expand Down Expand Up @@ -1372,11 +1372,11 @@ func (q *querier) GetWorkspaceByAgentID(ctx context.Context, agentID uuid.UUID)
return fetch(q.log, q.auth, q.db.GetWorkspaceByAgentID)(ctx, agentID)
}

func (q *querier) GetWorkspaceByID(ctx context.Context, id uuid.UUID) (database.Workspace, error) {
func (q *querier) GetWorkspaceByID(ctx context.Context, id uuid.UUID) (database.WorkspaceWithData, error) {
return fetch(q.log, q.auth, q.db.GetWorkspaceByID)(ctx, id)
}

func (q *querier) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg database.GetWorkspaceByOwnerIDAndNameParams) (database.Workspace, error) {
func (q *querier) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg database.GetWorkspaceByOwnerIDAndNameParams) (database.WorkspaceWithData, error) {
return fetch(q.log, q.auth, q.db.GetWorkspaceByOwnerIDAndName)(ctx, arg)
}

Expand Down
132 changes: 68 additions & 64 deletions coderd/database/dbfake/databasefake.go
Original file line number Diff line number Diff line change
Expand Up @@ -630,7 +630,7 @@ func (q *fakeQuerier) GetAuthorizedUserCount(ctx context.Context, params databas

// Call this to match the same function calls as the SQL implementation.
if prepared != nil {
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithoutACL())
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithoutACL(""))
if err != nil {
return -1, err
}
Expand Down Expand Up @@ -895,18 +895,12 @@ func (q *fakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U
}, nil
}

func (q *fakeQuerier) GetWorkspaces(ctx context.Context, arg database.GetWorkspacesParams) ([]database.GetWorkspacesRow, error) {
if err := validateDatabaseType(arg); err != nil {
return nil, err
}

// A nil auth filter means no auth filter.
workspaceRows, err := q.GetAuthorizedWorkspaces(ctx, arg, nil)
return workspaceRows, err
func (q *fakeQuerier) GetWorkspaces(ctx context.Context, arg database.GetWorkspacesParams) ([]database.WorkspaceWithData, error) {
return q.GetAuthorizedWorkspaces(ctx, arg, nil)
}

//nolint:gocyclo
func (q *fakeQuerier) GetAuthorizedWorkspaces(ctx context.Context, arg database.GetWorkspacesParams, prepared rbac.PreparedAuthorized) ([]database.GetWorkspacesRow, error) {
func (q *fakeQuerier) GetAuthorizedWorkspaces(ctx context.Context, arg database.GetWorkspacesParams, prepared rbac.PreparedAuthorized) ([]database.WorkspaceWithData, error) {
if err := validateDatabaseType(arg); err != nil {
return nil, err
}
Expand All @@ -916,7 +910,7 @@ func (q *fakeQuerier) GetAuthorizedWorkspaces(ctx context.Context, arg database.

if prepared != nil {
// Call this to match the same function calls as the SQL implementation.
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithoutACL())
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithoutACL(""))
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -1101,18 +1095,18 @@ func (q *fakeQuerier) GetAuthorizedWorkspaces(ctx context.Context, arg database.

if arg.Offset > 0 {
if int(arg.Offset) > len(workspaces) {
return []database.GetWorkspacesRow{}, nil
return []database.WorkspaceWithData{}, nil
}
workspaces = workspaces[arg.Offset:]
}
if arg.Limit > 0 {
if int(arg.Limit) > len(workspaces) {
return convertToWorkspaceRows(workspaces, int64(beforePageCount)), nil
return q.convertToWorkspaceData(workspaces, int64(beforePageCount)), nil
}
workspaces = workspaces[:arg.Limit]
}

return convertToWorkspaceRows(workspaces, int64(beforePageCount)), nil
return q.convertToWorkspaceData(workspaces, int64(beforePageCount)), nil
}

// mapAgentStatus determines the agent status based on different timestamps like created_at, last_connected_at, disconnected_at, etc.
Expand Down Expand Up @@ -1149,37 +1143,59 @@ func mapAgentStatus(dbAgent database.WorkspaceAgent, agentInactiveDisconnectTime
return status
}

func convertToWorkspaceRows(workspaces []database.Workspace, count int64) []database.GetWorkspacesRow {
rows := make([]database.GetWorkspacesRow, len(workspaces))
func (q fakeQuerier) convertToWorkspaceData(workspaces []database.Workspace, count int64) []database.WorkspaceWithData {
rows := make([]database.WorkspaceWithData, len(workspaces))
for i, w := range workspaces {
rows[i] = database.GetWorkspacesRow{
ID: w.ID,
CreatedAt: w.CreatedAt,
UpdatedAt: w.UpdatedAt,
OwnerID: w.OwnerID,
OrganizationID: w.OrganizationID,
TemplateID: w.TemplateID,
Deleted: w.Deleted,
Name: w.Name,
AutostartSchedule: w.AutostartSchedule,
Ttl: w.Ttl,
LastUsedAt: w.LastUsedAt,
Count: count,
// TODO: (@emyrk) Should these error?
owner, _ := q.GetUserByID(context.Background(), w.OwnerID)
latestBuild, _ := q.GetLatestWorkspaceBuildByWorkspaceID(context.Background(), w.ID)
job, _ := q.GetProvisionerJobByID(context.Background(), latestBuild.JobID)
latestBuildUser, _ := q.GetUserByID(context.Background(), latestBuild.InitiatorID)
tv, _ := q.GetTemplateVersionByID(context.Background(), latestBuild.TemplateVersionID)
tpl, _ := q.GetTemplateByID(context.Background(), w.TemplateID)

rows[i] = database.WorkspaceWithData{
Workspace: database.Workspace{
ID: w.ID,
CreatedAt: w.CreatedAt,
UpdatedAt: w.UpdatedAt,
OwnerID: w.OwnerID,
OrganizationID: w.OrganizationID,
TemplateID: w.TemplateID,
Deleted: w.Deleted,
Name: w.Name,
AutostartSchedule: w.AutostartSchedule,
Ttl: w.Ttl,
LastUsedAt: w.LastUsedAt,
},
OwnerUserName: owner.Username,
LatestBuildInitiatorUsername: latestBuildUser.Username,
LatestBuildTemplateVersionName: tv.Name,
TemplateName: tpl.Name,
TemplateIcon: tpl.Icon,
TemplateDisplayName: tpl.DisplayName,
TemplateAllowUserCancelWorkspaceJobs: tpl.AllowUserCancelWorkspaceJobs,
TemplateActiveVersionID: tpl.ActiveVersionID,
LatestBuild: latestBuild,
LatestBuildJob: job,
Count: count,
}
}
return rows
}

func (q *fakeQuerier) GetWorkspaceByID(_ context.Context, id uuid.UUID) (database.Workspace, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()

for _, workspace := range q.workspaces {
if workspace.ID == id {
return workspace, nil
}
func (q *fakeQuerier) GetWorkspaceByID(ctx context.Context, id uuid.UUID) (database.WorkspaceWithData, error) {
workspaces, err := q.GetWorkspaces(ctx, database.GetWorkspacesParams{
WorkspaceIds: []uuid.UUID{id},
Limit: 1,
})
if err != nil {
return database.WorkspaceWithData{}, err
}
return database.Workspace{}, sql.ErrNoRows
if len(workspaces) == 0 {
return database.WorkspaceWithData{}, sql.ErrNoRows
}
return workspaces[0], nil
}

func (q *fakeQuerier) GetWorkspaceByAgentID(_ context.Context, agentID uuid.UUID) (database.Workspace, error) {
Expand Down Expand Up @@ -1228,36 +1244,24 @@ func (q *fakeQuerier) GetWorkspaceByAgentID(_ context.Context, agentID uuid.UUID
return database.Workspace{}, sql.ErrNoRows
}

func (q *fakeQuerier) GetWorkspaceByOwnerIDAndName(_ context.Context, arg database.GetWorkspaceByOwnerIDAndNameParams) (database.Workspace, error) {
func (q *fakeQuerier) GetWorkspaceByOwnerIDAndName(ctx context.Context, arg database.GetWorkspaceByOwnerIDAndNameParams) (database.WorkspaceWithData, error) {
if err := validateDatabaseType(arg); err != nil {
return database.Workspace{}, err
return database.WorkspaceWithData{}, err
}

q.mutex.RLock()
defer q.mutex.RUnlock()

var found *database.Workspace
for _, workspace := range q.workspaces {
workspace := workspace
if workspace.OwnerID != arg.OwnerID {
continue
}
if !strings.EqualFold(workspace.Name, arg.Name) {
continue
}
if workspace.Deleted != arg.Deleted {
continue
}

// Return the most recent workspace with the given name
if found == nil || workspace.CreatedAt.After(found.CreatedAt) {
found = &workspace
}
workspaces, err := q.GetWorkspaces(ctx, database.GetWorkspacesParams{
OwnerID: arg.OwnerID,
ExactName: arg.Name,
Deleted: arg.Deleted,
Limit: 1,
})
if err != nil {
return database.WorkspaceWithData{}, err
}
if found != nil {
return *found, nil
if len(workspaces) == 0 {
return database.WorkspaceWithData{}, sql.ErrNoRows
}
return database.Workspace{}, sql.ErrNoRows
return workspaces[0], nil
}

func (q *fakeQuerier) GetWorkspaceByWorkspaceAppID(_ context.Context, workspaceAppID uuid.UUID) (database.Workspace, error) {
Expand Down Expand Up @@ -1699,7 +1703,7 @@ func (q *fakeQuerier) GetAuthorizedTemplates(ctx context.Context, arg database.G

// Call this to match the same function calls as the SQL implementation.
if prepared != nil {
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithACL())
_, err := prepared.CompileToSQL(ctx, rbac.ConfigWithACL(""))
if err != nil {
return nil, err
}
Expand Down
25 changes: 4 additions & 21 deletions coderd/database/modelmethods.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ func (g Group) RBACObject() rbac.Object {
InOrg(g.OrganizationID)
}

func (w WorkspaceWithData) RBACObject() rbac.Object {
return w.Workspace.RBACObject()
}

func (w Workspace) RBACObject() rbac.Object {
return rbac.ResourceWorkspace.WithID(w.ID).
InOrg(w.OrganizationID).
Expand Down Expand Up @@ -177,24 +181,3 @@ func ConvertUserRows(rows []GetUsersRow) []User {

return users
}

func ConvertWorkspaceRows(rows []GetWorkspacesRow) []Workspace {
workspaces := make([]Workspace, len(rows))
for i, r := range rows {
workspaces[i] = Workspace{
ID: r.ID,
CreatedAt: r.CreatedAt,
UpdatedAt: r.UpdatedAt,
OwnerID: r.OwnerID,
OrganizationID: r.OrganizationID,
TemplateID: r.TemplateID,
Deleted: r.Deleted,
Name: r.Name,
AutostartSchedule: r.AutostartSchedule,
Ttl: r.Ttl,
LastUsedAt: r.LastUsedAt,
}
}

return workspaces
}
Loading