diff --git a/cli/configssh.go b/cli/configssh.go index 2164996c1ae..193d915d2f1 100644 --- a/cli/configssh.go +++ b/cli/configssh.go @@ -474,25 +474,13 @@ func (r *RootCmd) configSSH() *serpent.Command { if configOptions.noWildcard { // Fetch all workspaces to generate individual host entries. - var wsNames []string - offset := 0 - const pageSize = 100 - for { - res, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{ - Owner: codersdk.Me, - Offset: offset, - Limit: pageSize, - }) - if err != nil { - return xerrors.Errorf("fetch workspaces: %w", err) - } - for _, ws := range res.Workspaces { - wsNames = append(wsNames, ws.Name) - } - if len(res.Workspaces) < pageSize { - break - } - offset += pageSize + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{Owner: codersdk.Me}) + if err != nil { + return xerrors.Errorf("fetch workspaces: %w", err) + } + wsNames := make([]string, 0, len(workspaces)) + for _, ws := range workspaces { + wsNames = append(wsNames, ws.Name) } configOptions.workspaceNames = wsNames } diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go index c49a228a54d..e3d97ea07f5 100644 --- a/cli/exp_scaletest.go +++ b/cli/exp_scaletest.go @@ -551,8 +551,6 @@ func (r *prebuildTemplateCleanupRunner) Run(ctx context.Context, _ string, _ io. // caught in the cleanup. If template is non-empty only workspaces for that // template are returned. func getScaletestPrebuildWorkspaces(ctx context.Context, client *codersdk.Client, template string) ([]codersdk.Workspace, error) { - const pageSize = 100 - templates, err := getScaletestPrebuildsTemplates(ctx, client, template) if err != nil { return nil, xerrors.Errorf("list scaletest prebuild templates: %w", err) @@ -562,23 +560,16 @@ func getScaletestPrebuildWorkspaces(ctx context.Context, client *codersdk.Client var result []codersdk.Workspace for _, tmpl := range templates { - for page := 0; ; page++ { - resp, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{ - Template: tmpl.Name, - Offset: page * pageSize, - Limit: pageSize, - }) - if err != nil { - return nil, xerrors.Errorf("list workspaces for template %q (page %d): %w", tmpl.Name, page, err) - } - for _, ws := range resp.Workspaces { - if _, ok := seen[ws.ID]; !ok { - seen[ws.ID] = struct{}{} - result = append(result, ws) - } - } - if len(resp.Workspaces) < pageSize { - break + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ + Template: tmpl.Name, + }) + if err != nil { + return nil, xerrors.Errorf("list workspaces for template %q: %w", tmpl.Name, err) + } + for _, ws := range workspaces { + if _, ok := seen[ws.ID]; !ok { + seen[ws.ID] = struct{}{} + result = append(result, ws) } } } @@ -2219,8 +2210,6 @@ func (r *runnableTraceWrapper) GetMetrics() map[string]any { func getScaletestWorkspaces(ctx context.Context, client *codersdk.Client, owner, template string) ([]codersdk.Workspace, int, error) { var ( - pageNumber = 0 - limit = 100 workspaces []codersdk.Workspace skipped int ) @@ -2236,35 +2225,24 @@ func getScaletestWorkspaces(ctx context.Context, client *codersdk.Client, owner, } noOwnerAccess := dv.Values != nil && dv.Values.DisableOwnerWorkspaceExec.Value() - for { - page, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{ - Name: "scaletest-", - Template: template, - Owner: owner, - Offset: pageNumber * limit, - Limit: limit, - }) - if err != nil { - return nil, 0, xerrors.Errorf("fetch scaletest workspaces page %d: %w", pageNumber, err) - } + all, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ + Name: "scaletest-", + Template: template, + Owner: owner, + }) + if err != nil { + return nil, 0, xerrors.Errorf("fetch scaletest workspaces: %w", err) + } - pageNumber++ - if len(page.Workspaces) == 0 { - break + for _, w := range all { + if !loadtestutil.IsScaleTestWorkspace(w.Name, w.OwnerName) { + continue } - - pageWorkspaces := make([]codersdk.Workspace, 0, len(page.Workspaces)) - for _, w := range page.Workspaces { - if !loadtestutil.IsScaleTestWorkspace(w.Name, w.OwnerName) { - continue - } - if noOwnerAccess && w.OwnerID != me.ID { - skipped++ - continue - } - pageWorkspaces = append(pageWorkspaces, w) + if noOwnerAccess && w.OwnerID != me.ID { + skipped++ + continue } - workspaces = append(workspaces, pageWorkspaces...) + workspaces = append(workspaces, w) } return workspaces, skipped, nil } diff --git a/cli/list.go b/cli/list.go index 8b4c56edbc5..09e15f2e663 100644 --- a/cli/list.go +++ b/cli/list.go @@ -168,12 +168,12 @@ func (r *RootCmd) list() *serpent.Command { // convert workspaces to scheduleListRow. func QueryConvertWorkspaces[T any](ctx context.Context, client *codersdk.Client, filter codersdk.WorkspaceFilter, convertF func(time.Time, codersdk.Workspace) T) ([]T, error) { var empty []T - workspaces, err := client.Workspaces(ctx, filter) + workspaces, err := client.AllWorkspaces(ctx, filter) if err != nil { return empty, xerrors.Errorf("query workspaces: %w", err) } - converted := make([]T, len(workspaces.Workspaces)) - for i, workspace := range workspaces.Workspaces { + converted := make([]T, len(workspaces)) + for i, workspace := range workspaces { converted[i] = convertF(time.Now(), workspace) } return converted, nil diff --git a/cli/ssh.go b/cli/ssh.go index d18ac8909f5..4fe13c3dfdd 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -171,7 +171,7 @@ func (r *RootCmd) ssh() *serpent.Command { return []string{} } - res, err := client.Workspaces(inv.Context(), codersdk.WorkspaceFilter{ + workspaces, err := client.AllWorkspaces(inv.Context(), codersdk.WorkspaceFilter{ Owner: codersdk.Me, }) if err != nil { @@ -181,7 +181,7 @@ func (r *RootCmd) ssh() *serpent.Command { var mu sync.Mutex var completions []string var wg sync.WaitGroup - for _, ws := range res.Workspaces { + for _, ws := range workspaces { wg.Add(1) go func() { defer wg.Done() diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 526fe4b48fa..22afa444e4b 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -13617,7 +13617,7 @@ const docTemplate = `{ }, { "type": "integer", - "description": "Page limit", + "description": "Page limit, from 1 to 1000. Defaults to 1000 when omitted.", "name": "limit", "in": "query" }, @@ -13634,6 +13634,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/codersdk.WorkspacesResponse" } + }, + "400": { + "description": "Invalid query parameters, including a limit outside 1 to 1000", + "schema": { + "$ref": "#/definitions/codersdk.Response" + } } }, "security": [ @@ -28726,6 +28732,7 @@ const docTemplate = `{ "type": "object", "properties": { "count": { + "description": "Count is the number of workspaces matching the filter before the limit and\noffset are applied. Workspaces the requester cannot fully read are omitted\nfrom the page after the limit is applied, so a page shorter than the limit\ndoes not mean the result set is exhausted.", "type": "integer" }, "workspaces": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index a009b6e7086..468f71567e8 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -12083,7 +12083,7 @@ }, { "type": "integer", - "description": "Page limit", + "description": "Page limit, from 1 to 1000. Defaults to 1000 when omitted.", "name": "limit", "in": "query" }, @@ -12100,6 +12100,12 @@ "schema": { "$ref": "#/definitions/codersdk.WorkspacesResponse" } + }, + "400": { + "description": "Invalid query parameters, including a limit outside 1 to 1000", + "schema": { + "$ref": "#/definitions/codersdk.Response" + } } }, "security": [ @@ -26476,6 +26482,7 @@ "type": "object", "properties": { "count": { + "description": "Count is the number of workspaces matching the filter before the limit and\noffset are applied. Workspaces the requester cannot fully read are omitted\nfrom the page after the limit is applied, so a page shorter than the limit\ndoes not mean the result set is exhausted.", "type": "integer" }, "workspaces": { diff --git a/coderd/pagination.go b/coderd/pagination.go index 011f8df9e7b..90db39002d4 100644 --- a/coderd/pagination.go +++ b/coderd/pagination.go @@ -1,6 +1,7 @@ package coderd import ( + "fmt" "net/http" "github.com/google/uuid" @@ -32,3 +33,38 @@ func ParsePagination(w http.ResponseWriter, r *http.Request) (p codersdk.Paginat return params, true } + +// ParsePaginationBounded extracts pagination query params from the http request +// and resolves limit against maxLimit. An omitted limit resolves to maxLimit. A +// limit that is present must be an integer in [1, maxLimit]; anything else is +// rejected rather than clamped. If an error is encountered, the error is written +// to w and ok is set to false. +func ParsePaginationBounded(w http.ResponseWriter, r *http.Request, maxLimit int) (p codersdk.Pagination, ok bool) { + ctx := r.Context() + queryParams := r.URL.Query() + parser := httpapi.NewQueryParamParser() + params := codersdk.Pagination{ + AfterID: parser.UUID(queryParams, uuid.Nil, "after_id"), + Offset: int(parser.PositiveInt32(queryParams, 0, "offset")), + } + + limitErrs := len(parser.Errors) + params.Limit = parser.Int(queryParams, maxLimit, "limit") + limitParsed := len(parser.Errors) == limitErrs + if limitParsed && (params.Limit < 1 || params.Limit > maxLimit) { + parser.Errors = append(parser.Errors, codersdk.ValidationError{ + Field: "limit", + Detail: fmt.Sprintf("Query param \"limit\" must be an integer between 1 and %d.", maxLimit), + }) + } + + if len(parser.Errors) > 0 { + httpapi.Write(ctx, w, http.StatusBadRequest, codersdk.Response{ + Message: "Query parameters have invalid values.", + Validations: parser.Errors, + }) + return params, false + } + + return params, true +} diff --git a/coderd/pagination_test.go b/coderd/pagination_test.go index f6e1aab7067..3fd39a9867a 100644 --- a/coderd/pagination_test.go +++ b/coderd/pagination_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd" + "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" ) @@ -139,3 +140,87 @@ func TestPagination(t *testing.T) { }) } } + +func TestPaginationBounded(t *testing.T) { + t.Parallel() + const maxLimit = 100 + testCases := []struct { + Name string + + // Limit is omitted from the query when nil. + Limit *string + Offset string + + ExpectedError string + ExpectedParams codersdk.Pagination + }{ + { + Name: "OmittedLimitDefaultsToMax", + ExpectedParams: codersdk.Pagination{Limit: maxLimit}, + }, + { + Name: "MaxLimit", + Limit: ptr.Ref("100"), + ExpectedParams: codersdk.Pagination{Limit: maxLimit}, + }, + { + Name: "BelowMaxLimit", + Limit: ptr.Ref("25"), + Offset: "50", + ExpectedParams: codersdk.Pagination{Limit: 25, Offset: 50}, + }, + { + Name: "ZeroLimit", + Limit: ptr.Ref("0"), + ExpectedError: "must be an integer between 1 and 100", + }, + { + Name: "AboveMaxLimit", + Limit: ptr.Ref("101"), + ExpectedError: "must be an integer between 1 and 100", + }, + { + Name: "NegativeLimit", + Limit: ptr.Ref("-1"), + ExpectedError: "must be an integer between 1 and 100", + }, + { + Name: "UnparseableLimit", + Limit: ptr.Ref("bogus"), + ExpectedError: "must be a valid integer", + }, + } + + for _, c := range testCases { + t.Run(c.Name, func(t *testing.T) { + t.Parallel() + rw := httptest.NewRecorder() + r, err := http.NewRequestWithContext(context.Background(), "GET", "https://example.com", nil) + require.NoError(t, err, "new request") + + query := r.URL.Query() + if c.Limit != nil { + query.Set("limit", *c.Limit) + } + if c.Offset != "" { + query.Set("offset", c.Offset) + } + r.URL.RawQuery = query.Encode() + + params, ok := coderd.ParsePaginationBounded(rw, r, maxLimit) + if c.ExpectedError == "" { + require.True(t, ok, "expect ok") + require.Equal(t, c.ExpectedParams, params, "expected params") + return + } + + require.False(t, ok, "expect !ok") + require.Equal(t, http.StatusBadRequest, rw.Code, "bad request status code") + var apiError codersdk.Error + require.NoError(t, json.NewDecoder(rw.Body).Decode(&apiError), "decode response") + require.Len(t, apiError.Validations, 1, "one validation error") + require.Equal(t, "limit", apiError.Validations[0].Field) + require.Contains(t, apiError.Validations[0].Detail, c.ExpectedError) + }) + } +} diff --git a/coderd/workspaces.go b/coderd/workspaces.go index 21d11d88b76..4cacb8c03c8 100644 --- a/coderd/workspaces.go +++ b/coderd/workspaces.go @@ -144,15 +144,16 @@ func (api *API) workspace(rw http.ResponseWriter, r *http.Request) { // @Produce json // @Tags Workspaces // @Param q query string false "Search query in the format `key:value`. Available keys are: owner, template, name, status, has-agent, dormant, last_used_after, last_used_before, has-ai-task, has_external_agent, healthy, include_agent_metadata (expands each agent with the named metadata keys rather than filtering; repeat the key for multiple items)." -// @Param limit query int false "Page limit" +// @Param limit query int false "Page limit, from 1 to 1000. Defaults to 1000 when omitted." // @Param offset query int false "Page offset" // @Success 200 {object} codersdk.WorkspacesResponse +// @Failure 400 {object} codersdk.Response "Invalid query parameters, including a limit outside 1 to 1000" // @Router /api/v2/workspaces [get] func (api *API) workspaces(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() apiKey := httpmw.APIKey(r) - page, ok := ParsePagination(rw, r) + page, ok := ParsePaginationBounded(rw, r, codersdk.WorkspacesPageLimit) if !ok { return } diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index fdf33461af1..22eb645abad 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -11,6 +11,7 @@ import ( "net/http/httptest" "regexp" "slices" + "strconv" "strings" "testing" "time" @@ -3329,6 +3330,34 @@ func TestOffsetLimit(t *testing.T) { require.Error(t, err) } +func TestWorkspacesPageLimit(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + // A limit outside [1, codersdk.WorkspacesPageLimit] is rejected rather than + // clamped. codersdk.Pagination omits a zero limit, so the query is built by + // hand. + for _, limit := range []string{"0", strconv.Itoa(codersdk.WorkspacesPageLimit + 1)} { + res, err := client.Request(ctx, http.MethodGet, "/api/v2/workspaces?limit="+limit, nil) + require.NoError(t, err) + _ = res.Body.Close() + require.Equal(t, http.StatusBadRequest, res.StatusCode) + } + + _, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{Limit: codersdk.WorkspacesPageLimit + 1}) + var apiErr *codersdk.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusBadRequest, apiErr.StatusCode()) + require.Len(t, apiErr.Validations, 1) + require.Equal(t, "limit", apiErr.Validations[0].Field) + + // The maximum is accepted. + _, err = client.Workspaces(ctx, codersdk.WorkspaceFilter{Limit: codersdk.WorkspacesPageLimit}) + require.NoError(t, err) +} + func TestWorkspaceUpdateAutostart(t *testing.T) { t.Parallel() dublinLoc := mustLocation(t, "Europe/Dublin") diff --git a/codersdk/pagination.go b/codersdk/pagination.go index 2201277aeca..3e6c8427af0 100644 --- a/codersdk/pagination.go +++ b/codersdk/pagination.go @@ -7,6 +7,12 @@ import ( "github.com/google/uuid" ) +// WorkspacesPageLimit is the largest page size the workspaces list endpoint +// accepts and the page size clients use when reading that endpoint to +// exhaustion. A request that omits limit receives this many rows; a limit of 0 +// or greater than this is rejected. +const WorkspacesPageLimit = 1000 + // Pagination sets pagination options for the endpoints that support it. type Pagination struct { // AfterID returns all or up to Limit results after the given @@ -15,9 +21,10 @@ type Pagination struct { // set AfterID to the last UUID returned by the previous // request. AfterID uuid.UUID `json:"after_id,omitempty" format:"uuid"` - // Limit sets the maximum number of users to be returned - // in a single page. If the limit is <= 0, there is no limit - // and all users are returned. + // Limit sets the maximum number of records to be returned in a single + // page. An endpoint that bounds its page size rejects a limit outside its + // range; otherwise a limit <= 0 means no limit and every record is + // returned. Limit int `json:"limit,omitempty"` // Offset is used to indicate which page to return. An offset of 0 // returns the first 'limit' number of users. diff --git a/codersdk/toolsdk/chatgpt.go b/codersdk/toolsdk/chatgpt.go index 4761bb7b1fa..dec06e6677d 100644 --- a/codersdk/toolsdk/chatgpt.go +++ b/codersdk/toolsdk/chatgpt.go @@ -69,14 +69,14 @@ func searchTemplates(ctx context.Context, deps Deps, query string) ([]SearchResu func searchWorkspaces(ctx context.Context, deps Deps, query string) ([]SearchResultItem, error) { serverURL := deps.ServerURL() - workspaces, err := deps.coderClient.Workspaces(ctx, codersdk.WorkspaceFilter{ + workspaces, err := deps.coderClient.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ FilterQuery: query, }) if err != nil { return nil, err } - results := make([]SearchResultItem, len(workspaces.Workspaces)) - for i, workspace := range workspaces.Workspaces { + results := make([]SearchResultItem, len(workspaces)) + for i, workspace := range workspaces { results[i] = SearchResultItem{ ID: createObjectID(ObjectTypeWorkspace, workspace.ID.String()).String(), Title: workspace.Name, diff --git a/codersdk/toolsdk/toolsdk.go b/codersdk/toolsdk/toolsdk.go index 81908820a61..91390aa8b2c 100644 --- a/codersdk/toolsdk/toolsdk.go +++ b/codersdk/toolsdk/toolsdk.go @@ -595,14 +595,14 @@ var ListWorkspaces = Tool[ListWorkspacesArgs, []MinimalWorkspace]{ if owner == "" { owner = codersdk.Me } - workspaces, err := deps.coderClient.Workspaces(ctx, codersdk.WorkspaceFilter{ + workspaces, err := deps.coderClient.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ Owner: owner, }) if err != nil { return nil, err } - minimalWorkspaces := make([]MinimalWorkspace, len(workspaces.Workspaces)) - for i, workspace := range workspaces.Workspaces { + minimalWorkspaces := make([]MinimalWorkspace, len(workspaces)) + for i, workspace := range workspaces { minimalWorkspaces[i] = MinimalWorkspace{ ID: workspace.ID.String(), Name: workspace.Name, diff --git a/codersdk/workspaces.go b/codersdk/workspaces.go index 6a78ecd7b36..9017324a8e4 100644 --- a/codersdk/workspaces.go +++ b/codersdk/workspaces.go @@ -96,7 +96,11 @@ type WorkspacesRequest struct { type WorkspacesResponse struct { Workspaces []Workspace `json:"workspaces"` - Count int `json:"count"` + // Count is the number of workspaces matching the filter before the limit and + // offset are applied. Workspaces the requester cannot fully read are omitted + // from the page after the limit is applied, so a page shorter than the limit + // does not mean the result set is exhausted. + Count int `json:"count"` } type ProvisionerLogLevel string @@ -612,7 +616,8 @@ func (f WorkspaceFilter) asRequestOption() RequestOption { } } -// Workspaces returns all workspaces the authenticated user has access to. +// Workspaces returns a single page of the workspaces the authenticated user has +// access to. An unset filter Limit resolves to the endpoint's maximum page size. func (c *Client) Workspaces(ctx context.Context, filter WorkspaceFilter) (WorkspacesResponse, error) { page := Pagination{ Offset: filter.Offset, @@ -632,6 +637,43 @@ func (c *Client) Workspaces(ctx context.Context, filter WorkspaceFilter) (Worksp return wres, ReadBodyAsJSON(res, &wres) } +// AllWorkspaces requests successive pages of workspaces matching the filter and +// returns every row it receives, skipping rows it has already returned. Limit +// and Offset on the filter are ignored. +// +// The offset advances by the requested page size rather than by the number of +// rows received, since a page can be shorter than the limit without the result +// set being exhausted. The scan ends when the offset reaches Count. +// +// The pages are separate requests, so the result is not a snapshot. The order +// depends on workspace state that changes between requests: a row that moves +// later is skipped here by ID, and one that moves earlier can pass the current +// offset and be missed. A page that fails ends the call and discards the rows +// already read. +func (c *Client) AllWorkspaces(ctx context.Context, filter WorkspaceFilter) ([]Workspace, error) { + filter.Limit = WorkspacesPageLimit + filter.Offset = 0 + var all []Workspace + seen := make(map[uuid.UUID]struct{}) + for { + page, err := c.Workspaces(ctx, filter) + if err != nil { + return nil, err + } + for _, workspace := range page.Workspaces { + if _, ok := seen[workspace.ID]; ok { + continue + } + seen[workspace.ID] = struct{}{} + all = append(all, workspace) + } + filter.Offset += WorkspacesPageLimit + if filter.Offset >= page.Count { + return all, nil + } + } +} + // WorkspaceByOwnerAndName returns a workspace by the owner's UUID and the workspace's name. func (c *Client) WorkspaceByOwnerAndName(ctx context.Context, owner string, name string, params WorkspaceOptions) (Workspace, error) { res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/users/%s/workspace/%s", owner, name), nil, func(r *http.Request) { diff --git a/codersdk/workspaces_test.go b/codersdk/workspaces_test.go index 63cb99e0624..9cead348dd4 100644 --- a/codersdk/workspaces_test.go +++ b/codersdk/workspaces_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strconv" "sync/atomic" "testing" @@ -310,3 +311,152 @@ func TestResolveWorkspace(t *testing.T) { require.EqualValues(t, 0, hits.Load(), "invalid identifiers should fail before any HTTP request") }) } + +func TestAllWorkspaces(t *testing.T) { + t.Parallel() + + // newClient serves pages of the given sizes, recording the limit and offset + // of every request. count is reported as the total on every response, and + // every row across every page gets a distinct ID. + newClient := func(t *testing.T, count int, pageSizes ...int) (*codersdk.Client, *[][2]int) { + t.Helper() + var requests [][2]int + page := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + limit, _ := strconv.Atoi(r.URL.Query().Get("limit")) + offset, _ := strconv.Atoi(r.URL.Query().Get("offset")) + requests = append(requests, [2]int{limit, offset}) + + rows := 0 + if page < len(pageSizes) { + rows = pageSizes[page] + } + page++ + + workspaces := make([]codersdk.Workspace, rows) + for i := range workspaces { + workspaces[i].ID = uuid.New() + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(codersdk.WorkspacesResponse{ + Workspaces: workspaces, + Count: count, + }) + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + return codersdk.New(u), &requests + } + + t.Run("SinglePage", func(t *testing.T) { + t.Parallel() + + client, requests := newClient(t, 3, 3) + ctx := testutil.Context(t, testutil.WaitShort) + + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Len(t, workspaces, 3) + require.Equal(t, [][2]int{{codersdk.WorkspacesPageLimit, 0}}, *requests) + }) + + t.Run("AdvancesByPageSize", func(t *testing.T) { + t.Parallel() + + const count = codersdk.WorkspacesPageLimit*2 + 10 + client, requests := newClient(t, count, + codersdk.WorkspacesPageLimit, codersdk.WorkspacesPageLimit, 10) + ctx := testutil.Context(t, testutil.WaitShort) + + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Len(t, workspaces, count) + require.Equal(t, [][2]int{ + {codersdk.WorkspacesPageLimit, 0}, + {codersdk.WorkspacesPageLimit, codersdk.WorkspacesPageLimit}, + {codersdk.WorkspacesPageLimit, codersdk.WorkspacesPageLimit * 2}, + }, *requests) + }) + + // A page can be shorter than the requested limit because the endpoint drops + // rows after applying the limit, so a short page must not end the scan. + t.Run("ShortPageContinues", func(t *testing.T) { + t.Parallel() + + const count = codersdk.WorkspacesPageLimit + 20 + client, requests := newClient(t, count, 40, 20) + ctx := testutil.Context(t, testutil.WaitShort) + + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Len(t, workspaces, 60) + require.Len(t, *requests, 2) + }) + + t.Run("EmptyResult", func(t *testing.T) { + t.Parallel() + + client, requests := newClient(t, 0, 0) + ctx := testutil.Context(t, testutil.WaitShort) + + workspaces, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Empty(t, workspaces) + require.Len(t, *requests, 1) + }) + + t.Run("OverridesCallerPagination", func(t *testing.T) { + t.Parallel() + + client, requests := newClient(t, 1, 1) + ctx := testutil.Context(t, testutil.WaitShort) + + _, err := client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ + Limit: codersdk.WorkspacesPageLimit + 50, + Offset: 500, + }) + require.NoError(t, err) + require.Equal(t, [][2]int{{codersdk.WorkspacesPageLimit, 0}}, *requests) + }) + + // The order the endpoint applies depends on workspace state, so a row can + // move to a later page while the pages are being read and be returned twice. + t.Run("SkipsRepeatedRows", func(t *testing.T) { + t.Parallel() + + repeated := codersdk.Workspace{ID: uuid.New()} + unique := codersdk.Workspace{ID: uuid.New()} + pages := [][]codersdk.Workspace{ + {repeated}, + {repeated, unique}, + } + page := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var workspaces []codersdk.Workspace + if page < len(pages) { + workspaces = pages[page] + } + page++ + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(codersdk.WorkspacesResponse{ + Workspaces: workspaces, + Count: codersdk.WorkspacesPageLimit + 1, + }) + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + ctx := testutil.Context(t, testutil.WaitShort) + + workspaces, err := codersdk.New(u).AllWorkspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Equal(t, []codersdk.Workspace{repeated, unique}, workspaces) + }) +} diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 1e6dbe6e0ef..e13521e889f 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -17657,10 +17657,10 @@ If the schedule is empty, the user will be updated to use the default schedule.| ### Properties -| Name | Type | Required | Restrictions | Description | -|--------------|---------------------------------------------------|----------|--------------|-------------| -| `count` | integer | false | | | -| `workspaces` | array of [codersdk.Workspace](#codersdkworkspace) | false | | | +| Name | Type | Required | Restrictions | Description | +|--------------|---------------------------------------------------|----------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `count` | integer | false | | Count is the number of workspaces matching the filter before the limit and offset are applied. Workspaces the requester cannot fully read are omitted from the page after the limit is applied, so a page shorter than the limit does not mean the result set is exhausted. | +| `workspaces` | array of [codersdk.Workspace](#codersdkworkspace) | false | | | ## derp.BytesSentRecv diff --git a/docs/reference/api/workspaces.md b/docs/reference/api/workspaces.md index 7a22c111e5e..2017fe60b9a 100644 --- a/docs/reference/api/workspaces.md +++ b/docs/reference/api/workspaces.md @@ -1107,7 +1107,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \ | Name | In | Type | Required | Description | |----------|-------|---------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `q` | query | string | false | Search query in the format `key:value`. Available keys are: owner, template, name, status, has-agent, dormant, last_used_after, last_used_before, has-ai-task, has_external_agent, healthy, include_agent_metadata (expands each agent with the named metadata keys rather than filtering; repeat the key for multiple items). | -| `limit` | query | integer | false | Page limit | +| `limit` | query | integer | false | Page limit, from 1 to 1000. Defaults to 1000 when omitted. | | `offset` | query | integer | false | Page offset | ### Example responses @@ -1385,9 +1385,10 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \ ### Responses -| Status | Meaning | Description | Schema | -|--------|---------------------------------------------------------|-------------|----------------------------------------------------------------------| -| 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.WorkspacesResponse](schemas.md#codersdkworkspacesresponse) | +| Status | Meaning | Description | Schema | +|--------|------------------------------------------------------------------|---------------------------------------------------------------|----------------------------------------------------------------------| +| 200 | [OK](https://tools.ietf.org/html/rfc7231#section-6.3.1) | OK | [codersdk.WorkspacesResponse](schemas.md#codersdkworkspacesresponse) | +| 400 | [Bad Request](https://tools.ietf.org/html/rfc7231#section-6.5.1) | Invalid query parameters, including a limit outside 1 to 1000 | [codersdk.Response](schemas.md#codersdkresponse) | To perform this operation, you must be authenticated. [Learn more](authentication.md). diff --git a/scaletest/prebuilds/run.go b/scaletest/prebuilds/run.go index 59c902a90b2..c04e2738c54 100644 --- a/scaletest/prebuilds/run.go +++ b/scaletest/prebuilds/run.go @@ -172,19 +172,19 @@ func (r *Runner) measureCreation(ctx context.Context, logger slog.Logger) error defer cancel() tkr := r.cfg.Clock.TickerFunc(workspacesCtx, workspacePollInterval, func() error { - workspaces, err := r.client.Workspaces(workspacesCtx, codersdk.WorkspaceFilter{ + workspaces, err := r.client.AllWorkspaces(workspacesCtx, codersdk.WorkspaceFilter{ Template: r.template.Name, }) if err != nil { return xerrors.Errorf("list workspaces: %w", err) } - createdCount := len(workspaces.Workspaces) + createdCount := len(workspaces) runningCount := 0 failedCount := 0 succeededCount := 0 - for _, ws := range workspaces.Workspaces { + for _, ws := range workspaces { switch ws.LatestBuild.Job.Status { case codersdk.ProvisionerJobRunning: runningCount++ @@ -228,13 +228,13 @@ func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error // The reconciler may have created extra workspaces beyond the configured // target (e.g. replacements for failed builds), so using targetNumWorkspaces // as the denominator would undercount completed deletions. - initialWorkspaces, err := r.client.Workspaces(deletionCtx, codersdk.WorkspaceFilter{ + initialWorkspaces, err := r.client.AllWorkspaces(deletionCtx, codersdk.WorkspaceFilter{ Template: r.template.Name, }) if err != nil { return xerrors.Errorf("list workspaces at deletion start: %w", err) } - initialWorkspaceCount := len(initialWorkspaces.Workspaces) + initialWorkspaceCount := len(initialWorkspaces) // retryCount tracks how many delete builds we've submitted per workspace. // lastRetriedBuildID prevents submitting a second retry for the same failed @@ -243,7 +243,7 @@ func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error lastRetriedBuildID := make(map[uuid.UUID]uuid.UUID) tkr := r.cfg.Clock.TickerFunc(deletionCtx, workspacePollInterval, func() error { - workspaces, err := r.client.Workspaces(deletionCtx, codersdk.WorkspaceFilter{ + workspaces, err := r.client.AllWorkspaces(deletionCtx, codersdk.WorkspaceFilter{ Template: r.template.Name, }) if err != nil { @@ -255,7 +255,7 @@ func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error failedCount := 0 exhaustedCount := 0 - for _, ws := range workspaces.Workspaces { + for _, ws := range workspaces { if ws.LatestBuild.Transition != codersdk.WorkspaceTransitionDelete { // The reconciler hasn't submitted a delete build yet. continue @@ -298,7 +298,7 @@ func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error } } - completedCount := initialWorkspaceCount - len(workspaces.Workspaces) + completedCount := initialWorkspaceCount - len(workspaces) createdCount += completedCount r.cfg.Metrics.SetDeletionJobsCreated(createdCount, r.template.Name) @@ -306,13 +306,13 @@ func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error r.cfg.Metrics.SetDeletionJobsFailed(failedCount, r.template.Name) r.cfg.Metrics.SetDeletionJobsCompleted(completedCount, r.template.Name) - if len(workspaces.Workspaces) == 0 { + if len(workspaces) == 0 { return errTickerDone } // If every remaining workspace has exhausted all retries, fail // immediately rather than waiting for the timeout. - if exhaustedCount > 0 && exhaustedCount == len(workspaces.Workspaces) { + if exhaustedCount > 0 && exhaustedCount == len(workspaces) { return xerrors.Errorf("%d workspace(s) failed to delete after %d attempts", exhaustedCount, maxDeletionRetries+1) } @@ -408,7 +408,9 @@ func (r *Runner) Cleanup(ctx context.Context, _ string, logs io.Writer) error { } // Workspaces must be deleted before the template can be deleted. - workspaces, err := allWorkspacesForTemplate(ctx, r.client, r.template.Name) + workspaces, err := r.client.AllWorkspaces(ctx, codersdk.WorkspaceFilter{ + Template: r.template.Name, + }) if err != nil { return xerrors.Errorf("list workspaces for template %q: %w", r.template.Name, err) } @@ -461,28 +463,6 @@ func (r *Runner) Cleanup(ctx context.Context, _ string, logs io.Writer) error { return nil } -// allWorkspacesForTemplate returns all workspaces belonging to templateName, -// paginating through results until exhausted. -func allWorkspacesForTemplate(ctx context.Context, client *codersdk.Client, templateName string) ([]codersdk.Workspace, error) { - const pageSize = 100 - var workspaces []codersdk.Workspace - for page := 0; ; page++ { - resp, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{ - Template: templateName, - Offset: page * pageSize, - Limit: pageSize, - }) - if err != nil { - return nil, xerrors.Errorf("list workspaces page %d: %w", page, err) - } - workspaces = append(workspaces, resp.Workspaces...) - if len(resp.Workspaces) < pageSize { - break - } - } - return workspaces, nil -} - //go:embed tf/main.tf.tpl var templateContent string diff --git a/site/src/api/api.test.ts b/site/src/api/api.test.ts index f832533a53b..5a4c83996f6 100644 --- a/site/src/api/api.test.ts +++ b/site/src/api/api.test.ts @@ -8,6 +8,7 @@ import { } from "#/testHelpers/entities"; import { API, getURLWithSearchParams, ParameterValidationError } from "./api"; import type * as TypesGen from "./typesGenerated"; +import { WorkspacesPageLimit } from "./typesGenerated"; const axiosInstance = API.getAxiosInstance(); @@ -660,3 +661,111 @@ describe("api.ts", () => { }); }); }); + +describe("getAllWorkspaces", () => { + let nextWorkspaceID = 0; + + // Every row gets an ID distinct from every other row handed out by this + // helper, so pages never overlap unless a test builds them that way. + const workspacePage = (count: number, rows: number) => ({ + workspaces: Array.from({ length: rows }, () => ({ + ...MockWorkspace, + id: `ws-${nextWorkspaceID++}`, + })), + count, + }); + + it("issues a single request when the total fits in one page", async () => { + const page = workspacePage(3, 3); + const getWorkspaces = vi + .spyOn(API, "getWorkspaces") + .mockResolvedValueOnce(page); + + const result = await API.getAllWorkspaces({ q: "owner:me" }); + + expect(getWorkspaces).toHaveBeenCalledTimes(1); + expect(getWorkspaces).toHaveBeenCalledWith( + { + q: "owner:me", + limit: WorkspacesPageLimit, + offset: 0, + }, + undefined, + ); + expect(result).toStrictEqual(page); + }); + + it("advances the offset by the page size until the total is reached", async () => { + const total = WorkspacesPageLimit * 2 + 10; + const getWorkspaces = vi + .spyOn(API, "getWorkspaces") + .mockResolvedValueOnce(workspacePage(total, WorkspacesPageLimit)) + .mockResolvedValueOnce(workspacePage(total, WorkspacesPageLimit)) + .mockResolvedValueOnce(workspacePage(total, 10)); + + const result = await API.getAllWorkspaces(); + + expect(getWorkspaces.mock.calls.map(([req]) => req?.offset)).toStrictEqual([ + 0, + WorkspacesPageLimit, + WorkspacesPageLimit * 2, + ]); + expect(result.workspaces).toHaveLength(total); + expect(result.count).toBe(total); + }); + + it("keeps requesting pages when a page is shorter than the page size", async () => { + const total = WorkspacesPageLimit + 20; + const getWorkspaces = vi + .spyOn(API, "getWorkspaces") + .mockResolvedValueOnce(workspacePage(total, 40)) + .mockResolvedValueOnce(workspacePage(total, 50)); + + const result = await API.getAllWorkspaces(); + + expect(getWorkspaces).toHaveBeenCalledTimes(2); + expect(result.workspaces).toHaveLength(90); + expect(result.count).toBe(total); + }); + + // The order the endpoint applies depends on workspace state, so a row can + // move to a later page while the pages are being read and be returned twice. + it("skips a workspace an earlier page already returned", async () => { + const total = WorkspacesPageLimit + 1; + const repeated = { ...MockWorkspace, id: "repeated" }; + const unique = { ...MockWorkspace, id: "unique" }; + vi.spyOn(API, "getWorkspaces") + .mockResolvedValueOnce({ workspaces: [repeated], count: total }) + .mockResolvedValueOnce({ workspaces: [repeated, unique], count: total }); + + const result = await API.getAllWorkspaces(); + + expect(result.workspaces).toStrictEqual([repeated, unique]); + }); + + it("passes the abort signal to every page", async () => { + const total = WorkspacesPageLimit + 50; + const controller = new AbortController(); + const getWorkspaces = vi + .spyOn(API, "getWorkspaces") + .mockResolvedValueOnce(workspacePage(total, WorkspacesPageLimit)) + .mockResolvedValueOnce(workspacePage(total, 50)); + + await API.getAllWorkspaces({ q: "owner:me" }, controller.signal); + + expect(getWorkspaces).toHaveBeenCalledTimes(2); + for (const [, signal] of getWorkspaces.mock.calls) { + expect(signal).toBe(controller.signal); + } + }); + + it("rejects without returning the pages it already read", async () => { + vi.spyOn(API, "getWorkspaces") + .mockResolvedValueOnce( + workspacePage(WorkspacesPageLimit + 50, WorkspacesPageLimit), + ) + .mockRejectedValueOnce(new Error("canceled")); + + await expect(API.getAllWorkspaces()).rejects.toThrow("canceled"); + }); +}); diff --git a/site/src/api/api.ts b/site/src/api/api.ts index 078da917beb..624cdf71ada 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -1253,12 +1253,62 @@ class ApiMethods { getWorkspaces = async ( req: TypesGen.WorkspacesRequest, + signal?: AbortSignal, ): Promise => { const url = getURLWithSearchParams("/api/v2/workspaces", req); - const response = await this.axios.get(url); + const response = await this.axios.get(url, { + signal, + }); return response.data; }; + /** + * Requests successive pages of workspaces until the offset reaches the total + * the server reports, skipping workspaces an earlier page already returned. + * The offset advances by the page size rather than by the number of rows + * received, since a page can be shorter than the limit without the result set + * being exhausted. + * + * The pages are separate requests, so the result is not a snapshot. The order + * depends on workspace state that changes between requests: a workspace that + * moves later is skipped here by ID, and one that moves earlier can pass the + * current offset and be missed. A page that fails rejects the whole call and + * discards the rows already read; aborting `signal` rejects the page in + * flight. + * + * `count` is the total the server reported for the last page and can exceed + * the length of `workspaces`. + */ + getAllWorkspaces = async ( + req: Omit = {}, + signal?: AbortSignal, + ): Promise => { + const workspaces: TypesGen.Workspace[] = []; + const seen = new Set(); + let count = 0; + let offset = 0; + do { + const page = await this.getWorkspaces( + { + ...req, + limit: TypesGen.WorkspacesPageLimit, + offset, + }, + signal, + ); + for (const workspace of page.workspaces) { + if (seen.has(workspace.id)) { + continue; + } + seen.add(workspace.id); + workspaces.push(workspace); + } + count = page.count; + offset += TypesGen.WorkspacesPageLimit; + } while (offset < count); + return { workspaces, count }; + }; + getWorkspaceByOwnerAndName = async ( username: string, workspaceName: string, diff --git a/site/src/api/queries/workspaces.test.ts b/site/src/api/queries/workspaces.test.ts index a4ef076b1a7..dba77b3c938 100644 --- a/site/src/api/queries/workspaces.test.ts +++ b/site/src/api/queries/workspaces.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import type { WorkspacesResponse } from "#/api/typesGenerated"; import { getWorkspaceQuotaQueryKey } from "./workspaceQuota"; import { + allWorkspacesKey, autoCreateWorkspace, buildLogsKey, createWorkspace, @@ -38,6 +39,9 @@ const seedWorkspaceFamilyQueries = (queryClient: QueryClient) => { limit: 25, offset: 50, }); + const exhaustiveListKey = allWorkspacesKey({ + q: "owner:me organization:default", + }); const usageKey = workspaceUsage({ usageApp: "reconnecting-pty", connectionStatus: "connected", @@ -66,6 +70,7 @@ const seedWorkspaceFamilyQueries = (queryClient: QueryClient) => { queryClient.setQueryData(rawListKey, workspacesResponse); queryClient.setQueryData(defaultListKey, workspacesResponse); queryClient.setQueryData(filteredListKey, workspacesResponse); + queryClient.setQueryData(exhaustiveListKey, workspacesResponse); queryClient.setQueryData(usageKey, { tracked: true }); queryClient.setQueryData(buildLogs, []); queryClient.setQueryData(workspacePermissionsKey, { read: true }); @@ -73,7 +78,7 @@ const seedWorkspaceFamilyQueries = (queryClient: QueryClient) => { queryClient.setQueryData(organizationWorkspacePermissionsKey, { read: true }); return { - listKeys: [rawListKey, defaultListKey, filteredListKey], + listKeys: [rawListKey, defaultListKey, filteredListKey, exhaustiveListKey], nonListKeys: [ usageKey, buildLogs, diff --git a/site/src/api/queries/workspaces.ts b/site/src/api/queries/workspaces.ts index 0b89fd5f1f7..e14d351b39c 100644 --- a/site/src/api/queries/workspaces.ts +++ b/site/src/api/queries/workspaces.ts @@ -225,7 +225,29 @@ export function workspacesKey(req: WorkspacesRequest = {}) { export function workspaces(req: WorkspacesRequest = {}) { return { queryKey: workspacesKey(req), - queryFn: () => API.getWorkspaces(req), + queryFn: ({ signal }) => API.getWorkspaces(req, signal), + } as const satisfies QueryOptions; +} + +type AllWorkspacesRequest = Omit< + WorkspacesRequest, + "limit" | "offset" | "after_id" +>; + +export function allWorkspacesKey(req: AllWorkspacesRequest = {}) { + // The `all` marker keeps this distinct from the single-page key for the same + // filter. The key stays two segments long with an object at the end so it + // keeps the shape a workspace list key is matched by. + return [...workspacesQueryKeyPrefix, { ...req, all: true }] as const; +} + +// allWorkspaces reads every page of the filtered result set. The request count +// scales with the result set and a page that fails discards the rows already +// read, so prefer a filter that fits in one page. +export function allWorkspaces(req: AllWorkspacesRequest = {}) { + return { + queryKey: allWorkspacesKey(req), + queryFn: ({ signal }) => API.getAllWorkspaces(req, signal), } as const satisfies QueryOptions; } diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 0222095336d..8556b1fe699 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -6984,9 +6984,10 @@ export interface Pagination { */ readonly after_id?: string; /** - * Limit sets the maximum number of users to be returned - * in a single page. If the limit is <= 0, there is no limit - * and all users are returned. + * Limit sets the maximum number of records to be returned in a single + * page. An endpoint that bounds its page size rejects a limit outside its + * range; otherwise a limit <= 0 means no limit and every record is + * returned. */ readonly limit?: number; /** @@ -11499,6 +11500,15 @@ export interface WorkspaceUser extends MinimalUser { readonly role: WorkspaceRole; } +// From codersdk/pagination.go +/** + * WorkspacesPageLimit is the largest page size the workspaces list endpoint + * accepts and the page size clients use when reading that endpoint to + * exhaustion. A request that omits limit receives this many rows; a limit of 0 + * or greater than this is rejected. + */ +export const WorkspacesPageLimit = 1000; + // From codersdk/workspaces.go export interface WorkspacesRequest extends Pagination { readonly q?: string; @@ -11507,5 +11517,11 @@ export interface WorkspacesRequest extends Pagination { // From codersdk/workspaces.go export interface WorkspacesResponse { readonly workspaces: readonly Workspace[]; + /** + * Count is the number of workspaces matching the filter before the limit and + * offset are applied. Workspaces the requester cannot fully read are omitted + * from the page after the limit is applied, so a page shorter than the limit + * does not mean the result set is exhausted. + */ readonly count: number; } diff --git a/site/src/pages/AgentsPage/AgentChatPage.tsx b/site/src/pages/AgentsPage/AgentChatPage.tsx index 4fd0b0d09a3..3b4f5e9008b 100644 --- a/site/src/pages/AgentsPage/AgentChatPage.tsx +++ b/site/src/pages/AgentsPage/AgentChatPage.tsx @@ -52,9 +52,9 @@ import { deploymentSSHConfig } from "#/api/queries/deployment"; import { userSkills } from "#/api/queries/userSkills"; import { preferenceSettings } from "#/api/queries/users"; import { + allWorkspaces, workspaceById, workspaceByIdKey, - workspaces, } from "#/api/queries/workspaces"; import type * as TypesGen from "#/api/typesGenerated"; import type { ChatMessagePart } from "#/api/typesGenerated"; @@ -949,7 +949,7 @@ const AgentChatPage: FC = () => { const preferencesQuery = useQuery(preferenceSettings()); const userDebugLoggingQuery = useQuery(userChatDebugLogging()); const mcpServersQuery = useQuery(mcpServerConfigs()); - const workspacesQuery = useQuery(workspaces({ q: "owner:me", limit: 0 })); + const workspacesQuery = useQuery(allWorkspaces({ q: "owner:me" })); const workspaceOptions = getWorkspaceOptionsWithLinkedWorkspace( workspacesQuery.data?.workspaces ?? [], workspace, diff --git a/site/src/pages/AgentsPage/AgentCreatePage.tsx b/site/src/pages/AgentsPage/AgentCreatePage.tsx index 38f019e9cd8..10afbf4775a 100644 --- a/site/src/pages/AgentsPage/AgentCreatePage.tsx +++ b/site/src/pages/AgentsPage/AgentCreatePage.tsx @@ -13,7 +13,7 @@ import { userChatProviderConfigs, } from "#/api/queries/chats"; import { preferenceSettings } from "#/api/queries/users"; -import { workspaces } from "#/api/queries/workspaces"; +import { allWorkspaces } from "#/api/queries/workspaces"; import type * as TypesGen from "#/api/typesGenerated"; import { useWebpushNotifications } from "#/contexts/useWebpushNotifications"; import { useAuthenticated } from "#/hooks/useAuthenticated"; @@ -55,7 +55,7 @@ const AgentCreatePage: FC = () => { ); const preferencesQuery = useQuery(preferenceSettings()); const mcpServersQuery = useQuery(mcpServerConfigs()); - const workspacesQuery = useQuery(workspaces({ q: "owner:me", limit: 0 })); + const workspacesQuery = useQuery(allWorkspaces({ q: "owner:me" })); const createMutation = useMutation(createChat(queryClient)); const webPush = useWebpushNotifications(); const [chimeEnabled, setChimeEnabledState] = useState(getChimeEnabled); diff --git a/site/src/pages/AgentsPage/components/AgentCreateForm.tsx b/site/src/pages/AgentsPage/components/AgentCreateForm.tsx index e70a0914eee..9c4f86a90c2 100644 --- a/site/src/pages/AgentsPage/components/AgentCreateForm.tsx +++ b/site/src/pages/AgentsPage/components/AgentCreateForm.tsx @@ -379,14 +379,9 @@ export const AgentCreateForm: FC = ({ const isForbidden = !canCreateChat; - // Filter workspaces by the selected organization. We use - // client-side filtering of the full "owner:me" fetch rather - // than re-querying with an org filter because it avoids - // extra loading/error states on org change. The full list is - // already small (user's own workspaces) and limit: 0 - // guarantees completeness. If workspace counts grow large - // enough to warrant pagination, this should switch to a - // server-side organization: query filter. + // Filter workspaces by the selected organization. Filtering the fetched + // "owner:me" list on the client rather than re-querying with an org filter + // avoids extra loading and error states on org change. const filteredWorkspaces = showOrganizations && selectedOrg ? workspaceOptions.filter((ws) => ws.organization_id === selectedOrg.id) diff --git a/site/src/pages/AgentsPage/components/UsageIndicator.stories.tsx b/site/src/pages/AgentsPage/components/UsageIndicator.stories.tsx index 2b449c7e416..51c26809cc6 100644 --- a/site/src/pages/AgentsPage/components/UsageIndicator.stories.tsx +++ b/site/src/pages/AgentsPage/components/UsageIndicator.stories.tsx @@ -99,7 +99,7 @@ const noBudgetStatus = aiSpendStatus({ const userWorkspacesRequest = { q: `owner:me organization:${MockDefaultOrganization.name}`, - limit: 0, + limit: 1, }; const noWorkspaceQuota = { credits_consumed: 0, diff --git a/site/src/pages/AgentsPage/components/UsageIndicator.tsx b/site/src/pages/AgentsPage/components/UsageIndicator.tsx index 872e20b76a1..7086a83fe4b 100644 --- a/site/src/pages/AgentsPage/components/UsageIndicator.tsx +++ b/site/src/pages/AgentsPage/components/UsageIndicator.tsx @@ -69,7 +69,8 @@ export const UsageIndicator: FC = () => { const workspacesQuery = useQuery({ ...workspaces({ q: `owner:me organization:${organizationName}`, - limit: 0, + // Only the total is read, so a single row is enough. + limit: 1, }), enabled: hasWorkspaceQuotaUsage && organizationName !== "", }); diff --git a/site/src/pages/OrganizationSettingsPage/DisableWorkspaceSharingDialog.tsx b/site/src/pages/OrganizationSettingsPage/DisableWorkspaceSharingDialog.tsx index b95b9aea1d1..290011f7d48 100644 --- a/site/src/pages/OrganizationSettingsPage/DisableWorkspaceSharingDialog.tsx +++ b/site/src/pages/OrganizationSettingsPage/DisableWorkspaceSharingDialog.tsx @@ -39,7 +39,8 @@ export const DisableWorkspaceSharingDialog: FC< queryFn: async () => { const response = await API.getWorkspaces({ q: `organization:${organizationId} shared:true`, - limit: 0, // Avoid fetching workspaces as we only need the count. + // Only the total is read, so a single row is enough. + limit: 1, }); return response.count; }, diff --git a/site/src/pages/TemplatePage/TemplatePageHeader.stories.tsx b/site/src/pages/TemplatePage/TemplatePageHeader.stories.tsx index b3246423799..2557f782a6d 100644 --- a/site/src/pages/TemplatePage/TemplatePageHeader.stories.tsx +++ b/site/src/pages/TemplatePage/TemplatePageHeader.stories.tsx @@ -18,6 +18,7 @@ const meta: Meta = { { key: workspacesKey({ q: `organization:${MockTemplate.organization_name} template:${MockTemplate.name}`, + limit: 1, }), data: { workspaces: [], @@ -57,6 +58,7 @@ export const HasWorkspaces: Story = { { key: workspacesKey({ q: `organization:${MockTemplate.organization_name} template:${MockTemplate.name}`, + limit: 1, }), data: { workspaces: [MockWorkspace], diff --git a/site/src/pages/TemplatePage/TemplatePageHeader.tsx b/site/src/pages/TemplatePage/TemplatePageHeader.tsx index 2caa17843f5..fe28a19a4a2 100644 --- a/site/src/pages/TemplatePage/TemplatePageHeader.tsx +++ b/site/src/pages/TemplatePage/TemplatePageHeader.tsx @@ -69,7 +69,8 @@ const TemplateMenu: FC = ({ const getLink = useLinks(); const queryText = `organization:${organizationName} template:${templateName}`; const workspaceCountQuery = useQuery({ - ...workspaces({ q: queryText }), + // Only the total is read, so a single row is enough. + ...workspaces({ q: queryText, limit: 1 }), select: (res) => res.count, }); const safeToDeleteTemplate = workspaceCountQuery.data === 0; diff --git a/site/src/pages/TemplateSettingsPage/TemplateSchedulePage/useWorkspacesToBeDeleted.ts b/site/src/pages/TemplateSettingsPage/TemplateSchedulePage/useWorkspacesToBeDeleted.ts index 4ef61dc79ed..f6c3567e65b 100644 --- a/site/src/pages/TemplateSettingsPage/TemplateSchedulePage/useWorkspacesToBeDeleted.ts +++ b/site/src/pages/TemplateSettingsPage/TemplateSchedulePage/useWorkspacesToBeDeleted.ts @@ -1,6 +1,6 @@ import dayjs from "dayjs"; import { useQuery } from "react-query"; -import { workspaces } from "#/api/queries/workspaces"; +import { allWorkspaces } from "#/api/queries/workspaces"; import type { Template, Workspace } from "#/api/typesGenerated"; import type { TemplateScheduleFormValues } from "./formHelpers"; @@ -10,7 +10,7 @@ export const useWorkspacesToGoDormant = ( fromDate: Date, ) => { const { data } = useQuery( - workspaces({ + allWorkspaces({ q: `template:${template.name}`, }), ); @@ -44,7 +44,7 @@ export const useWorkspacesToBeDeleted = ( fromDate: Date, ) => { const { data } = useQuery( - workspaces({ + allWorkspaces({ q: `template:${template.name} dormant:true`, }), ); diff --git a/support/support.go b/support/support.go index 40be6ddf62c..0cf1081b947 100644 --- a/support/support.go +++ b/support/support.go @@ -265,18 +265,20 @@ func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logge // List workspaces (paginated) eg.Go(func() error { var ( - offset int - limit = 200 - all []codersdk.Workspace - count int + offset int + all []codersdk.Workspace + seen = make(map[uuid.UUID]struct{}) + count int + incomplete bool ) capTotal := workspacesCap for { - resp, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{Offset: offset, Limit: limit}) + resp, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{Offset: offset, Limit: codersdk.WorkspacesPageLimit}) if err != nil { // Log and continue if forbidden; otherwise return error if cerr, ok := codersdk.AsError(err); ok && (cerr.StatusCode() == http.StatusForbidden || cerr.StatusCode() == http.StatusUnauthorized) { - log.Warn(ctx, "unable to list workspaces") + log.Warn(ctx, "unable to list workspaces", slog.F("offset", offset)) + incomplete = true break } return xerrors.Errorf("list workspaces: %w", err) @@ -284,35 +286,49 @@ func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logge if d.Workspaces == nil { d.Workspaces = &resp } - // sanitize env vars on agents in each workspace before appending - for i := range resp.Workspaces { - ws := &resp.Workspaces[i] + // The order depends on build state, which changes between requests, so a + // workspace can move to a page that has not been read yet. + for _, ws := range resp.Workspaces { + if _, ok := seen[ws.ID]; ok { + continue + } + seen[ws.ID] = struct{}{} for _, res := range ws.LatestBuild.Resources { for _, agt := range res.Agents { // safe to call even if map is nil (range in sanitizeEnv would be empty) sanitizeEnv(agt.EnvironmentVariables) } } + all = append(all, ws) } - all = append(all, resp.Workspaces...) count = resp.Count - // Stop early once we've reached the cap; trim any overflow from the last page. + // Stop early once we've reached the cap; trim any overflow from the last + // page. The cap limits how many workspaces are kept, not how many are + // requested per page. if capTotal > 0 && len(all) >= capTotal { if len(all) > capTotal { all = all[:capTotal] } break } - if offset+len(resp.Workspaces) >= count || len(resp.Workspaces) == 0 { + // The offset advances by the requested limit rather than by the number + // of rows returned, since a page can be shorter than the limit without + // the set being exhausted. + offset += codersdk.WorkspacesPageLimit + if offset >= count { break } - offset += len(resp.Workspaces) } if d.Workspaces != nil { // Replace with aggregated list d.Workspaces.Workspaces = all // Preserve server-reported total so Run() can log accurate truncation. d.Workspaces.Count = count + if incomplete { + // The scan stopped early, so the server total describes a set this + // list does not cover. + d.Workspaces.Count = len(all) + } } return nil }) diff --git a/support/support_test.go b/support/support_test.go index 35bfa556731..f8f2870e994 100644 --- a/support/support_test.go +++ b/support/support_test.go @@ -7,11 +7,14 @@ import ( "fmt" "io" "net/http" + "net/http/httptest" + "net/url" "os" "path/filepath" "testing" "time" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" @@ -334,3 +337,97 @@ func assertNotNilNotEmpty[T any](t *testing.T, v T, msg string) { assert.NotEmpty(t, v, msg+" but was empty") } } + +// TestDeploymentInfoWorkspacesIncomplete serves one full page and then denies +// the next one, which is what a token losing access mid-scan looks like. +func TestDeploymentInfoWorkspacesIncomplete(t *testing.T) { + t.Parallel() + + const serverTotal = codersdk.WorkspacesPageLimit * 2 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v2/workspaces" { + w.WriteHeader(http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + // The client omits a zero offset, so an absent value is the first page. + if offset := r.URL.Query().Get("offset"); offset != "" && offset != "0" { + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(codersdk.Response{Message: "forbidden"}) + return + } + + workspaces := make([]codersdk.Workspace, codersdk.WorkspacesPageLimit) + for i := range workspaces { + workspaces[i].ID = uuid.New() + } + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(codersdk.WorkspacesResponse{ + Workspaces: workspaces, + Count: serverTotal, + }) + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + ctx := testutil.Context(t, testutil.WaitShort) + + d := support.DeploymentInfo(ctx, codersdk.New(u), + slog.Make(sloghuman.Sink(io.Discard)), 0) + + require.NotNil(t, d.Workspaces) + require.Len(t, d.Workspaces.Workspaces, codersdk.WorkspacesPageLimit) + // The count describes the rows in the bundle, not the larger set the server + // reported for a scan that did not finish. + require.Equal(t, codersdk.WorkspacesPageLimit, d.Workspaces.Count) +} + +// TestDeploymentInfoWorkspacesRepeated returns the same workspace on two pages, +// which is what a build finishing between requests looks like. +func TestDeploymentInfoWorkspacesRepeated(t *testing.T) { + t.Parallel() + + repeated := codersdk.Workspace{ID: uuid.New(), Name: "repeated"} + unique := codersdk.Workspace{ID: uuid.New(), Name: "unique"} + pages := [][]codersdk.Workspace{ + {repeated}, + {repeated, unique}, + } + var page int + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v2/workspaces" { + w.WriteHeader(http.StatusNotFound) + return + } + + var workspaces []codersdk.Workspace + if page < len(pages) { + workspaces = pages[page] + } + page++ + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(codersdk.WorkspacesResponse{ + Workspaces: workspaces, + Count: codersdk.WorkspacesPageLimit + 1, + }) + })) + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + require.NoError(t, err) + ctx := testutil.Context(t, testutil.WaitShort) + + d := support.DeploymentInfo(ctx, codersdk.New(u), + slog.Make(sloghuman.Sink(io.Discard)), 0) + + require.NotNil(t, d.Workspaces) + names := make([]string, 0, len(d.Workspaces.Workspaces)) + for _, ws := range d.Workspaces.Workspaces { + names = append(names, ws.Name) + } + require.Equal(t, []string{"repeated", "unique"}, names) +}