Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 7364933

Browse files
kylecarbskylecarbs
authored
refactor: Allow provisioner jobs to be disconnected from projects (#194)
* Nest jobs under an organization * Rename project parameter to parameter schema * Update references when computing project parameters * Add files endpoint * Allow one-off project import jobs * Allow variables to be injected that are not defined by the schema * Update API to use jobs first * Fix CLI tests * Fix linting * Fix hex length for files table * Reduce memory allocation for windows
1 parent 4c5e443 commit 7364933

37 files changed

+1373
-988
lines changed

‎coderd/coderd.go

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,14 +102,27 @@ func New(options *Options) http.Handler {
102102
})
103103
})
104104

105+
r.Route("/files", func(r chi.Router) {
106+
r.Use(httpmw.ExtractAPIKey(options.Database, nil))
107+
r.Post("/", api.postFiles)
108+
})
109+
105110
r.Route("/provisioners", func(r chi.Router) {
106111
r.Route("/daemons", func(r chi.Router) {
107112
r.Get("/", api.provisionerDaemons)
108113
r.Get("/serve", api.provisionerDaemonsServe)
109114
})
110-
r.Route("/jobs/{provisionerjob}", func(r chi.Router) {
111-
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
112-
r.Get("/logs", api.provisionerJobLogsByID)
115+
r.Route("/jobs/{organization}", func(r chi.Router) {
116+
r.Use(
117+
httpmw.ExtractAPIKey(options.Database, nil),
118+
httpmw.ExtractOrganizationParam(options.Database),
119+
)
120+
r.Post("/import", api.postProvisionerImportJobByOrganization)
121+
r.Route("/{provisionerjob}", func(r chi.Router) {
122+
r.Use(httpmw.ExtractProvisionerJobParam(options.Database))
123+
r.Get("/", api.provisionerJobByOrganization)
124+
r.Get("/logs", api.provisionerJobLogsByID)
125+
})
113126
})
114127
})
115128
})

‎coderd/coderdtest/coderdtest.go

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -122,40 +122,44 @@ func CreateInitialUser(t *testing.T, client *codersdk.Client) coderd.CreateIniti
122122
return req
123123
}
124124

125-
// CreateProject creates a project with the "echo" provisioner for
126-
// compatibility with testing. The name assigned is randomly generated.
127-
func CreateProject(t *testing.T, client *codersdk.Client, organization string) coderd.Project {
128-
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
129-
Name: randomUsername(),
130-
Provisioner: database.ProvisionerTypeEcho,
125+
// CreateProjectImportProvisionerJob creates a project import provisioner job
126+
// with the responses provided. It uses the "echo" provisioner for compatibility
127+
// with testing.
128+
func CreateProjectImportProvisionerJob(t *testing.T, client *codersdk.Client, organization string, res *echo.Responses) coderd.ProvisionerJob {
129+
data, err := echo.Tar(res)
130+
require.NoError(t, err)
131+
file, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, data)
132+
require.NoError(t, err)
133+
job, err := client.CreateProjectVersionImportProvisionerJob(context.Background(), organization, coderd.CreateProjectImportJobRequest{
134+
StorageSource: file.Hash,
135+
StorageMethod: database.ProvisionerStorageMethodFile,
136+
Provisioner: database.ProvisionerTypeEcho,
131137
})
132138
require.NoError(t, err)
133-
return project
139+
return job
134140
}
135141

136-
// CreateProjectVersion creates a project version for the "echo" provisioner
137-
// for compatibility with testing.
138-
func CreateProjectVersion(t *testing.T, client *codersdk.Client, organization, project string, responses *echo.Responses) coderd.ProjectVersion {
139-
data, err := echo.Tar(responses)
140-
require.NoError(t, err)
141-
version, err := client.CreateProjectVersion(context.Background(), organization, project, coderd.CreateProjectVersionRequest{
142-
StorageMethod: database.ProjectStorageMethodInlineArchive,
143-
StorageSource: data,
142+
// CreateProject creates a project with the "echo" provisioner for
143+
// compatibility with testing. The name assigned is randomly generated.
144+
func CreateProject(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.Project {
145+
project, err := client.CreateProject(context.Background(), organization, coderd.CreateProjectRequest{
146+
Name: randomUsername(),
147+
VersionImportJobID: job,
144148
})
145149
require.NoError(t, err)
146-
return version
150+
return project
147151
}
148152

149-
// AwaitProjectVersionImported awaits for the project import job to reach completed status.
150-
func AwaitProjectVersionImported(t *testing.T, client *codersdk.Client, organization, project, version string) coderd.ProjectVersion {
151-
var projectVersion coderd.ProjectVersion
153+
// AwaitProvisionerJob awaits for a job to reach completed status.
154+
func AwaitProvisionerJob(t *testing.T, client *codersdk.Client, organization string, job uuid.UUID) coderd.ProvisionerJob {
155+
var provisionerJob coderd.ProvisionerJob
152156
require.Eventually(t, func() bool {
153157
var err error
154-
projectVersion, err = client.ProjectVersion(context.Background(), organization, project, version)
158+
provisionerJob, err = client.ProvisionerJob(context.Background(), organization, job)
155159
require.NoError(t, err)
156-
return projectVersion.Import.Status.Completed()
160+
return provisionerJob.Status.Completed()
157161
}, 3*time.Second, 25*time.Millisecond)
158-
return projectVersion
162+
return provisionerJob
159163
}
160164

161165
// CreateWorkspace creates a workspace for the user and project provided.
@@ -169,18 +173,6 @@ func CreateWorkspace(t *testing.T, client *codersdk.Client, user string, project
169173
return workspace
170174
}
171175

172-
// AwaitWorkspaceHistoryProvisioned awaits for the workspace provision job to reach completed status.
173-
func AwaitWorkspaceHistoryProvisioned(t *testing.T, client *codersdk.Client, user, workspace, history string) coderd.WorkspaceHistory {
174-
var workspaceHistory coderd.WorkspaceHistory
175-
require.Eventually(t, func() bool {
176-
var err error
177-
workspaceHistory, err = client.WorkspaceHistory(context.Background(), user, workspace, history)
178-
require.NoError(t, err)
179-
return workspaceHistory.Provision.Status.Completed()
180-
}, 3*time.Second, 25*time.Millisecond)
181-
return workspaceHistory
182-
}
183-
184176
func randomUsername() string {
185177
return strings.ReplaceAll(namesgenerator.GetRandomName(0), "_", "-")
186178
}

‎coderd/coderdtest/coderdtest_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,15 @@ func TestNew(t *testing.T) {
2222
client := coderdtest.New(t)
2323
user := coderdtest.CreateInitialUser(t, client)
2424
closer := coderdtest.NewProvisionerDaemon(t, client)
25-
project := coderdtest.CreateProject(t, client, user.Organization)
26-
version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
27-
coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
25+
job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
26+
coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
27+
project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
2828
workspace := coderdtest.CreateWorkspace(t, client, "me", project.ID)
2929
history, err := client.CreateWorkspaceHistory(context.Background(), "me", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
30-
ProjectVersionID: version.ID,
30+
ProjectVersionID: project.ActiveVersionID,
3131
Transition: database.WorkspaceTransitionStart,
3232
})
3333
require.NoError(t, err)
34-
coderdtest.AwaitWorkspaceHistoryProvisioned(t, client, "me", workspace.Name, history.Name)
34+
coderdtest.AwaitProvisionerJob(t, client, user.Organization, history.ProvisionJobID)
3535
closer.Close()
3636
}

‎coderd/files.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package coderd
2+
3+
import (
4+
"crypto/sha256"
5+
"encoding/hex"
6+
"fmt"
7+
"io"
8+
"net/http"
9+
10+
"github.com/go-chi/render"
11+
12+
"github.com/coder/coder/database"
13+
"github.com/coder/coder/httpapi"
14+
"github.com/coder/coder/httpmw"
15+
)
16+
17+
type UploadFileResponse struct {
18+
Hash string `json:"hash"`
19+
}
20+
21+
func (api *api) postFiles(rw http.ResponseWriter, r *http.Request) {
22+
apiKey := httpmw.APIKey(r)
23+
contentType := r.Header.Get("Content-Type")
24+
25+
switch contentType {
26+
case "application/x-tar":
27+
default:
28+
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
29+
Message: fmt.Sprintf("unsupported content type: %s", contentType),
30+
})
31+
return
32+
}
33+
34+
r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
35+
data, err := io.ReadAll(r.Body)
36+
if err != nil {
37+
httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{
38+
Message: fmt.Sprintf("read file: %s", err),
39+
})
40+
return
41+
}
42+
hashBytes := sha256.Sum256(data)
43+
file, err := api.Database.InsertFile(r.Context(), database.InsertFileParams{
44+
Hash: hex.EncodeToString(hashBytes[:]),
45+
CreatedBy: apiKey.UserID,
46+
CreatedAt: database.Now(),
47+
Mimetype: contentType,
48+
Data: data,
49+
})
50+
if err != nil {
51+
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
52+
Message: fmt.Sprintf("insert file: %s", err),
53+
})
54+
return
55+
}
56+
render.Status(r, http.StatusCreated)
57+
render.JSON(rw, r, UploadFileResponse{
58+
Hash: file.Hash,
59+
})
60+
}

‎coderd/files_test.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package coderd_test
2+
3+
import (
4+
"context"
5+
"testing"
6+
7+
"github.com/stretchr/testify/require"
8+
9+
"github.com/coder/coder/coderd/coderdtest"
10+
"github.com/coder/coder/codersdk"
11+
)
12+
13+
func TestPostFiles(t *testing.T) {
14+
t.Parallel()
15+
t.Run("BadContentType", func(t *testing.T) {
16+
t.Parallel()
17+
client := coderdtest.New(t)
18+
_ = coderdtest.CreateInitialUser(t, client)
19+
_, err := client.UploadFile(context.Background(), "bad", []byte{'a'})
20+
require.Error(t, err)
21+
})
22+
23+
t.Run("Insert", func(t *testing.T) {
24+
t.Parallel()
25+
client := coderdtest.New(t)
26+
_ = coderdtest.CreateInitialUser(t, client)
27+
_, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, make([]byte, 1024))
28+
require.NoError(t, err)
29+
})
30+
}

0 commit comments

Comments
 (0)