feat: Allow running standalone provisioner daemons #3563
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package cli | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "golang.org/x/xerrors" | ||
|
|
||
| "github.com/coder/coder/codersdk" | ||
| ) | ||
|
|
||
| func provisionerCreate() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "create [name]", | ||
| Short: "Create a provisioner daemon instance", | ||
| Args: cobra.ExactArgs(1), | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| client, err := CreateClient(cmd) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| provisionerName := args[0] | ||
|
|
||
| provisionerDaemon, err := client.CreateProvisionerDaemon(cmd.Context(), codersdk.CreateProvisionerDaemonRequest{ | ||
| Name: provisionerName, | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if provisionerDaemon.AuthToken == nil { | ||
| return xerrors.New("provisioner daemon was created without an auth token") | ||
| } | ||
| tokenArg := provisionerDaemon.AuthToken.String() | ||
|
|
||
| _, _ = fmt.Fprintln(cmd.OutOrStderr(), `A new provisioner daemon has been registered. | ||
|
|
||
| Start the provisioner daemon with the following command: | ||
|
|
||
| coder provisioners run --token `+tokenArg) | ||
|
|
||
| return nil | ||
| }, | ||
| } | ||
| return cmd | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| package cli_test | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "context" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/google/uuid" | ||
| "github.com/stretchr/testify/require" | ||
|
|
||
| "github.com/coder/coder/cli/clitest" | ||
| "github.com/coder/coder/coderd/coderdtest" | ||
| ) | ||
|
|
||
| func TestProvisionerCreate(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| t.Run("OK", func(t *testing.T) { | ||
| t.Parallel() | ||
| client := coderdtest.New(t, nil) | ||
| _ = coderdtest.CreateFirstUser(t, client) | ||
| cmd, root := clitest.New(t, "provisioners", "create", "foobar") | ||
| clitest.SetupConfig(t, client, root) | ||
| buf := new(bytes.Buffer) | ||
| cmd.SetOut(buf) | ||
| err := cmd.Execute() | ||
| require.NoError(t, err) | ||
|
|
||
| var token *uuid.UUID | ||
| const tokenPrefix = "coder provisioners run --token " | ||
| s := bufio.NewScanner(buf) | ||
| for s.Scan() { | ||
| line := s.Text() | ||
| if strings.HasPrefix(line, tokenPrefix) { | ||
| tokenString := strings.TrimPrefix(line, tokenPrefix) | ||
| parsedToken, err := uuid.Parse(tokenString) | ||
| require.NoError(t, err, "provisioner token has invalid format") | ||
| token = &parsedToken | ||
| } | ||
| } | ||
| require.NotNil(t, token, "provisioner token not generated in output") | ||
|
|
||
| provisioners, err := client.ProvisionerDaemons(context.Background()) | ||
| require.NoError(t, err) | ||
| tokensByName := make(map[string]*uuid.UUID) | ||
| for _, p := range provisioners { | ||
| tokensByName[p.Name] = p.AuthToken | ||
| } | ||
| require.Equal(t, token, tokensByName["foobar"]) | ||
| }) | ||
|
|
||
| t.Run("Unprivileged", func(t *testing.T) { | ||
| t.Parallel() | ||
| adminClient := coderdtest.New(t, nil) | ||
| admin := coderdtest.CreateFirstUser(t, adminClient) | ||
| otherClient := coderdtest.CreateAnotherUser(t, adminClient, admin.OrganizationID) | ||
| cmd, root := clitest.New(t, "provisioners", "create", "foobar") | ||
| clitest.SetupConfig(t, otherClient, root) | ||
| err := cmd.Execute() | ||
| require.Error(t, err, "unprivileged user was allowed to create provisioner") | ||
| }) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package cli | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "os/signal" | ||
| "path/filepath" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "golang.org/x/xerrors" | ||
|
|
||
| "cdr.dev/slog" | ||
| "cdr.dev/slog/sloggers/sloghuman" | ||
| "github.com/coder/coder/cli/cliflag" | ||
| "github.com/coder/coder/cli/cliui" | ||
| ) | ||
|
|
||
| func provisionerRun() *cobra.Command { | ||
| var ( | ||
| cacheDir string | ||
| verbose bool | ||
| useEchoProvisioner bool | ||
| ) | ||
| cmd := &cobra.Command{ | ||
| Use: "run", | ||
| Short: "Run a standalone Coder provisioner", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| ctx := cmd.Context() | ||
| notifyCtx, notifyStop := signal.NotifyContext(ctx, interruptSignals...) | ||
| defer notifyStop() | ||
|
|
||
| logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) | ||
| if verbose { | ||
| logger = logger.Leveled(slog.LevelDebug) | ||
| } | ||
|
|
||
| client, err := CreateClient(cmd) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| errCh := make(chan error, 1) | ||
| provisionerDaemon, err := newProvisionerDaemon(ctx, client.ListenProvisionerDaemon, logger, cacheDir, errCh, useEchoProvisioner) | ||
|
There was a problem hiding this comment. I'm not saying it has to happen here, but since the provisioner has a name, consider using This will allow multiple provisioners to run on the same machine without potentially breaking |
||
| if err != nil { | ||
| return xerrors.Errorf("create provisioner daemon: %w", err) | ||
| } | ||
|
|
||
| var exitErr error | ||
| select { | ||
| case <-notifyCtx.Done(): | ||
| exitErr = notifyCtx.Err() | ||
| _, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Bold.Render( | ||
| "Interrupt caught, gracefully exiting. Use ctrl+\\ to force quit", | ||
| )) | ||
| case exitErr = <-errCh: | ||
| } | ||
|
|
||
| err = provisionerDaemon.Close() | ||
| if err != nil { | ||
| cmd.PrintErrf("Close provisioner daemon: %s\n", err) | ||
| return err | ||
| } | ||
|
|
||
| return exitErr | ||
| }, | ||
| } | ||
| defaultCacheDir := filepath.Join(os.TempDir(), "coder-cache") | ||
|
There was a problem hiding this comment. Suggestion (optional): We could consider fixing #2534 (partially) here too? Something like adrg/xdg and using Then again, perhaps we should do a more thorough fix all at once, (i.e. respect XDG elsewhere too, like config). There was a problem hiding this comment. Great suggestion, but I think this changeset is already kind of big and sprawling as it is (it fixes four separate issues) so I would lean toward doing that in a separate PR. |
||
| if dir := os.Getenv("CACHE_DIRECTORY"); dir != "" { | ||
| // For compatibility with systemd. | ||
| defaultCacheDir = dir | ||
| } | ||
| cliflag.StringVarP(cmd.Flags(), &cacheDir, "cache-dir", "", "CODER_CACHE_DIRECTORY", defaultCacheDir, "Specifies a directory to cache binaries for provision operations. If unspecified and $CACHE_DIRECTORY is set, it will be used for compatibility with systemd.") | ||
| cliflag.BoolVarP(cmd.Flags(), &verbose, "verbose", "v", "CODER_VERBOSE", false, "Enables verbose logging.") | ||
| // flags for testing only | ||
| cmd.Flags().BoolVarP(&useEchoProvisioner, "test.use-echo-provisioner", "", false, "Enable the echo provisioner") | ||
| _ = cmd.Flags().MarkHidden("test.use-echo-provisioner") | ||
| return cmd | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,103 @@ | ||||||
| package cli_test | ||||||
|
|
||||||
| import ( | ||||||
| "context" | ||||||
| "testing" | ||||||
|
|
||||||
| "github.com/stretchr/testify/require" | ||||||
|
|
||||||
| "github.com/coder/coder/cli/clitest" | ||||||
| "github.com/coder/coder/coderd/coderdtest" | ||||||
| "github.com/coder/coder/coderd/database" | ||||||
| "github.com/coder/coder/codersdk" | ||||||
| "github.com/coder/coder/provisioner/echo" | ||||||
| "github.com/coder/coder/pty/ptytest" | ||||||
| ) | ||||||
|
|
||||||
| func TestProvisionerRun(t *testing.T) { | ||||||
| t.Parallel() | ||||||
| t.Run("Provisioner", func(t *testing.T) { | ||||||
| t.Parallel() | ||||||
| client := coderdtest.New(t, nil) | ||||||
| _ = coderdtest.CreateFirstUser(t, client) | ||||||
| provisionerResponse, err := client.CreateProvisionerDaemon(context.Background(), | ||||||
| codersdk.CreateProvisionerDaemonRequest{ | ||||||
| Name: "foobar", | ||||||
| }, | ||||||
| ) | ||||||
| require.NoError(t, err) | ||||||
| token := provisionerResponse.AuthToken | ||||||
| require.NotNil(t, token) | ||||||
|
|
||||||
| doneCh := make(chan error) | ||||||
| defer func() { | ||||||
| err := <-doneCh | ||||||
| require.ErrorIs(t, err, context.Canceled, "provisioner command terminated with error") | ||||||
| }() | ||||||
|
|
||||||
| ctx, cancelFunc := context.WithCancel(context.Background()) | ||||||
|
There was a problem hiding this comment. Consider allowing tests to timeout individually, e.g.:
Suggested change
|
||||||
| defer cancelFunc() | ||||||
|
|
||||||
| cmd, root := clitest.New(t, "provisioners", "run", | ||||||
| "--token", token.String(), | ||||||
| "--verbose", // to test debug-level logs | ||||||
| "--test.use-echo-provisioner", | ||||||
| ) | ||||||
| pty := ptytest.New(t) | ||||||
| defer pty.Close() | ||||||
| cmd.SetErr(pty.Output()) | ||||||
| // command should only have access to provisioner auth token, not user credentials | ||||||
| err = root.URL().Write(client.URL.String()) | ||||||
| require.NoError(t, err) | ||||||
|
|
||||||
| go func() { | ||||||
| defer close(doneCh) | ||||||
| doneCh <- cmd.ExecuteContext(ctx) | ||||||
| }() | ||||||
|
|
||||||
| pty.ExpectMatch("\tprovisioner client connected") | ||||||
|
|
||||||
| source := clitest.CreateTemplateVersionSource(t, &echo.Responses{ | ||||||
| Parse: echo.ParseComplete, | ||||||
| Provision: provisionCompleteWithAgent, | ||||||
| }) | ||||||
| args := []string{ | ||||||
| "templates", | ||||||
| "create", | ||||||
| "my-template", | ||||||
| "--directory", source, | ||||||
| "--test.provisioner", string(database.ProvisionerTypeEcho), | ||||||
| "--max-ttl", "24h", | ||||||
| "--min-autostart-interval", "2h", | ||||||
| } | ||||||
| createCmd, root := clitest.New(t, args...) | ||||||
| clitest.SetupConfig(t, client, root) | ||||||
| pty = ptytest.New(t) | ||||||
| defer pty.Close() | ||||||
| createCmd.SetIn(pty.Input()) | ||||||
| createCmd.SetOut(pty.Output()) | ||||||
|
|
||||||
| execDone := make(chan error) | ||||||
| go func() { | ||||||
| execDone <- createCmd.Execute() | ||||||
| }() | ||||||
|
|
||||||
| matches := []struct { | ||||||
| match string | ||||||
| write string | ||||||
| }{ | ||||||
| {match: "Create and upload", write: "yes"}, | ||||||
| {match: "compute.main"}, | ||||||
| {match: "smith (linux, i386)"}, | ||||||
| {match: "Confirm create?", write: "yes"}, | ||||||
| } | ||||||
| for _, m := range matches { | ||||||
| pty.ExpectMatch(m.match) | ||||||
| if len(m.write) > 0 { | ||||||
| pty.WriteLine(m.write) | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| require.NoError(t, <-execDone) | ||||||
| }) | ||||||
| } | ||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package cli | ||
|
|
||
| import ( | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| func provisioners() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "provisioners", | ||
| Short: "Create, manage and run standalone provisioner daemons", | ||
| Aliases: []string{"provisioner"}, | ||
| } | ||
| cmd.AddCommand( | ||
| provisionerRun(), | ||
| provisionerCreate(), | ||
| ) | ||
|
|
||
| return cmd | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Kind of related to my other question about nullability of the
auth_tokenfield, but why would this ever be allowed to happen duringcreate?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah, it shouldn't be possible. A null
auth_tokenwould indicate that the provisioner was incorrectly registered as "in-process" rather than "out-of-process". But if that does somehow happen, this is just a sanity check so that we generate a meaningful error message rather than panicking.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could we make it so the API errors instead (and doesn't return a nullable UUID)? (I think that would be nicer for consumers in general.)