From acff47a999d7eb99b5a9f8353368aa7c844e1837 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 12:30:45 +0100 Subject: [PATCH 01/18] add SSH_PRIVATE_KEY_PATH option, add some tests --- git_test.go | 86 ++++++++++++++++++++++++++++++++ go.mod | 4 +- options.go | 7 +++ testutil/gittest/gittest.go | 99 +++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) diff --git a/git_test.go b/git_test.go index 0d034728..f4a9827a 100644 --- a/git_test.go +++ b/git_test.go @@ -2,6 +2,7 @@ package envbuilder_test import ( "context" + "crypto/ed25519" "fmt" "io" "net/http/httptest" @@ -14,8 +15,11 @@ import ( "github.com/coder/envbuilder/testutil/gittest" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" + "github.com/go-git/go-billy/v5/osfs" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/stretchr/testify/require" + gossh "golang.org/x/crypto/ssh" ) func TestCloneRepo(t *testing.T) { @@ -159,6 +163,78 @@ func TestCloneRepo(t *testing.T) { } } +func TestCloneRepoSSH(t *testing.T) { + t.Parallel() + + t.Run("PrivateKeyOK", func(t *testing.T) { + t.Parallel() + t.Skip("TODO: need to figure out how to properly add advertised refs") + // TODO: Can't we use a memfs here? + tmpDir := t.TempDir() + srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) + + signer := randKeygen(t) + _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + gitURL := tr.String() + clientFS := memfs.New() + + cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + Path: "/workspace", + RepoURL: gitURL, + Storage: clientFS, + RepoAuth: &gitssh.PublicKeys{ + User: "", + Signer: signer, + HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ + HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts + }, + }, + }) + require.NoError(t, err) // TODO: error: repository not found + require.True(t, cloned) + + readme := mustRead(t, clientFS, "/workspace/README.md") + require.Equal(t, "Hello, world!", readme) + gitConfig := mustRead(t, clientFS, "/workspace/.git/config") + // Ensure we do not modify the git URL that folks pass in. + require.Regexp(t, fmt.Sprintf(`(?m)^\s+url\s+=\s+%s\s*$`, regexp.QuoteMeta(gitURL)), gitConfig) + }) + + t.Run("PrivateKeyError", func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() + srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) + + signer := randKeygen(t) + anotherSigner := randKeygen(t) + _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + gitURL := tr.String() + clientFS := memfs.New() + + cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + Path: "/workspace", + RepoURL: gitURL, + Storage: clientFS, + RepoAuth: &gitssh.PublicKeys{ + User: "", + Signer: anotherSigner, + HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ + HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts + }, + }, + }) + require.ErrorContains(t, err, "handshake failed") + require.False(t, cloned) + }) + + t.Run("PrivateKeyUnknownHost", func(t *testing.T) { + t.Parallel() + t.Skip("TODO: add host key checking") + }) +} + func mustRead(t *testing.T, fs billy.Filesystem, path string) string { t.Helper() f, err := fs.OpenFile(path, os.O_RDONLY, 0644) @@ -167,3 +243,13 @@ func mustRead(t *testing.T, fs billy.Filesystem, path string) string { require.NoError(t, err) return string(content) } + +// generates a random ed25519 private key +func randKeygen(t *testing.T) gossh.Signer { + t.Helper() + _, key, err := ed25519.GenerateKey(nil) + require.NoError(t, err) + signer, err := gossh.NewSignerFromKey(key) + require.NoError(t, err) + return signer +} diff --git a/go.mod b/go.mod index 00b3dfc2..ee0d2b86 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/docker/cli v26.1.0+incompatible github.com/docker/docker v23.0.8+incompatible github.com/fatih/color v1.16.0 + github.com/gliderlabs/ssh v0.3.7 github.com/go-git/go-billy/v5 v5.5.0 github.com/go-git/go-git/v5 v5.12.0 github.com/google/go-containerregistry v0.15.2 @@ -36,6 +37,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.9.0 github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a + golang.org/x/crypto v0.21.0 golang.org/x/sync v0.7.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) @@ -70,6 +72,7 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/akutz/memconn v0.1.0 // indirect github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 // indirect + github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/aws/aws-sdk-go-v2 v1.20.3 // indirect @@ -261,7 +264,6 @@ require ( go4.org/mem v0.0.0-20220726221520-4f986261bf13 // indirect go4.org/netipx v0.0.0-20230728180743-ad4cb58a6516 // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect - golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/mod v0.15.0 // indirect golang.org/x/net v0.23.0 // indirect diff --git a/options.go b/options.go index 807f3f18..027c8020 100644 --- a/options.go +++ b/options.go @@ -37,6 +37,7 @@ type Options struct { GitCloneSingleBranch bool GitUsername string GitPassword string + GitSSHPrivateKeyPath string GitHTTPProxyURL string WorkspaceFolder string SSLCertBase64 string @@ -242,6 +243,12 @@ func (o *Options) CLI() serpent.OptionSet { Value: serpent.StringOf(&o.GitPassword), Description: "The password to use for Git authentication. This is optional.", }, + { + Flag: "git-ssh-private-key-path", + Env: "GIT_SSH_PRIVATE_KEY_PATH", + Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), + Description: "Path to a SSH private key to be used for Git authentication.", + }, { Flag: "git-http-proxy-url", Env: "GIT_HTTP_PROXY_URL", diff --git a/testutil/gittest/gittest.go b/testutil/gittest/gittest.go index 28629fee..95805f6c 100644 --- a/testutil/gittest/gittest.go +++ b/testutil/gittest/gittest.go @@ -1,12 +1,20 @@ package gittest import ( + "fmt" + "io" "log" + "net" "net/http" "os" + "os/exec" + "sync" "testing" "time" + gossh "golang.org/x/crypto/ssh" + + "github.com/gliderlabs/ssh" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -97,6 +105,97 @@ func NewServer(fs billy.Filesystem) http.Handler { return mux } +func NewServerSSH(t *testing.T, fs billy.Filesystem, pubkeys ...gossh.PublicKey) *transport.Endpoint { + t.Helper() + + l, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + t.Cleanup(func() { _ = l.Close() }) + + srvOpts := []ssh.Option{ + ssh.PublicKeyAuth(func(ctx ssh.Context, key ssh.PublicKey) bool { + for _, pk := range pubkeys { + if ssh.KeysEqual(pk, key) { + return true + } + } + return false + }), + } + + done := make(chan struct{}, 1) + go func() { + _ = ssh.Serve(l, handleSession, srvOpts...) + close(done) + }() + t.Cleanup(func() { + _ = l.Close() + <-done + }) + + addr, ok := l.Addr().(*net.TCPAddr) + require.True(t, ok) + tr, err := transport.NewEndpoint(fmt.Sprintf("ssh://git@%s:%d/", addr.IP, addr.Port)) + require.NoError(t, err) + return tr +} + +func handleSession(sess ssh.Session) { + c := sess.Command() + if len(c) < 1 { + _, _ = fmt.Fprintf(os.Stderr, "invalid command: %q\n", c) + } + + cmd := exec.Command(c[0], c[1:]...) + stdout, err := cmd.StdoutPipe() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "cmd stdout pipe: %s\n", err.Error()) + return + } + + stdin, err := cmd.StdinPipe() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "cmd stdin pipe: %s\n", err.Error()) + return + } + + stderr, err := cmd.StderrPipe() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "cmd stderr pipe: %s\n", err.Error()) + return + } + + err = cmd.Start() + if err != nil { + _, _ = fmt.Fprintf(os.Stderr, "start cmd: %s\n", err.Error()) + return + } + + go func() { + defer stdin.Close() + _, _ = io.Copy(stdin, sess) + }() + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + _, _ = io.Copy(sess.Stderr(), stderr) + }() + + go func() { + defer wg.Done() + _, _ = io.Copy(sess, stdout) + }() + + wg.Wait() + + if err := cmd.Wait(); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "wait cmd: %s\n", err.Error()) + } +} + // CommitFunc commits to a repo. type CommitFunc func(billy.Filesystem, *git.Repository) From 8ff5c1a92c9eecae4b844d7602dfd20f552e8ef4 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 15:52:55 +0100 Subject: [PATCH 02/18] handle known_hosts --- README.md | 2 ++ envbuilder.go | 38 +++++++++++++++++++++++ git.go | 61 ++++++++++++++++++++++++++++++++++++ options.go | 68 +++++++++++++++++++++++------------------ testdata/options.golden | 7 +++++ 5 files changed, 146 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index e9f0abcc..fa6939e6 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,8 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de | `--git-clone-single-branch` | `GIT_CLONE_SINGLE_BRANCH` | | Clone only a single branch of the Git repository. | | `--git-username` | `GIT_USERNAME` | | The username to use for Git authentication. This is optional. | | `--git-password` | `GIT_PASSWORD` | | The password to use for Git authentication. This is optional. | +| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` | | Path to a SSH private key to be used for Git authentication. | +| `--git-ssh-known-hosts-base64` | `GIT_SSH_KNOWN_HOSTS_BASE64` | | Base64-encoded content of a known hosts file. If not specified, host keys will be scanned and logged, but not checked. | | `--git-http-proxy-url` | `GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | | `--workspace-folder` | `WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | | `--ssl-cert-base64` | `SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | diff --git a/envbuilder.go b/envbuilder.go index 334450da..2bce8189 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -45,6 +45,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing/transport" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/sirupsen/logrus" @@ -156,6 +157,34 @@ func Run(ctx context.Context, options Options) error { } } + gitURLParsed, err := url.Parse(options.GitURL) + if err != nil { + return fmt.Errorf("invalid git URL: %w", err) + } + // If we're cloning over SSH, we need a known_hosts file. + if gitURLParsed.Scheme == "ssh" { + var knownHostsContent []byte + if options.GitSSHKnownHostsBase64 != "" { + if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil { + return fmt.Errorf("invalid known_hosts content: %w", err) + } else { + knownHostsContent = kh + } + } else { + kh, err := GenerateKnownHosts(options.Logger, gitURLParsed) + if err != nil { + return fmt.Errorf("invalid known_hosts content: %w", err) + } else { + knownHostsContent = kh + } + } + knownHostsPath := filepath.Join(MagicDir, "known_hosts") + if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil { + return fmt.Errorf("write known_hosts file: %w", err) + } + _ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) + } + var fallbackErr error var cloned bool if options.GitURL != "" { @@ -201,6 +230,15 @@ func Run(ctx context.Context, options Options) error { Username: options.GitUsername, Password: options.GitPassword, } + } else if options.GitSSHPrivateKeyPath != "" { + signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) + if err != nil { + return xerrors.Errorf("read private key: %w", err) + } + cloneOpts.RepoAuth = &gitssh.PublicKeys{ + User: "git", + Signer: signer, + } } if options.GitHTTPProxyURL != "" { cloneOpts.ProxyOptions = transport.ProxyOptions{ diff --git a/git.go b/git.go index 9f542add..27084117 100644 --- a/git.go +++ b/git.go @@ -1,11 +1,19 @@ package envbuilder import ( + "bytes" "context" + "encoding/base64" "errors" "fmt" + "io" + "net" "net/url" + "os" + "strconv" + "strings" + "github.com/coder/coder/v2/codersdk" "github.com/go-git/go-billy/v5" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -14,6 +22,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/filesystem" + gossh "golang.org/x/crypto/ssh" ) type CloneRepoOptions struct { @@ -113,3 +122,55 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { } return true, nil } + +func ReadPrivateKey(path string) (gossh.Signer, error) { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open private key file: %w", err) + } + var buf bytes.Buffer + if _, err := io.Copy(&buf, f); err != nil { + return nil, fmt.Errorf("read private key file: %w", err) + } + k, err := gossh.ParsePrivateKey(buf.Bytes()) + if err != nil { + return nil, fmt.Errorf("parse private key file: %w", err) + } + return k, nil +} + +// GenerateKnownHosts dials the server located at gitURL and fetches the SSH +// public keys returned in a format accepted by known_hosts. +func GenerateKnownHosts(log LoggerFunc, gitURL *url.URL) ([]byte, error) { + var buf bytes.Buffer + conf := &gossh.ClientConfig{ + // Accept and record all host keys + HostKeyCallback: func(dialAddr string, addr net.Addr, key gossh.PublicKey) error { + h := strings.Split(dialAddr, ":")[0] + k64 := base64.StdEncoding.EncodeToString(key.Marshal()) + log(codersdk.LogLevelInfo, "ssh keyscan: %s %s %s", h, key.Type(), k64) + buf.WriteString(fmt.Sprintf("%s %s %s\n", h, key.Type(), k64)) + return nil + }, + } + dialAddr := hostPort(gitURL) + client, err := gossh.Dial("tcp", dialAddr, conf) + if err != nil { + // The dial may fail due to no authentication methods, but this is fine. + if netErr, ok := err.(net.Error); ok { + return nil, fmt.Errorf("keyscan %s: %w", dialAddr, netErr) + } + // If it's not a net.Error then we will assume we were successful. + } else { + _ = client.Close() + } + return buf.Bytes(), nil +} + +func hostPort(u *url.URL) string { + p := 22 // assume default SSH port + if _p, err := strconv.Atoi(u.Port()); err == nil { + p = _p + } + return fmt.Sprintf("%s:%d", u.Host, p) +} diff --git a/options.go b/options.go index 027c8020..e098110c 100644 --- a/options.go +++ b/options.go @@ -13,36 +13,37 @@ type LoggerFunc func(level codersdk.LogLevel, format string, args ...interface{} // Options contains the configuration for the envbuilder. type Options struct { - SetupScript string - InitScript string - InitCommand string - InitArgs string - CacheRepo string - BaseImageCacheDir string - LayerCacheDir string - DevcontainerDir string - DevcontainerJSONPath string - DockerfilePath string - BuildContextPath string - CacheTTLDays int64 - DockerConfigBase64 string - FallbackImage string - ExitOnBuildFailure bool - ForceSafe bool - Insecure bool - IgnorePaths []string - SkipRebuild bool - GitURL string - GitCloneDepth int64 - GitCloneSingleBranch bool - GitUsername string - GitPassword string - GitSSHPrivateKeyPath string - GitHTTPProxyURL string - WorkspaceFolder string - SSLCertBase64 string - ExportEnvFile string - PostStartScriptPath string + SetupScript string + InitScript string + InitCommand string + InitArgs string + CacheRepo string + BaseImageCacheDir string + LayerCacheDir string + DevcontainerDir string + DevcontainerJSONPath string + DockerfilePath string + BuildContextPath string + CacheTTLDays int64 + DockerConfigBase64 string + FallbackImage string + ExitOnBuildFailure bool + ForceSafe bool + Insecure bool + IgnorePaths []string + SkipRebuild bool + GitURL string + GitCloneDepth int64 + GitCloneSingleBranch bool + GitUsername string + GitPassword string + GitSSHPrivateKeyPath string + GitSSHKnownHostsBase64 string + GitHTTPProxyURL string + WorkspaceFolder string + SSLCertBase64 string + ExportEnvFile string + PostStartScriptPath string // Logger is the logger to use for all operations. Logger LoggerFunc // Filesystem is the filesystem to use for all operations. @@ -249,6 +250,13 @@ func (o *Options) CLI() serpent.OptionSet { Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), Description: "Path to a SSH private key to be used for Git authentication.", }, + { + Flag: "git-ssh-known-hosts-base64", + Env: "GIT_SSH_KNOWN_HOSTS_BASE64", + Value: serpent.StringOf(&o.GitSSHKnownHostsBase64), + Description: "Base64-encoded content of a known hosts file. If not specified, " + + "host keys will be scanned and logged, but not checked.", + }, { Flag: "git-http-proxy-url", Env: "GIT_HTTP_PROXY_URL", diff --git a/testdata/options.golden b/testdata/options.golden index ac814cf0..2dfae67f 100644 --- a/testdata/options.golden +++ b/testdata/options.golden @@ -90,6 +90,13 @@ OPTIONS: --git-password string, $GIT_PASSWORD The password to use for Git authentication. This is optional. + --git-ssh-known-hosts-base64 string, $GIT_SSH_KNOWN_HOSTS_BASE64 + Base64-encoded content of a known hosts file. If not specified, host + keys will be scanned and logged, but not checked. + + --git-ssh-private-key-path string, $GIT_SSH_PRIVATE_KEY_PATH + Path to a SSH private key to be used for Git authentication. + --git-url string, $GIT_URL The URL of the Git repository to clone. This is optional. From 015c67aa59597fbbbea327c2a626c0df83495c06 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 15:11:13 +0000 Subject: [PATCH 03/18] add test for unknown host key --- git_test.go | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/git_test.go b/git_test.go index f4a9827a..7728b374 100644 --- a/git_test.go +++ b/git_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "regexp" "testing" @@ -164,10 +165,9 @@ func TestCloneRepo(t *testing.T) { } func TestCloneRepoSSH(t *testing.T) { - t.Parallel() + // nolint: paralleltest // t.Setenv t.Run("PrivateKeyOK", func(t *testing.T) { - t.Parallel() t.Skip("TODO: need to figure out how to properly add advertised refs") // TODO: Can't we use a memfs here? tmpDir := t.TempDir() @@ -201,8 +201,8 @@ func TestCloneRepoSSH(t *testing.T) { require.Regexp(t, fmt.Sprintf(`(?m)^\s+url\s+=\s+%s\s*$`, regexp.QuoteMeta(gitURL)), gitConfig) }) + // nolint: paralleltest // t.Setenv t.Run("PrivateKeyError", func(t *testing.T) { - t.Parallel() tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) @@ -229,9 +229,33 @@ func TestCloneRepoSSH(t *testing.T) { require.False(t, cloned) }) - t.Run("PrivateKeyUnknownHost", func(t *testing.T) { - t.Parallel() - t.Skip("TODO: add host key checking") + // nolint: paralleltest // t.Setenv + t.Run("PrivateKeyHostKeyUnknown", func(t *testing.T) { + tmpDir := t.TempDir() + srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) + + knownHostsPath := filepath.Join(tmpDir, "known_hosts") + require.NoError(t, os.WriteFile(knownHostsPath, []byte{}, 0o600)) + t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) + + signer := randKeygen(t) + anotherSigner := randKeygen(t) + _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) + tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + gitURL := tr.String() + clientFS := memfs.New() + + cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ + Path: "/workspace", + RepoURL: gitURL, + Storage: clientFS, + RepoAuth: &gitssh.PublicKeys{ + User: "", + Signer: anotherSigner, + }, + }) + require.ErrorContains(t, err, "key is unknown") + require.False(t, cloned) }) } From 054abc8a5f22f7bdfe638d93297586f91bcb7fed Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 15:45:23 +0000 Subject: [PATCH 04/18] fixup! add test for unknown host key --- git.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git.go b/git.go index 27084117..4ea35f70 100644 --- a/git.go +++ b/git.go @@ -123,6 +123,8 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) { return true, nil } +// ReadPrivateKey attempts to read an SSH private key from path +// and returns an ssh.Signer. func ReadPrivateKey(path string) (gossh.Signer, error) { f, err := os.Open(path) if err != nil { From 30285c090b554504c45d7ec42c23bddca9d3e61f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:11:26 +0000 Subject: [PATCH 05/18] improve unit tests --- git_test.go | 67 ++++++++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/git_test.go b/git_test.go index 7728b374..e683a51c 100644 --- a/git_test.go +++ b/git_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "net/url" "os" - "path/filepath" "regexp" "testing" @@ -165,17 +164,18 @@ func TestCloneRepo(t *testing.T) { } func TestCloneRepoSSH(t *testing.T) { + t.Parallel() - // nolint: paralleltest // t.Setenv - t.Run("PrivateKeyOK", func(t *testing.T) { - t.Skip("TODO: need to figure out how to properly add advertised refs") - // TODO: Can't we use a memfs here? + t.Run("AuthSuccess", func(t *testing.T) { + t.Parallel() + + // TODO: test the rest of the cloning flow. This just tests successful auth. tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - signer := randKeygen(t) _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) - tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + key := randKeygen(t) + tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() clientFS := memfs.New() @@ -185,43 +185,42 @@ func TestCloneRepoSSH(t *testing.T) { Storage: clientFS, RepoAuth: &gitssh.PublicKeys{ User: "", - Signer: signer, + Signer: key, HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ - HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts + // Not testing host keys here. + HostKeyCallback: gossh.InsecureIgnoreHostKey(), }, }, }) - require.NoError(t, err) // TODO: error: repository not found - require.True(t, cloned) - - readme := mustRead(t, clientFS, "/workspace/README.md") - require.Equal(t, "Hello, world!", readme) - gitConfig := mustRead(t, clientFS, "/workspace/.git/config") - // Ensure we do not modify the git URL that folks pass in. - require.Regexp(t, fmt.Sprintf(`(?m)^\s+url\s+=\s+%s\s*$`, regexp.QuoteMeta(gitURL)), gitConfig) + // TODO: ideally, we want to test the entire cloning flow. + // For now, this indicates successful ssh key auth. + require.ErrorContains(t, err, "repository not found") + require.False(t, cloned) }) - // nolint: paralleltest // t.Setenv - t.Run("PrivateKeyError", func(t *testing.T) { + t.Run("AuthFailure", func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - signer := randKeygen(t) - anotherSigner := randKeygen(t) _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) - tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + key := randKeygen(t) + tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() clientFS := memfs.New() + anotherKey := randKeygen(t) cloned, err := envbuilder.CloneRepo(context.Background(), envbuilder.CloneRepoOptions{ Path: "/workspace", RepoURL: gitURL, Storage: clientFS, RepoAuth: &gitssh.PublicKeys{ User: "", - Signer: anotherSigner, + Signer: anotherKey, HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ - HostKeyCallback: gossh.InsecureIgnoreHostKey(), // TODO: known_hosts + // Not testing host keys here. + HostKeyCallback: gossh.InsecureIgnoreHostKey(), }, }, }) @@ -230,18 +229,15 @@ func TestCloneRepoSSH(t *testing.T) { }) // nolint: paralleltest // t.Setenv - t.Run("PrivateKeyHostKeyUnknown", func(t *testing.T) { + t.Run("PrivateKeyHostKeyMismatch", func(t *testing.T) { + t.Parallel() + tmpDir := t.TempDir() srvFS := osfs.New(tmpDir, osfs.WithChrootOS()) - knownHostsPath := filepath.Join(tmpDir, "known_hosts") - require.NoError(t, os.WriteFile(knownHostsPath, []byte{}, 0o600)) - t.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) - - signer := randKeygen(t) - anotherSigner := randKeygen(t) _ = gittest.NewRepo(t, srvFS, gittest.Commit(t, "README.md", "Hello, world!", "Wow!")) - tr := gittest.NewServerSSH(t, srvFS, signer.PublicKey()) + key := randKeygen(t) + tr := gittest.NewServerSSH(t, srvFS, key.PublicKey()) gitURL := tr.String() clientFS := memfs.New() @@ -251,10 +247,13 @@ func TestCloneRepoSSH(t *testing.T) { Storage: clientFS, RepoAuth: &gitssh.PublicKeys{ User: "", - Signer: anotherSigner, + Signer: key, + HostKeyCallbackHelper: gitssh.HostKeyCallbackHelper{ + HostKeyCallback: gossh.FixedHostKey(randKeygen(t).PublicKey()), + }, }, }) - require.ErrorContains(t, err, "key is unknown") + require.ErrorContains(t, err, "ssh: host key mismatch") require.False(t, cloned) }) } From 64bdbdc64795617a67fb1e39d4d74a484cda701c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:11:48 +0000 Subject: [PATCH 06/18] refactor keyscan --- envbuilder.go | 2 +- git.go | 45 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 2bce8189..6ab8a397 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -171,7 +171,7 @@ func Run(ctx context.Context, options Options) error { knownHostsContent = kh } } else { - kh, err := GenerateKnownHosts(options.Logger, gitURLParsed) + kh, err := KeyScan(options.Logger, gitURLParsed) if err != nil { return fmt.Errorf("invalid known_hosts content: %w", err) } else { diff --git a/git.go b/git.go index 4ea35f70..b90a370c 100644 --- a/git.go +++ b/git.go @@ -141,17 +141,21 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { return k, nil } -// GenerateKnownHosts dials the server located at gitURL and fetches the SSH +// KeyScan dials the server located at gitURL and fetches the SSH // public keys returned in a format accepted by known_hosts. -func GenerateKnownHosts(log LoggerFunc, gitURL *url.URL) ([]byte, error) { +// If no host keys found, returns an error. +func KeyScan(log LoggerFunc, gitURL *url.URL) ([]byte, error) { var buf bytes.Buffer conf := &gossh.ClientConfig{ // Accept and record all host keys - HostKeyCallback: func(dialAddr string, addr net.Addr, key gossh.PublicKey) error { - h := strings.Split(dialAddr, ":")[0] - k64 := base64.StdEncoding.EncodeToString(key.Marshal()) - log(codersdk.LogLevelInfo, "ssh keyscan: %s %s %s", h, key.Type(), k64) - buf.WriteString(fmt.Sprintf("%s %s %s\n", h, key.Type(), k64)) + HostKeyCallback: func(dialAddr string, _ net.Addr, key gossh.PublicKey) error { + kh, err := KnownHostsLine(dialAddr, key) + if err != nil { + return fmt.Errorf("ssh keyscan: generate known hosts line: %w", err) + } + log(codersdk.LogLevelInfo, "ssh keyscan: %s", kh) + buf.WriteString(kh) + buf.WriteString("\n") return nil }, } @@ -160,15 +164,34 @@ func GenerateKnownHosts(log LoggerFunc, gitURL *url.URL) ([]byte, error) { if err != nil { // The dial may fail due to no authentication methods, but this is fine. if netErr, ok := err.(net.Error); ok { - return nil, fmt.Errorf("keyscan %s: %w", dialAddr, netErr) + return nil, fmt.Errorf("ssh keyscan: dial %s: %w", dialAddr, netErr) } - // If it's not a net.Error then we will assume we were successful. - } else { - _ = client.Close() + // Otherwise, assume success. + } + defer func() { + if client != nil { + _ = client.Close() + } + }() + + bs := buf.Bytes() + if len(bs) == 0 { + return nil, fmt.Errorf("ssh keyscan: found no host keys") } return buf.Bytes(), nil } +// KnownHostsLine generates a corresponding line for known_hosts +// given a dial address in the format host:port and a public key. +func KnownHostsLine(dialAddr string, key gossh.PublicKey) (string, error) { + if !strings.Contains(dialAddr, ":") { + return "", fmt.Errorf("invalid dialAddr, expected host:port") + } + h := strings.Split(dialAddr, ":")[0] + k64 := base64.StdEncoding.EncodeToString(key.Marshal()) + return fmt.Sprintf("%s %s %s", h, key.Type(), k64), nil +} + func hostPort(u *url.URL) string { p := 22 // assume default SSH port if _p, err := strconv.Atoi(u.Port()); err == nil { From e9f183fb48eb664498c0b8f5630690404ae93549 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:28:13 +0000 Subject: [PATCH 07/18] prepend ssh:// to urls missing schema --- envbuilder.go | 3 ++- git.go | 13 +++++++++++++ git_test.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/envbuilder.go b/envbuilder.go index 6ab8a397..81010a50 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -157,7 +157,7 @@ func Run(ctx context.Context, options Options) error { } } - gitURLParsed, err := url.Parse(options.GitURL) + gitURLParsed, err := ParseGitURL(options.GitURL) if err != nil { return fmt.Errorf("invalid git URL: %w", err) } @@ -182,6 +182,7 @@ func Run(ctx context.Context, options Options) error { if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil { return fmt.Errorf("write known_hosts file: %w", err) } + // go-git will read this file to validate the server host keys. _ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) } diff --git a/git.go b/git.go index b90a370c..7814b903 100644 --- a/git.go +++ b/git.go @@ -10,6 +10,7 @@ import ( "net" "net/url" "os" + "regexp" "strconv" "strings" @@ -199,3 +200,15 @@ func hostPort(u *url.URL) string { } return fmt.Sprintf("%s:%d", u.Host, p) } + +var schemaRe = regexp.MustCompile(`^[a-zA-Z]+://`) + +// ParseGitURL will normalize a git URL without a leading schema. +// If no schema is provided, we will default to ssh://. +func ParseGitURL(gitURL string) (*url.URL, error) { + if !schemaRe.MatchString(gitURL) { + gitURL = "ssh://" + gitURL + + } + return url.Parse(gitURL) +} diff --git a/git_test.go b/git_test.go index e683a51c..4d9b2291 100644 --- a/git_test.go +++ b/git_test.go @@ -18,6 +18,7 @@ import ( "github.com/go-git/go-billy/v5/osfs" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gossh "golang.org/x/crypto/ssh" ) @@ -258,6 +259,41 @@ func TestCloneRepoSSH(t *testing.T) { }) } +func TestParseGitURL(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + url string + expected string + expectedError string + }{ + { + url: "https://user:pass@example.com/repo", + expected: "https://user:pass@example.com/repo", + }, + { + url: "http://user:pass@example.com/repo", + expected: "http://user:pass@example.com/repo", + }, + { + url: "ssh://git@example.com/repo", + expected: "ssh://git@example.com/repo", + }, + { + url: "git@example.com/repo", + expected: "ssh://git@example.com/repo", + }, + } { + actual, err := envbuilder.ParseGitURL(tc.url) + if tc.expectedError == "" { + assert.NoError(t, err) + assert.Equal(t, tc.expected, actual.String()) + continue + } + assert.ErrorContains(t, err, tc.expectedError) + } +} + func mustRead(t *testing.T, fs billy.Filesystem, path string) string { t.Helper() f, err := fs.OpenFile(path, os.O_RDONLY, 0644) From 0ead6a058e835342cc075d84056d28ee12e3bdf2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:29:52 +0000 Subject: [PATCH 08/18] io.ReadAll --- git.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/git.go b/git.go index 7814b903..c068b9bb 100644 --- a/git.go +++ b/git.go @@ -131,11 +131,11 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { if err != nil { return nil, fmt.Errorf("open private key file: %w", err) } - var buf bytes.Buffer - if _, err := io.Copy(&buf, f); err != nil { + bs, err := io.ReadAll(f) + if err != nil { return nil, fmt.Errorf("read private key file: %w", err) } - k, err := gossh.ParsePrivateKey(buf.Bytes()) + k, err := gossh.ParsePrivateKey(bs) if err != nil { return nil, fmt.Errorf("parse private key file: %w", err) } From 2c79a1b50ee46a237f6414c45ef401f458654101 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:31:46 +0000 Subject: [PATCH 09/18] no underscore vars --- git.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git.go b/git.go index c068b9bb..c0159a8f 100644 --- a/git.go +++ b/git.go @@ -195,8 +195,8 @@ func KnownHostsLine(dialAddr string, key gossh.PublicKey) (string, error) { func hostPort(u *url.URL) string { p := 22 // assume default SSH port - if _p, err := strconv.Atoi(u.Port()); err == nil { - p = _p + if tmp, err := strconv.Atoi(u.Port()); err == nil { + p = tmp } return fmt.Sprintf("%s:%d", u.Host, p) } From e5c39d32723da7c649bca65ca069bd6d5a669c2c Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 2 May 2024 20:42:22 +0000 Subject: [PATCH 10/18] fix tests --- envbuilder.go | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 81010a50..78c90bf3 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -157,33 +157,34 @@ func Run(ctx context.Context, options Options) error { } } - gitURLParsed, err := ParseGitURL(options.GitURL) - if err != nil { - return fmt.Errorf("invalid git URL: %w", err) - } - // If we're cloning over SSH, we need a known_hosts file. - if gitURLParsed.Scheme == "ssh" { - var knownHostsContent []byte - if options.GitSSHKnownHostsBase64 != "" { - if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil { - return fmt.Errorf("invalid known_hosts content: %w", err) + if options.GitURL != "" { + gitURLParsed, err := ParseGitURL(options.GitURL) + if err != nil { + return fmt.Errorf("invalid git URL: %w", err) + } + // If we're cloning over SSH, we need a known_hosts file. + if gitURLParsed.Scheme == "ssh" { + var knownHostsContent []byte + if options.GitSSHKnownHostsBase64 != "" { + if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil { + return fmt.Errorf("invalid known_hosts content: %w", err) + } else { + knownHostsContent = kh + } } else { - knownHostsContent = kh + // This is a best-effort. + kh, err := KeyScan(options.Logger, gitURLParsed) + if err == nil { + knownHostsContent = kh + } } - } else { - kh, err := KeyScan(options.Logger, gitURLParsed) - if err != nil { - return fmt.Errorf("invalid known_hosts content: %w", err) - } else { - knownHostsContent = kh + knownHostsPath := filepath.Join(MagicDir, "known_hosts") + if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil { + return fmt.Errorf("write known_hosts file: %w", err) } + // go-git will read this file to validate the server host keys. + _ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) } - knownHostsPath := filepath.Join(MagicDir, "known_hosts") - if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil { - return fmt.Errorf("write known_hosts file: %w", err) - } - // go-git will read this file to validate the server host keys. - _ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) } var fallbackErr error From 18d0dc2c118171db1a690c92fce640e993ec7a0d Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 08:55:33 +0100 Subject: [PATCH 11/18] use knownhosts.WriteKnownHost instead --- git.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/git.go b/git.go index c0159a8f..27e74f68 100644 --- a/git.go +++ b/git.go @@ -23,6 +23,7 @@ import ( "github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/storage/filesystem" + "github.com/skeema/knownhosts" gossh "golang.org/x/crypto/ssh" ) @@ -149,14 +150,11 @@ func KeyScan(log LoggerFunc, gitURL *url.URL) ([]byte, error) { var buf bytes.Buffer conf := &gossh.ClientConfig{ // Accept and record all host keys - HostKeyCallback: func(dialAddr string, _ net.Addr, key gossh.PublicKey) error { - kh, err := KnownHostsLine(dialAddr, key) - if err != nil { + HostKeyCallback: func(dialAddr string, remote net.Addr, key gossh.PublicKey) error { + if err := knownhosts.WriteKnownHost(&buf, dialAddr, remote, key); err != nil { return fmt.Errorf("ssh keyscan: generate known hosts line: %w", err) } - log(codersdk.LogLevelInfo, "ssh keyscan: %s", kh) - buf.WriteString(kh) - buf.WriteString("\n") + log(codersdk.LogLevelInfo, "ssh keyscan: %s", strings.TrimSpace(buf.String())) return nil }, } From b0587e2bd3a50df33cc5a18ec5072c8836f62e23 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 10:11:34 +0000 Subject: [PATCH 12/18] simply by removing GIT_SSH_KNOWN_HOSTS_BASE64, refactor repo auth setup --- envbuilder.go | 50 +----------- git.go | 139 +++++++++++++++++--------------- git_test.go | 171 ++++++++++++++++++++++++++++++++-------- options.go | 68 +++++++--------- testdata/options.golden | 4 - 5 files changed, 247 insertions(+), 185 deletions(-) diff --git a/envbuilder.go b/envbuilder.go index 78c90bf3..8ba7ac47 100644 --- a/envbuilder.go +++ b/envbuilder.go @@ -44,8 +44,6 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing/transport" - githttp "github.com/go-git/go-git/v5/plumbing/transport/http" - gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/sirupsen/logrus" @@ -157,36 +155,6 @@ func Run(ctx context.Context, options Options) error { } } - if options.GitURL != "" { - gitURLParsed, err := ParseGitURL(options.GitURL) - if err != nil { - return fmt.Errorf("invalid git URL: %w", err) - } - // If we're cloning over SSH, we need a known_hosts file. - if gitURLParsed.Scheme == "ssh" { - var knownHostsContent []byte - if options.GitSSHKnownHostsBase64 != "" { - if kh, err := base64.StdEncoding.DecodeString(options.GitSSHKnownHostsBase64); err != nil { - return fmt.Errorf("invalid known_hosts content: %w", err) - } else { - knownHostsContent = kh - } - } else { - // This is a best-effort. - kh, err := KeyScan(options.Logger, gitURLParsed) - if err == nil { - knownHostsContent = kh - } - } - knownHostsPath := filepath.Join(MagicDir, "known_hosts") - if err := os.WriteFile(knownHostsPath, knownHostsContent, 0644); err != nil { - return fmt.Errorf("write known_hosts file: %w", err) - } - // go-git will read this file to validate the server host keys. - _ = os.Setenv("SSH_KNOWN_HOSTS", knownHostsPath) - } - } - var fallbackErr error var cloned bool if options.GitURL != "" { @@ -225,23 +193,7 @@ func Run(ctx context.Context, options Options) error { CABundle: caBundle, } - if options.GitUsername != "" || options.GitPassword != "" { - // NOTE: we previously inserted the credentials into the repo URL. - // This was removed in https://github.com/coder/envbuilder/pull/141 - cloneOpts.RepoAuth = &githttp.BasicAuth{ - Username: options.GitUsername, - Password: options.GitPassword, - } - } else if options.GitSSHPrivateKeyPath != "" { - signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) - if err != nil { - return xerrors.Errorf("read private key: %w", err) - } - cloneOpts.RepoAuth = &gitssh.PublicKeys{ - User: "git", - Signer: signer, - } - } + cloneOpts.RepoAuth = SetupRepoAuth(&options) if options.GitHTTPProxyURL != "" { cloneOpts.ProxyOptions = transport.ProxyOptions{ URL: options.GitHTTPProxyURL, diff --git a/git.go b/git.go index 27e74f68..67967744 100644 --- a/git.go +++ b/git.go @@ -1,17 +1,13 @@ package envbuilder import ( - "bytes" "context" - "encoding/base64" "errors" "fmt" "io" "net" "net/url" "os" - "regexp" - "strconv" "strings" "github.com/coder/coder/v2/codersdk" @@ -22,6 +18,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/protocol/packp/capability" "github.com/go-git/go-git/v5/plumbing/protocol/packp/sideband" "github.com/go-git/go-git/v5/plumbing/transport" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" + gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage/filesystem" "github.com/skeema/knownhosts" gossh "golang.org/x/crypto/ssh" @@ -143,70 +141,85 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { return k, nil } -// KeyScan dials the server located at gitURL and fetches the SSH -// public keys returned in a format accepted by known_hosts. -// If no host keys found, returns an error. -func KeyScan(log LoggerFunc, gitURL *url.URL) ([]byte, error) { - var buf bytes.Buffer - conf := &gossh.ClientConfig{ - // Accept and record all host keys - HostKeyCallback: func(dialAddr string, remote net.Addr, key gossh.PublicKey) error { - if err := knownhosts.WriteKnownHost(&buf, dialAddr, remote, key); err != nil { - return fmt.Errorf("ssh keyscan: generate known hosts line: %w", err) - } - log(codersdk.LogLevelInfo, "ssh keyscan: %s", strings.TrimSpace(buf.String())) - return nil - }, - } - dialAddr := hostPort(gitURL) - client, err := gossh.Dial("tcp", dialAddr, conf) - if err != nil { - // The dial may fail due to no authentication methods, but this is fine. - if netErr, ok := err.(net.Error); ok { - return nil, fmt.Errorf("ssh keyscan: dial %s: %w", dialAddr, netErr) - } - // Otherwise, assume success. - } - defer func() { - if client != nil { - _ = client.Close() +// LogHostKeyCallback is a HostKeyCallback that just logs host keys +// and does nothing else. +func LogHostKeyCallback(log LoggerFunc) gossh.HostKeyCallback { + return func(hostname string, remote net.Addr, key gossh.PublicKey) error { + var sb strings.Builder + _ = knownhosts.WriteKnownHost(&sb, hostname, remote, key) + // skeema/knownhosts uses a fake public key to determine the host key + // algorithms. Ignore this one. + if s := sb.String(); !strings.Contains(s, "fake-public-key ZmFrZSBwdWJsaWMga2V5") { + log(codersdk.LogLevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s)) } - }() - - bs := buf.Bytes() - if len(bs) == 0 { - return nil, fmt.Errorf("ssh keyscan: found no host keys") + return nil } - return buf.Bytes(), nil -} - -// KnownHostsLine generates a corresponding line for known_hosts -// given a dial address in the format host:port and a public key. -func KnownHostsLine(dialAddr string, key gossh.PublicKey) (string, error) { - if !strings.Contains(dialAddr, ":") { - return "", fmt.Errorf("invalid dialAddr, expected host:port") - } - h := strings.Split(dialAddr, ":")[0] - k64 := base64.StdEncoding.EncodeToString(key.Marshal()) - return fmt.Sprintf("%s %s %s", h, key.Type(), k64), nil } -func hostPort(u *url.URL) string { - p := 22 // assume default SSH port - if tmp, err := strconv.Atoi(u.Port()); err == nil { - p = tmp +// SetupRepoAuth determines the desired AuthMethod based on options.GitURL: +// +// | Git URL format | GIT_USERNAME | GIT_PASSWORD | Auth Method | +// | ------------------------|--------------|--------------|-------------| +// | https?://host.tld/repo | Not Set | Not Set | None | +// | https?://host.tld/repo | Not Set | Set | HTTP Basic | +// | https?://host.tld/repo | Set | Not Set | HTTP Basic | +// | https?://host.tld/repo | Set | Set | HTTP Basic | +// | All other formats | - | - | SSH | +// +// For SSH authentication, the default username is "git" but will honour +// GIT_USERNAME if set. +// +// If SSH_PRIVATE_KEY_PATH is set, an SSH private key will be read from +// that path and the SSH auth method will be configured with that key. +// +// If SSH_KNOWN_HOSTS is not set, the SSH auth method will be configured +// to accept and log all host keys. Otherwise, host key checking will be +// performed as usual. +func SetupRepoAuth(options *Options) transport.AuthMethod { + if options.GitURL == "" { + options.Logger(codersdk.LogLevelInfo, "#1: ❔ No Git URL supplied!") + return nil + } + if strings.HasPrefix(options.GitURL, "http://") || strings.HasPrefix(options.GitURL, "https://") { + // Special case: no auth + if options.GitUsername == "" && options.GitPassword == "" { + options.Logger(codersdk.LogLevelInfo, "#1: 👤 Using no authentication!") + return nil + } + // Basic Auth + // NOTE: we previously inserted the credentials into the repo URL. + // This was removed in https://github.com/coder/envbuilder/pull/141 + options.Logger(codersdk.LogLevelInfo, "#1: 🔒 Using HTTP basic authentication!") + return &githttp.BasicAuth{ + Username: options.GitUsername, + Password: options.GitPassword, + } + } + // Assume SSH auth for all other formats. + auth := &gitssh.PublicKeys{} + options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using SSH authentication!") + // Generally git clones over SSH use the 'git' user, but respect + // GIT_USERNAME if set. + auth.User = options.GitUsername + if auth.User == "" { + auth.User = "git" + } + // If we are told about an SSH private key, attempt to read it. + if options.GitSSHPrivateKeyPath != "" { + signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) + if err != nil { + options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) + // go-git will still attempt to fall back to other auth methods from the + // environment. This enables usage of SSH_AUTH_SOCK, for example. + return auth + } + options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using %s key!", signer.PublicKey().Type()) + auth.Signer = signer } - return fmt.Sprintf("%s:%d", u.Host, p) -} - -var schemaRe = regexp.MustCompile(`^[a-zA-Z]+://`) - -// ParseGitURL will normalize a git URL without a leading schema. -// If no schema is provided, we will default to ssh://. -func ParseGitURL(gitURL string) (*url.URL, error) { - if !schemaRe.MatchString(gitURL) { - gitURL = "ssh://" + gitURL + if os.Getenv("SSH_KNOWN_HOSTS") == "" { + options.Logger(codersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + auth.HostKeyCallback = LogHostKeyCallback(options.Logger) } - return url.Parse(gitURL) + return auth } diff --git a/git_test.go b/git_test.go index 4d9b2291..dc0bea44 100644 --- a/git_test.go +++ b/git_test.go @@ -8,9 +8,11 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "regexp" "testing" + "github.com/coder/coder/v2/codersdk" "github.com/coder/envbuilder" "github.com/coder/envbuilder/testutil/gittest" "github.com/go-git/go-billy/v5" @@ -18,7 +20,6 @@ import ( "github.com/go-git/go-billy/v5/osfs" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" gossh "golang.org/x/crypto/ssh" ) @@ -259,39 +260,141 @@ func TestCloneRepoSSH(t *testing.T) { }) } -func TestParseGitURL(t *testing.T) { +func TestSetupRepoAuth(t *testing.T) { t.Parallel() - for _, tc := range []struct { - url string - expected string - expectedError string - }{ - { - url: "https://user:pass@example.com/repo", - expected: "https://user:pass@example.com/repo", - }, - { - url: "http://user:pass@example.com/repo", - expected: "http://user:pass@example.com/repo", - }, - { - url: "ssh://git@example.com/repo", - expected: "ssh://git@example.com/repo", - }, - { - url: "git@example.com/repo", - expected: "ssh://git@example.com/repo", - }, - } { - actual, err := envbuilder.ParseGitURL(tc.url) - if tc.expectedError == "" { - assert.NoError(t, err) - assert.Equal(t, tc.expected, actual.String()) - continue + t.Run("Empty", func(t *testing.T) { + t.Parallel() + + opts := &envbuilder.Options{ + Logger: testLog(t), } - assert.ErrorContains(t, err, tc.expectedError) - } + auth := envbuilder.SetupRepoAuth(opts) + require.Nil(t, auth) + }) + + t.Run("HTTP/NoAuth", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "http://host.tld/repo", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + require.Nil(t, auth) + }) + + t.Run("HTTP/BasicAuth", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "http://host.tld/repo", + GitUsername: "user", + GitPassword: "pass", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + ba, ok := auth.(*githttp.BasicAuth) + require.True(t, ok) + require.Equal(t, opts.GitUsername, ba.Username) + require.Equal(t, opts.GitPassword, ba.Password) + }) + + t.Run("HTTPS/BasicAuth", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "https://host.tld/repo", + GitUsername: "user", + GitPassword: "pass", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + ba, ok := auth.(*githttp.BasicAuth) + require.True(t, ok) + require.Equal(t, opts.GitUsername, ba.Username) + require.Equal(t, opts.GitPassword, ba.Password) + }) + + t.Run("SSH/WithScheme", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "ssh://host.tld/repo", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.Equal(t, "git", pk.User) + }) + + t.Run("SSH/NoScheme", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "git@host.tld:repo/path", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.Equal(t, "git", pk.User) + }) + + t.Run("SSH/GitUsername", func(t *testing.T) { + t.Parallel() + opts := &envbuilder.Options{ + GitURL: "host.tld:12345/repo/path", + GitUsername: "user", + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.Equal(t, "user", pk.User) + }) + + t.Run("SSH/PrivateKey", func(t *testing.T) { + t.Parallel() + + // nolint:gosec // Throw-away key for testing. DO NOT REUSE. + testKey := `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuwAAAKgM05mUDNOZ +lAAAAAtzc2gtZWQyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuw +AAAEDCawwtjrM4AGYXD1G6uallnbsgMed4cfkFsQ+mLZtOkFc4aACB78Rtx6OoBKplrqzw +8VdmV7noOIYS3RVieWO7AAAAHmNpYW5AY2RyLW1icC1mdmZmdzBuOHEwNXAuaG9tZQECAw +QFBgc= +-----END OPENSSH PRIVATE KEY-----` + tmpDir := t.TempDir() + kPath := filepath.Join(tmpDir, "test.key") + require.NoError(t, os.WriteFile(kPath, []byte(testKey), 0o600)) + opts := &envbuilder.Options{ + GitURL: "ssh://git@host.tld:repo/path", + GitSSHPrivateKeyPath: kPath, + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.NotNil(t, pk.Signer) + actualSigner, err := gossh.ParsePrivateKey([]byte(testKey)) + require.NoError(t, err) + require.Equal(t, actualSigner, pk.Signer) + }) + + t.Run("SSH/MissingPrivateKey", func(t *testing.T) { + t.Parallel() + + tmpDir := t.TempDir() + kPath := filepath.Join(tmpDir, "test.key") + require.NoError(t, os.WriteFile(kPath, []byte(`invalid`), 0o600)) + opts := &envbuilder.Options{ + GitURL: "ssh://git@host.tld:repo/path", + GitSSHPrivateKeyPath: kPath, + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + pk, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + require.Nil(t, pk.Signer) + }) } func mustRead(t *testing.T, fs billy.Filesystem, path string) string { @@ -312,3 +415,9 @@ func randKeygen(t *testing.T) gossh.Signer { require.NoError(t, err) return signer } + +func testLog(t *testing.T) envbuilder.LoggerFunc { + return func(_ codersdk.LogLevel, format string, args ...interface{}) { + t.Logf(format, args...) + } +} diff --git a/options.go b/options.go index e098110c..027c8020 100644 --- a/options.go +++ b/options.go @@ -13,37 +13,36 @@ type LoggerFunc func(level codersdk.LogLevel, format string, args ...interface{} // Options contains the configuration for the envbuilder. type Options struct { - SetupScript string - InitScript string - InitCommand string - InitArgs string - CacheRepo string - BaseImageCacheDir string - LayerCacheDir string - DevcontainerDir string - DevcontainerJSONPath string - DockerfilePath string - BuildContextPath string - CacheTTLDays int64 - DockerConfigBase64 string - FallbackImage string - ExitOnBuildFailure bool - ForceSafe bool - Insecure bool - IgnorePaths []string - SkipRebuild bool - GitURL string - GitCloneDepth int64 - GitCloneSingleBranch bool - GitUsername string - GitPassword string - GitSSHPrivateKeyPath string - GitSSHKnownHostsBase64 string - GitHTTPProxyURL string - WorkspaceFolder string - SSLCertBase64 string - ExportEnvFile string - PostStartScriptPath string + SetupScript string + InitScript string + InitCommand string + InitArgs string + CacheRepo string + BaseImageCacheDir string + LayerCacheDir string + DevcontainerDir string + DevcontainerJSONPath string + DockerfilePath string + BuildContextPath string + CacheTTLDays int64 + DockerConfigBase64 string + FallbackImage string + ExitOnBuildFailure bool + ForceSafe bool + Insecure bool + IgnorePaths []string + SkipRebuild bool + GitURL string + GitCloneDepth int64 + GitCloneSingleBranch bool + GitUsername string + GitPassword string + GitSSHPrivateKeyPath string + GitHTTPProxyURL string + WorkspaceFolder string + SSLCertBase64 string + ExportEnvFile string + PostStartScriptPath string // Logger is the logger to use for all operations. Logger LoggerFunc // Filesystem is the filesystem to use for all operations. @@ -250,13 +249,6 @@ func (o *Options) CLI() serpent.OptionSet { Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), Description: "Path to a SSH private key to be used for Git authentication.", }, - { - Flag: "git-ssh-known-hosts-base64", - Env: "GIT_SSH_KNOWN_HOSTS_BASE64", - Value: serpent.StringOf(&o.GitSSHKnownHostsBase64), - Description: "Base64-encoded content of a known hosts file. If not specified, " + - "host keys will be scanned and logged, but not checked.", - }, { Flag: "git-http-proxy-url", Env: "GIT_HTTP_PROXY_URL", diff --git a/testdata/options.golden b/testdata/options.golden index 2dfae67f..7226f6fe 100644 --- a/testdata/options.golden +++ b/testdata/options.golden @@ -90,10 +90,6 @@ OPTIONS: --git-password string, $GIT_PASSWORD The password to use for Git authentication. This is optional. - --git-ssh-known-hosts-base64 string, $GIT_SSH_KNOWN_HOSTS_BASE64 - Base64-encoded content of a known hosts file. If not specified, host - keys will be scanned and logged, but not checked. - --git-ssh-private-key-path string, $GIT_SSH_PRIVATE_KEY_PATH Path to a SSH private key to be used for Git authentication. From 02226ae54570fa65abf9ca8b4feb74c77d4bd003 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 10:13:07 +0000 Subject: [PATCH 13/18] fixup! simply by removing GIT_SSH_KNOWN_HOSTS_BASE64, refactor repo auth setup --- options.go | 2 +- testdata/options.golden | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/options.go b/options.go index 027c8020..0b940929 100644 --- a/options.go +++ b/options.go @@ -247,7 +247,7 @@ func (o *Options) CLI() serpent.OptionSet { Flag: "git-ssh-private-key-path", Env: "GIT_SSH_PRIVATE_KEY_PATH", Value: serpent.StringOf(&o.GitSSHPrivateKeyPath), - Description: "Path to a SSH private key to be used for Git authentication.", + Description: "Path to an SSH private key to be used for Git authentication.", }, { Flag: "git-http-proxy-url", diff --git a/testdata/options.golden b/testdata/options.golden index 7226f6fe..0beffb6e 100644 --- a/testdata/options.golden +++ b/testdata/options.golden @@ -91,7 +91,7 @@ OPTIONS: The password to use for Git authentication. This is optional. --git-ssh-private-key-path string, $GIT_SSH_PRIVATE_KEY_PATH - Path to a SSH private key to be used for Git authentication. + Path to an SSH private key to be used for Git authentication. --git-url string, $GIT_URL The URL of the Git repository to clone. This is optional. From 36871a3e370940e5359be43a71bacaadf8610a16 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 10:13:47 +0000 Subject: [PATCH 14/18] make docs --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index fa6939e6..bad87e1e 100644 --- a/README.md +++ b/README.md @@ -288,8 +288,7 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de | `--git-clone-single-branch` | `GIT_CLONE_SINGLE_BRANCH` | | Clone only a single branch of the Git repository. | | `--git-username` | `GIT_USERNAME` | | The username to use for Git authentication. This is optional. | | `--git-password` | `GIT_PASSWORD` | | The password to use for Git authentication. This is optional. | -| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` | | Path to a SSH private key to be used for Git authentication. | -| `--git-ssh-known-hosts-base64` | `GIT_SSH_KNOWN_HOSTS_BASE64` | | Base64-encoded content of a known hosts file. If not specified, host keys will be scanned and logged, but not checked. | +| `--git-ssh-private-key-path` | `GIT_SSH_PRIVATE_KEY_PATH` | | Path to an SSH private key to be used for Git authentication. | | `--git-http-proxy-url` | `GIT_HTTP_PROXY_URL` | | The URL for the HTTP proxy. This is optional. | | `--workspace-folder` | `WORKSPACE_FOLDER` | | The path to the workspace folder that will be built. This is optional. | | `--ssl-cert-base64` | `SSL_CERT_BASE64` | | The content of an SSL cert file. This is useful for self-signed certificates. | From 39088ffe1e4992f43db033ad33510ef4b72d08be Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 10:59:24 +0000 Subject: [PATCH 15/18] fix SSH agent auth --- git.go | 54 ++++++++++++++++++++++++++++++++++++++++------------- git_test.go | 12 ++++-------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/git.go b/git.go index 67967744..b674b393 100644 --- a/git.go +++ b/git.go @@ -22,6 +22,7 @@ import ( gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage/filesystem" "github.com/skeema/knownhosts" + "golang.org/x/crypto/ssh" gossh "golang.org/x/crypto/ssh" ) @@ -195,28 +196,55 @@ func SetupRepoAuth(options *Options) transport.AuthMethod { Password: options.GitPassword, } } - // Assume SSH auth for all other formats. - auth := &gitssh.PublicKeys{} - options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using SSH authentication!") + // Generally git clones over SSH use the 'git' user, but respect // GIT_USERNAME if set. - auth.User = options.GitUsername - if auth.User == "" { - auth.User = "git" + authUser := options.GitUsername + if authUser == "" { + authUser = "git" } - // If we are told about an SSH private key, attempt to read it. + + // Assume SSH auth for all other formats. + options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using SSH authentication!") + + var signer ssh.Signer if options.GitSSHPrivateKeyPath != "" { - signer, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) + s, err := ReadPrivateKey(options.GitSSHPrivateKeyPath) if err != nil { options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error()) - // go-git will still attempt to fall back to other auth methods from the - // environment. This enables usage of SSH_AUTH_SOCK, for example. - return auth + } else { + options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type()) + signer = s + } + } + + // If no SSH key set, fall back to agent auth. + if signer == nil { + options.Logger(codersdk.LogLevelError, "#1: 🔑 No SSH key found, falling back to agent!") + auth, err := gitssh.NewSSHAgentAuth(authUser) + if err != nil { + options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error()) + return nil // nothing else we can do + } + if os.Getenv("SSH_KNOWN_HOSTS") == "" { + options.Logger(codersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") + auth.HostKeyCallback = LogHostKeyCallback(options.Logger) } - options.Logger(codersdk.LogLevelInfo, "#1: 🔑 Using %s key!", signer.PublicKey().Type()) - auth.Signer = signer + return auth + } + + auth := &gitssh.PublicKeys{ + User: options.GitUsername, + Signer: signer, + } + + // Generally git clones over SSH use the 'git' user, but respect + // GIT_USERNAME if set. + if auth.User == "" { + auth.User = "git" } + // Duplicated code due to Go's type system. if os.Getenv("SSH_KNOWN_HOSTS") == "" { options.Logger(codersdk.LogLevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!") auth.HostKeyCallback = LogHostKeyCallback(options.Logger) diff --git a/git_test.go b/git_test.go index dc0bea44..45666976 100644 --- a/git_test.go +++ b/git_test.go @@ -320,9 +320,8 @@ func TestSetupRepoAuth(t *testing.T) { Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - pk, ok := auth.(*gitssh.PublicKeys) + _, ok := auth.(*gitssh.PublicKeysCallback) require.True(t, ok) - require.Equal(t, "git", pk.User) }) t.Run("SSH/NoScheme", func(t *testing.T) { @@ -332,9 +331,8 @@ func TestSetupRepoAuth(t *testing.T) { Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - pk, ok := auth.(*gitssh.PublicKeys) + _, ok := auth.(*gitssh.PublicKeysCallback) require.True(t, ok) - require.Equal(t, "git", pk.User) }) t.Run("SSH/GitUsername", func(t *testing.T) { @@ -345,9 +343,8 @@ func TestSetupRepoAuth(t *testing.T) { Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - pk, ok := auth.(*gitssh.PublicKeys) + _, ok := auth.(*gitssh.PublicKeysCallback) require.True(t, ok) - require.Equal(t, "user", pk.User) }) t.Run("SSH/PrivateKey", func(t *testing.T) { @@ -391,9 +388,8 @@ QFBgc= Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - pk, ok := auth.(*gitssh.PublicKeys) + _, ok := auth.(*gitssh.PublicKeysCallback) require.True(t, ok) - require.Nil(t, pk.Signer) }) } From 7f30872984cb29dd8963db036bb9afb6c33df85f Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 11:09:18 +0000 Subject: [PATCH 16/18] fix tests --- git_test.go | 86 +++++++++++++++++++++++++---------------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/git_test.go b/git_test.go index 45666976..659eea7d 100644 --- a/git_test.go +++ b/git_test.go @@ -260,12 +260,10 @@ func TestCloneRepoSSH(t *testing.T) { }) } +// nolint:paralleltest // t.Setenv for SSH_AUTH_SOCK func TestSetupRepoAuth(t *testing.T) { - t.Parallel() - + t.Setenv("SSH_AUTH_SOCK", "") t.Run("Empty", func(t *testing.T) { - t.Parallel() - opts := &envbuilder.Options{ Logger: testLog(t), } @@ -274,7 +272,6 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTP/NoAuth", func(t *testing.T) { - t.Parallel() opts := &envbuilder.Options{ GitURL: "http://host.tld/repo", Logger: testLog(t), @@ -284,7 +281,6 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTP/BasicAuth", func(t *testing.T) { - t.Parallel() opts := &envbuilder.Options{ GitURL: "http://host.tld/repo", GitUsername: "user", @@ -299,7 +295,6 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("HTTPS/BasicAuth", func(t *testing.T) { - t.Parallel() opts := &envbuilder.Options{ GitURL: "https://host.tld/repo", GitUsername: "user", @@ -314,54 +309,44 @@ func TestSetupRepoAuth(t *testing.T) { }) t.Run("SSH/WithScheme", func(t *testing.T) { - t.Parallel() + kPath := writeTestPrivateKey(t) opts := &envbuilder.Options{ - GitURL: "ssh://host.tld/repo", - Logger: testLog(t), + GitURL: "ssh://host.tld/repo", + GitSSHPrivateKeyPath: kPath, + Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - _, ok := auth.(*gitssh.PublicKeysCallback) + _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) t.Run("SSH/NoScheme", func(t *testing.T) { - t.Parallel() + kPath := writeTestPrivateKey(t) opts := &envbuilder.Options{ - GitURL: "git@host.tld:repo/path", - Logger: testLog(t), + GitURL: "git@host.tld:repo/path", + GitSSHPrivateKeyPath: kPath, + Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - _, ok := auth.(*gitssh.PublicKeysCallback) + _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) t.Run("SSH/GitUsername", func(t *testing.T) { - t.Parallel() + kPath := writeTestPrivateKey(t) opts := &envbuilder.Options{ - GitURL: "host.tld:12345/repo/path", - GitUsername: "user", - Logger: testLog(t), + GitURL: "host.tld:12345/repo/path", + GitSSHPrivateKeyPath: kPath, + GitUsername: "user", + Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - _, ok := auth.(*gitssh.PublicKeysCallback) + _, ok := auth.(*gitssh.PublicKeys) require.True(t, ok) }) t.Run("SSH/PrivateKey", func(t *testing.T) { - t.Parallel() - - // nolint:gosec // Throw-away key for testing. DO NOT REUSE. - testKey := `-----BEGIN OPENSSH PRIVATE KEY----- -b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW -QyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuwAAAKgM05mUDNOZ -lAAAAAtzc2gtZWQyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuw -AAAEDCawwtjrM4AGYXD1G6uallnbsgMed4cfkFsQ+mLZtOkFc4aACB78Rtx6OoBKplrqzw -8VdmV7noOIYS3RVieWO7AAAAHmNpYW5AY2RyLW1icC1mdmZmdzBuOHEwNXAuaG9tZQECAw -QFBgc= ------END OPENSSH PRIVATE KEY-----` - tmpDir := t.TempDir() - kPath := filepath.Join(tmpDir, "test.key") - require.NoError(t, os.WriteFile(kPath, []byte(testKey), 0o600)) + kPath := writeTestPrivateKey(t) opts := &envbuilder.Options{ GitURL: "ssh://git@host.tld:repo/path", GitSSHPrivateKeyPath: kPath, @@ -376,20 +361,13 @@ QFBgc= require.Equal(t, actualSigner, pk.Signer) }) - t.Run("SSH/MissingPrivateKey", func(t *testing.T) { - t.Parallel() - - tmpDir := t.TempDir() - kPath := filepath.Join(tmpDir, "test.key") - require.NoError(t, os.WriteFile(kPath, []byte(`invalid`), 0o600)) + t.Run("SSH/NoAuthMethods", func(t *testing.T) { opts := &envbuilder.Options{ - GitURL: "ssh://git@host.tld:repo/path", - GitSSHPrivateKeyPath: kPath, - Logger: testLog(t), + GitURL: "ssh://git@host.tld:repo/path", + Logger: testLog(t), } auth := envbuilder.SetupRepoAuth(opts) - _, ok := auth.(*gitssh.PublicKeysCallback) - require.True(t, ok) + require.Nil(t, auth) // TODO: actually test SSH_AUTH_SOCK }) } @@ -417,3 +395,21 @@ func testLog(t *testing.T) envbuilder.LoggerFunc { t.Logf(format, args...) } } + +// nolint:gosec // Throw-away key for testing. DO NOT REUSE. +var testKey = `-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuwAAAKgM05mUDNOZ +lAAAAAtzc2gtZWQyNTUxOQAAACBXOGgAge/EbcejqASqZa6s8PFXZle56DiGEt0VYnljuw +AAAEDCawwtjrM4AGYXD1G6uallnbsgMed4cfkFsQ+mLZtOkFc4aACB78Rtx6OoBKplrqzw +8VdmV7noOIYS3RVieWO7AAAAHmNpYW5AY2RyLW1icC1mdmZmdzBuOHEwNXAuaG9tZQECAw +QFBgc= +-----END OPENSSH PRIVATE KEY-----` + +func writeTestPrivateKey(t *testing.T) string { + t.Helper() + tmpDir := t.TempDir() + kPath := filepath.Join(tmpDir, "test.key") + require.NoError(t, os.WriteFile(kPath, []byte(testKey), 0o600)) + return kPath +} From 5dd031cdb3c9b337f43d82742156b5f6c6024023 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 13:17:25 +0000 Subject: [PATCH 17/18] address PR comments --- git.go | 8 ++++---- git_test.go | 13 +++++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/git.go b/git.go index b674b393..692434bd 100644 --- a/git.go +++ b/git.go @@ -131,6 +131,7 @@ func ReadPrivateKey(path string) (gossh.Signer, error) { if err != nil { return nil, fmt.Errorf("open private key file: %w", err) } + defer f.Close() bs, err := io.ReadAll(f) if err != nil { return nil, fmt.Errorf("read private key file: %w", err) @@ -199,9 +200,8 @@ func SetupRepoAuth(options *Options) transport.AuthMethod { // Generally git clones over SSH use the 'git' user, but respect // GIT_USERNAME if set. - authUser := options.GitUsername - if authUser == "" { - authUser = "git" + if options.GitUsername == "" { + options.GitUsername = "git" } // Assume SSH auth for all other formats. @@ -221,7 +221,7 @@ func SetupRepoAuth(options *Options) transport.AuthMethod { // If no SSH key set, fall back to agent auth. if signer == nil { options.Logger(codersdk.LogLevelError, "#1: 🔑 No SSH key found, falling back to agent!") - auth, err := gitssh.NewSSHAgentAuth(authUser) + auth, err := gitssh.NewSSHAgentAuth(options.GitUsername) if err != nil { options.Logger(codersdk.LogLevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error()) return nil // nothing else we can do diff --git a/git_test.go b/git_test.go index 659eea7d..5a575723 100644 --- a/git_test.go +++ b/git_test.go @@ -332,6 +332,19 @@ func TestSetupRepoAuth(t *testing.T) { require.True(t, ok) }) + t.Run("SSH/OtherScheme", func(t *testing.T) { + // Anything that is not https:// or http:// is treated as SSH. + kPath := writeTestPrivateKey(t) + opts := &envbuilder.Options{ + GitURL: "git://git@host.tld:repo/path", + GitSSHPrivateKeyPath: kPath, + Logger: testLog(t), + } + auth := envbuilder.SetupRepoAuth(opts) + _, ok := auth.(*gitssh.PublicKeys) + require.True(t, ok) + }) + t.Run("SSH/GitUsername", func(t *testing.T) { kPath := writeTestPrivateKey(t) opts := &envbuilder.Options{ From f200030fae680a8d933b17097cef54648a60c484 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 3 May 2024 14:08:44 +0000 Subject: [PATCH 18/18] update README --- README.md | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bad87e1e..0b230cb8 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,12 @@ DOCKER_CONFIG_BASE64=ewoJImF1dGhzIjogewoJCSJodHRwczovL2luZGV4LmRvY2tlci5pby92MS8 ## Git Authentication -`GIT_USERNAME` and `GIT_PASSWORD` are environment variables to provide Git authentication for private repositories. +Two methods of authentication are supported: + +### HTTP Authentication + +If the `GIT_URL` supplied starts with `http://` or `https://`, envbuilder will +supply HTTP basic authentication using `GIT_USERNAME` and `GIT_PASSWORD`, if set. For access token-based authentication, follow the following schema (if empty, there's no need to provide the field): @@ -161,6 +166,42 @@ resource "docker_container" "dev" { } ``` +### SSH Authentication + +If the `GIT_URL` supplied does not start with `http://` or `https://`, +envbuilder will assume SSH authentication. You have the following options: + +1. Public/Private key authentication: set `GIT_SSH_KEY_PATH` to the path of an + SSH private key mounted inside the container. Envbuilder will use this SSH + key to authenticate. Example: + + ```bash + docker run -it --rm \ + -v /tmp/envbuilder:/workspaces \ + -e GIT_URL=git@example.com:path/to/private/repo.git \ + -e GIT_SSH_KEY_PATH=/.ssh/id_rsa \ + -v /home/user/id_rsa:/.ssh/id_rsa \ + -e INIT_SCRIPT=bash \ + ghcr.io/coder/envbuilder + ``` + +1. Agent-based authentication: set `SSH_AUTH_SOCK` and mount in your agent socket, for example: + + ```bash + docker run -it --rm \ + -v /tmp/envbuilder:/workspaces \ + -e GIT_URL=git@example.com:path/to/private/repo.git \ + -e INIT_SCRIPT=bash \ + -e SSH_AUTH_SOCK=/tmp/ssh-auth-sock \ + -v $SSH_AUTH_SOCK:/tmp/ssh-auth-sock \ + ghcr.io/coder/envbuilder + ``` + +> Note: by default, envbuilder will accept and log all host keys. If you need +> strict host key checking, set `SSH_KNOWN_HOSTS` and mount in a `known_hosts` +> file. + + ## Layer Caching Cache layers in a container registry to speed up builds. To enable caching, [authenticate with your registry](#container-registry-authentication) and set the `CACHE_REPO` environment variable.