Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: Add TURN proxying to enable offline deployments #1000

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions .github/workflows/coder.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 +169,6 @@ jobs:
terraform_version: 1.1.2
terraform_wrapper: false

- name: Install socat
if: runner.os == 'Linux'
run: sudo apt-get install -y socat

- name: Test with Mock Database
shell: bash
env:
Expand Down
41 changes: 20 additions & 21 deletions agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,23 +32,23 @@ type Options struct {
Logger slog.Logger
}

type Dialer func(ctx context.Context, options *peer.ConnOptions) (*peerbroker.Listener, error)
type Dialer func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error)

func New(dialer Dialer, options *peer.ConnOptions) io.Closer {
func New(dialer Dialer, logger slog.Logger) io.Closer {
ctx, cancelFunc := context.WithCancel(context.Background())
server := &agent{
clientDialer: dialer,
options: options,
closeCancel: cancelFunc,
closed: make(chan struct{}),
dialer: dialer,
logger: logger,
closeCancel: cancelFunc,
closed: make(chan struct{}),
}
server.init(ctx)
return server
}

type agent struct {
clientDialer Dialer
options *peer.ConnOptions
dialer Dialer
logger slog.Logger

connCloseWait sync.WaitGroup
closeCancel context.CancelFunc
Expand All @@ -64,18 +64,18 @@ func (a *agent) run(ctx context.Context) {
// An exponential back-off occurs when the connection is failing to dial.
// This is to prevent server spam in case of a coderd outage.
for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); {
peerListener, err = a.clientDialer(ctx, a.options)
peerListener, err = a.dialer(ctx, a.logger)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
if a.isClosed() {
return
}
a.options.Logger.Warn(context.Background(), "failed to dial", slog.Error(err))
a.logger.Warn(context.Background(), "failed to dial", slog.Error(err))
continue
}
a.options.Logger.Info(context.Background(), "connected")
a.logger.Info(context.Background(), "connected")
break
}
select {
Expand All @@ -90,7 +90,7 @@ func (a *agent) run(ctx context.Context) {
if a.isClosed() {
return
}
a.options.Logger.Debug(ctx, "peer listener accept exited; restarting connection", slog.Error(err))
a.logger.Debug(ctx, "peer listener accept exited; restarting connection", slog.Error(err))
a.run(ctx)
return
}
Expand All @@ -105,10 +105,9 @@ func (a *agent) handlePeerConn(ctx context.Context, conn *peer.Conn) {
go func() {
select {
case <-a.closed:
_ = conn.Close()
case <-conn.Closed():
}
<-conn.Closed()
_ = conn.Close()
a.connCloseWait.Done()
}()
for {
Expand All @@ -117,15 +116,15 @@ func (a *agent) handlePeerConn(ctx context.Context, conn *peer.Conn) {
if errors.Is(err, peer.ErrClosed) || a.isClosed() {
return
}
a.options.Logger.Debug(ctx, "accept channel from peer connection", slog.Error(err))
a.logger.Debug(ctx, "accept channel from peer connection", slog.Error(err))
return
}

switch channel.Protocol() {
case "ssh":
go a.sshServer.HandleConn(channel.NetConn())
default:
a.options.Logger.Warn(ctx, "unhandled protocol from channel",
a.logger.Warn(ctx, "unhandled protocol from channel",
slog.F("protocol", channel.Protocol()),
slog.F("label", channel.Label()),
)
Expand All @@ -145,7 +144,7 @@ func (a *agent) init(ctx context.Context) {
if err != nil {
panic(err)
}
sshLogger := a.options.Logger.Named("ssh-server")
sshLogger := a.logger.Named("ssh-server")
forwardHandler := &ssh.ForwardedTCPHandler{}
a.sshServer = &ssh.Server{
ChannelHandlers: map[string]ssh.ChannelHandler{
Expand All @@ -158,7 +157,7 @@ func (a *agent) init(ctx context.Context) {
Handler: func(session ssh.Session) {
err := a.handleSSHSession(session)
if err != nil {
a.options.Logger.Warn(ctx, "ssh session failed", slog.Error(err))
a.logger.Warn(ctx, "ssh session failed", slog.Error(err))
_ = session.Exit(1)
return
}
Expand Down Expand Up @@ -194,15 +193,15 @@ func (a *agent) init(ctx context.Context) {
"sftp": func(session ssh.Session) {
server, err := sftp.NewServer(session)
if err != nil {
a.options.Logger.Debug(session.Context(), "initialize sftp server", slog.Error(err))
a.logger.Debug(session.Context(), "initialize sftp server", slog.Error(err))
return
}
defer server.Close()
err = server.Serve()
if errors.Is(err, io.EOF) {
return
}
a.options.Logger.Debug(session.Context(), "sftp server exited with error", slog.Error(err))
a.logger.Debug(session.Context(), "sftp server exited with error", slog.Error(err))
},
},
}
Expand Down Expand Up @@ -250,7 +249,7 @@ func (a *agent) handleSSHSession(session ssh.Session) error {
for win := range windowSize {
err = ptty.Resize(uint16(win.Width), uint16(win.Height))
if err != nil {
a.options.Logger.Warn(context.Background(), "failed to resize tty", slog.Error(err))
a.logger.Warn(context.Background(), "failed to resize tty", slog.Error(err))
}
}
}()
Expand Down
8 changes: 3 additions & 5 deletions agent/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,11 +170,9 @@ func setupSSHSession(t *testing.T) *ssh.Session {

func setupAgent(t *testing.T) *agent.Conn {
client, server := provisionersdk.TransportPipe()
closer := agent.New(func(ctx context.Context, opts *peer.ConnOptions) (*peerbroker.Listener, error) {
return peerbroker.Listen(server, nil, opts)
}, &peer.ConnOptions{
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
})
closer := agent.New(func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) {
return peerbroker.Listen(server, nil)
}, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
t.Cleanup(func() {
_ = client.Close()
_ = server.Close()
Expand Down
5 changes: 1 addition & 4 deletions cli/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/coder/coder/agent"
"github.com/coder/coder/cli/cliflag"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/peer"
"github.com/coder/retry"
)

Expand Down Expand Up @@ -110,9 +109,7 @@ func workspaceAgent() *cobra.Command {
return xerrors.Errorf("writing agent session token to config: %w", err)
}

closer := agent.New(client.ListenWorkspaceAgent, &peer.ConnOptions{
Logger: logger,
})
closer := agent.New(client.ListenWorkspaceAgent, logger)
<-cmd.Context().Done()
return closer.Close()
},
Expand Down
4 changes: 2 additions & 2 deletions cli/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func TestWorkspaceAgent(t *testing.T) {
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
resources, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID)
require.NoError(t, err)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil, nil)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
_, err = dialer.Ping()
Expand Down Expand Up @@ -115,7 +115,7 @@ func TestWorkspaceAgent(t *testing.T) {
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
resources, err := client.WorkspaceResourcesByBuild(ctx, workspace.LatestBuild.ID)
require.NoError(t, err)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil, nil)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
_, err = dialer.Ping()
Expand Down
7 changes: 2 additions & 5 deletions cli/configssh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import (
"github.com/coder/coder/cli/clitest"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/peer"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/provisionersdk/proto"
"github.com/coder/coder/pty/ptytest"
Expand Down Expand Up @@ -72,17 +71,15 @@ func TestConfigSSH(t *testing.T) {
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
agentClient := codersdk.New(client.URL)
agentClient.SessionToken = authToken
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
Logger: slogtest.Make(t, nil),
})
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil))
t.Cleanup(func() {
_ = agentCloser.Close()
})
tempFile, err := os.CreateTemp(t.TempDir(), "")
require.NoError(t, err)
_ = tempFile.Close()
resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil, nil)
agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer agentConn.Close()

Expand Down
2 changes: 1 addition & 1 deletion cli/gitssh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func TestGitSSH(t *testing.T) {
coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID)
resources, err := client.WorkspaceResourcesByBuild(context.Background(), workspace.LatestBuild.ID)
require.NoError(t, err)
dialer, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil, nil)
dialer, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
_, err = dialer.Ping()
Expand Down
5 changes: 1 addition & 4 deletions cli/ssh.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/google/uuid"
"github.com/mattn/go-isatty"
"github.com/pion/webrtc/v3"
"github.com/spf13/cobra"
gossh "golang.org/x/crypto/ssh"
"golang.org/x/term"
Expand Down Expand Up @@ -99,9 +98,7 @@ func ssh() *cobra.Command {
return xerrors.Errorf("await agent: %w", err)
}

conn, err := client.DialWorkspaceAgent(cmd.Context(), agent.ID, []webrtc.ICEServer{{
URLs: []string{"stun:stun.l.google.com:19302"},
}}, nil)
conn, err := client.DialWorkspaceAgent(cmd.Context(), agent.ID, nil)
if err != nil {
return err
}
Expand Down
9 changes: 2 additions & 7 deletions cli/ssh_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"github.com/coder/coder/cli/clitest"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/peer"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/provisionersdk/proto"
"github.com/coder/coder/pty/ptytest"
Expand Down Expand Up @@ -70,9 +69,7 @@ func TestSSH(t *testing.T) {
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
agentClient := codersdk.New(client.URL)
agentClient.SessionToken = agentToken
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
})
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
t.Cleanup(func() {
_ = agentCloser.Close()
})
Expand Down Expand Up @@ -115,9 +112,7 @@ func TestSSH(t *testing.T) {
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
agentClient := codersdk.New(client.URL)
agentClient.SessionToken = agentToken
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &peer.ConnOptions{
Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug),
})
agentCloser := agent.New(agentClient.ListenWorkspaceAgent, slogtest.Make(t, nil).Leveled(slog.LevelDebug))
t.Cleanup(func() {
_ = agentCloser.Close()
})
Expand Down
15 changes: 15 additions & 0 deletions cli/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (

"github.com/briandowns/spinner"
"github.com/coreos/go-systemd/daemon"
"github.com/pion/turn/v2"
"github.com/spf13/cobra"
"golang.org/x/xerrors"
"google.golang.org/api/idtoken"
Expand All @@ -34,6 +35,7 @@ import (
"github.com/coder/coder/coderd/database/databasefake"
"github.com/coder/coder/coderd/devtunnel"
"github.com/coder/coder/coderd/gitsshkey"
"github.com/coder/coder/coderd/turnconn"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/provisioner/terraform"
"github.com/coder/coder/provisionerd"
Expand All @@ -56,11 +58,13 @@ func start() *cobra.Command {
tlsEnable bool
tlsKeyFile string
tlsMinVersion string
turnRelayAddress string
skipTunnel bool
traceDatadog bool
secureAuthCookie bool
sshKeygenAlgorithmRaw string
)

root := &cobra.Command{
Use: "start",
RunE: func(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -156,6 +160,14 @@ func start() *cobra.Command {
return xerrors.Errorf("parse ssh keygen algorithm %s: %w", sshKeygenAlgorithmRaw, err)
}

turnServer, err := turnconn.New(&turn.RelayAddressGeneratorStatic{
RelayAddress: net.ParseIP(turnRelayAddress),
Address: turnRelayAddress,
})
if err != nil {
return xerrors.Errorf("create turn server: %w", err)
}

options := &coderd.Options{
AccessURL: accessURLParsed,
Logger: logger.Named("coderd"),
Expand All @@ -164,6 +176,7 @@ func start() *cobra.Command {
GoogleTokenValidator: validator,
SecureAuthCookie: secureAuthCookie,
SSHKeygenAlgorithm: sshKeygenAlgorithm,
TURNServer: turnServer,
}

_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "access-url: %s\n", accessURL)
Expand Down Expand Up @@ -376,6 +389,8 @@ func start() *cobra.Command {
cliflag.BoolVarP(root.Flags(), &skipTunnel, "skip-tunnel", "", "CODER_DEV_SKIP_TUNNEL", false, "Skip serving dev mode through an exposed tunnel for simple setup.")
_ = root.Flags().MarkHidden("skip-tunnel")
cliflag.BoolVarP(root.Flags(), &traceDatadog, "trace-datadog", "", "CODER_TRACE_DATADOG", false, "Send tracing data to a datadog agent")
cliflag.StringVarP(root.Flags(), &turnRelayAddress, "turn-relay-address", "", "CODER_TURN_RELAY_ADDRESS", "127.0.0.1",
"Specifies the address to bind TURN connections.")
cliflag.BoolVarP(root.Flags(), &secureAuthCookie, "secure-auth-cookie", "", "CODER_SECURE_AUTH_COOKIE", false, "Specifies if the 'Secure' property is set on browser session cookies")
cliflag.StringVarP(root.Flags(), &sshKeygenAlgorithmRaw, "ssh-keygen-algorithm", "", "CODER_SSH_KEYGEN_ALGORITHM", "ed25519", "Specifies the algorithm to use for generating ssh keys. "+
`Accepted values are "ed25519", "ecdsa", or "rsa4096"`)
Expand Down
Loading