From 789e87802f94e9fbf2cc7ca1973ad1364cd1217e Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Mon, 27 Jun 2022 18:38:46 +0000 Subject: [PATCH 01/54] fix: Add coder user to docker group on installation This makes for a simpler setup, and reduces the likelihood a user runs into a strange issue. --- preinstall.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/preinstall.sh b/preinstall.sh index 932dab26b04f7..79614f195df5b 100644 --- a/preinstall.sh +++ b/preinstall.sh @@ -11,4 +11,14 @@ if ! id -u $USER >/dev/null 2>&1; then --user-group \ --shell /bin/false \ $USER + + # Add the Coder user to the Docker group. + # Coder is frequently used with Docker, so + # this prevents failures when building. + # + # It's fine if this fails! + usermod \ + --append \ + --groups docker \ + $USER 2>/dev/null || true fi From c5f91a862a18acbcfa4338841badba47b3bebdba Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Tue, 28 Jun 2022 12:21:55 +0000 Subject: [PATCH 02/54] Add wgnet --- .vscode/settings.json | 25 + agent/agent.go | 112 ++++- agent/agent_test.go | 5 + agent/wireguard.go | 97 ---- cli/agent.go | 9 +- coderd/coderd.go | 26 +- coderd/coderd_test.go | 74 +++ coderd/database/databasefake/databasefake.go | 40 +- coderd/database/dbtypes/dbtypes.go | 76 +-- coderd/database/dump.sql | 8 +- coderd/database/dump/main.go | 12 - .../migrations/000029_tailnet.down.sql | 0 .../database/migrations/000029_tailnet.up.sql | 6 + coderd/database/models.go | 42 +- coderd/database/postgres/postgres.go | 17 +- coderd/database/querier.go | 2 +- coderd/database/queries.sql.go | 132 +++--- coderd/database/queries/workspaceagents.sql | 16 +- coderd/database/sqlc.yaml | 10 +- coderd/provisionerdaemons.go | 16 +- coderd/workspaceagents.go | 206 ++++---- codersdk/workspaceagents.go | 46 +- codersdk/workspaceresources.go | 13 +- go.mod | 11 +- go.sum | 11 +- peer/peerwg/derp.go | 69 +-- peer/peerwg/wireguard.go | 46 +- peer/peerwg/wireguard_test.go | 116 +++++ site/src/api/typesGenerated.ts | 14 +- tailnet/tailnet.go | 439 ++++++++++++++++++ tailnet/tailnet_test.go | 147 ++++++ 31 files changed, 1302 insertions(+), 541 deletions(-) delete mode 100644 agent/wireguard.go create mode 100644 coderd/database/migrations/000029_tailnet.down.sql create mode 100644 coderd/database/migrations/000029_tailnet.up.sql create mode 100644 peer/peerwg/wireguard_test.go create mode 100644 tailnet/tailnet.go create mode 100644 tailnet/tailnet_test.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 0d81d15cc32b5..ccb4a61733cb7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,6 +11,8 @@ "coderdtest", "codersdk", "cronstrue", + "DERP", + "derphttp", "devel", "drpc", "drpcconn", @@ -23,6 +25,7 @@ "goarch", "gographviz", "goleak", + "gonet", "gossh", "gsyslog", "hashicorp", @@ -37,13 +40,21 @@ "Keygen", "kirsle", "ldflags", + "magicsock", "manifoldco", "mapstructure", "mattn", "mitchellh", "moby", + "namespacing", + "netaddr", + "netmap", + "netns", + "netstack", + "nettype", "nfpms", "nhooyr", + "nmcfg", "nolint", "nosec", "ntqry", @@ -57,6 +68,7 @@ "provisionersdk", "ptty", "ptytest", + "reconfig", "retrier", "rpty", "sdkproto", @@ -65,6 +77,10 @@ "sourcemapped", "Srcs", "stretchr", + "stuntest", + "tailcfg", + "tailnet", + "Tailscale", "TCGETS", "tcpip", "TCSETS", @@ -76,13 +92,22 @@ "tfplan", "tfstate", "trimprefix", + "tsdial", + "tslogger", + "tstun", "turnconn", "typegen", "unconvert", "Untar", + "Userspace", "VMID", "weblinks", "webrtc", + "wgcfg", + "wgconfig", + "wgengine", + "wgmonitor", + "wgnet", "workspaceagent", "workspaceapp", "workspaceapps", diff --git a/agent/agent.go b/agent/agent.go index d596511dd1522..e6b7caeecc006 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -28,14 +28,14 @@ import ( gossh "golang.org/x/crypto/ssh" "golang.org/x/xerrors" "inet.af/netaddr" - "tailscale.com/types/key" + "tailscale.com/tailcfg" "cdr.dev/slog" "github.com/coder/coder/agent/usershell" "github.com/coder/coder/peer" - "github.com/coder/coder/peer/peerwg" "github.com/coder/coder/peerbroker" "github.com/coder/coder/pty" + "github.com/coder/coder/tailnet" "github.com/coder/retry" ) @@ -47,30 +47,28 @@ const ( type Options struct { EnableWireguard bool - UploadWireguardKeys UploadWireguardKeys - ListenWireguardPeers ListenWireguardPeers + UpdateTailscaleNode UpdateTailscaleNode + ListenTailscaleNodes ListenTailscaleNodes ReconnectingPTYTimeout time.Duration EnvironmentVariables map[string]string Logger slog.Logger } type Metadata struct { - WireguardAddresses []netaddr.IPPrefix `json:"addresses"` - OwnerEmail string `json:"owner_email"` - OwnerUsername string `json:"owner_username"` - EnvironmentVariables map[string]string `json:"environment_variables"` - StartupScript string `json:"startup_script"` - Directory string `json:"directory"` -} - -type WireguardPublicKeys struct { - Public key.NodePublic `json:"public"` - Disco key.DiscoPublic `json:"disco"` + TailscaleAddresses []netaddr.IPPrefix `json:"tailscale_addresses"` + TailscaleDERPMap *tailcfg.DERPMap `json:"tailscale_derpmap"` + + OwnerEmail string `json:"owner_email"` + OwnerUsername string `json:"owner_username"` + EnvironmentVariables map[string]string `json:"environment_variables"` + StartupScript string `json:"startup_script"` + Directory string `json:"directory"` } type Dialer func(ctx context.Context, logger slog.Logger) (Metadata, *peerbroker.Listener, error) -type UploadWireguardKeys func(ctx context.Context, keys WireguardPublicKeys) error -type ListenWireguardPeers func(ctx context.Context, logger slog.Logger) (<-chan peerwg.Handshake, func(), error) + +type UpdateTailscaleNode func(ctx context.Context, node *tailnet.Node) error +type ListenTailscaleNodes func(ctx context.Context, logger slog.Logger) (<-chan *tailnet.Node, func(), error) func New(dialer Dialer, options *Options) io.Closer { if options == nil { @@ -88,8 +86,8 @@ func New(dialer Dialer, options *Options) io.Closer { closed: make(chan struct{}), envVars: options.EnvironmentVariables, enableWireguard: options.EnableWireguard, - postKeys: options.UploadWireguardKeys, - listenWireguardPeers: options.ListenWireguardPeers, + updateTailscaleNode: options.UpdateTailscaleNode, + listenTailscaleNodes: options.ListenTailscaleNodes, } server.init(ctx) return server @@ -114,9 +112,9 @@ type agent struct { sshServer *ssh.Server enableWireguard bool - network *peerwg.Network - postKeys UploadWireguardKeys - listenWireguardPeers ListenWireguardPeers + network *tailnet.Server + updateTailscaleNode UpdateTailscaleNode + listenTailscaleNodes ListenTailscaleNodes } func (a *agent) run(ctx context.Context) { @@ -160,8 +158,9 @@ func (a *agent) run(ctx context.Context) { }() } - if a.enableWireguard { - err = a.startWireguard(ctx, metadata.WireguardAddresses) + // We don't want to reinitialize the network if it already exists. + if a.enableWireguard && a.network == nil { + err = a.startWireguard(ctx, metadata.TailscaleAddresses, metadata.TailscaleDERPMap) if err != nil { a.logger.Error(ctx, "start wireguard", slog.Error(err)) } @@ -668,6 +667,71 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne } } +func (a *agent) startWireguard(ctx context.Context, addresses []netaddr.IPPrefix, derpMap *tailcfg.DERPMap) error { + var err error + a.network, err = tailnet.New(&tailnet.Options{ + Addresses: addresses, + DERPMap: derpMap, + Logger: a.logger.Named("tailnet"), + }) + if err != nil { + return err + } + a.network.SetNodeCallback(func(node *tailnet.Node) { + err := a.updateTailscaleNode(ctx, node) + if err != nil { + a.logger.Error(ctx, "update tailscale node", slog.Error(err)) + } + }) + go func() { + for { + var nodes <-chan *tailnet.Node + var err error + var listenClose func() + for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { + nodes, listenClose, err = a.listenTailscaleNodes(ctx, a.logger) + if err != nil { + if errors.Is(err, context.Canceled) { + return + } + a.logger.Warn(ctx, "listen for tailscale nodes", slog.Error(err)) + continue + } + defer listenClose() + a.logger.Info(context.Background(), "listening for tailscale nodes") + break + } + for { + var node *tailnet.Node + select { + case <-ctx.Done(): + case node = <-nodes: + } + if node == nil { + // The channel ended! + break + } + a.network.UpdateNodes([]*tailnet.Node{node}) + } + } + }() + + sshListener, err := a.network.Listen("tcp", ":12212") + if err != nil { + return xerrors.Errorf("listen for ssh: %w", err) + } + go func() { + for { + conn, err := sshListener.Accept() + if err != nil { + return + } + go a.sshServer.HandleConn(conn) + } + }() + return nil +} + // dialResponse is written to datachannels with protocol "dial" by the agent as // the first packet to signify whether the dial succeeded or failed. type dialResponse struct { diff --git a/agent/agent_test.go b/agent/agent_test.go index 22ee920c2c8f7..55c53c20a0503 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -395,6 +395,11 @@ func TestAgent(t *testing.T) { require.ErrorContains(t, err, "no such file") require.Nil(t, netConn) }) + + t.Run("Tailscale", func(t *testing.T) { + t.Parallel() + + }) } func setupSSHCommand(t *testing.T, beforeArgs []string, afterArgs []string) *exec.Cmd { diff --git a/agent/wireguard.go b/agent/wireguard.go deleted file mode 100644 index 603b5616e4740..0000000000000 --- a/agent/wireguard.go +++ /dev/null @@ -1,97 +0,0 @@ -package agent - -import ( - "context" - "net" - "strconv" - - "golang.org/x/xerrors" - "inet.af/netaddr" - - "cdr.dev/slog" - "github.com/coder/coder/peer/peerwg" -) - -func (a *agent) startWireguard(ctx context.Context, addrs []netaddr.IPPrefix) error { - if a.network != nil { - _ = a.network.Close() - a.network = nil - } - - // We can't create a wireguard network without these. - if len(addrs) == 0 || a.listenWireguardPeers == nil || a.postKeys == nil { - return xerrors.New("wireguard is enabled, but no addresses were provided or necessary functions were not provided") - } - - wg, err := peerwg.New(a.logger.Named("wireguard"), addrs) - if err != nil { - return xerrors.Errorf("create wireguard network: %w", err) - } - - // A new keypair is generated on each agent start. - // This keypair must be sent to Coder to allow for incoming connections. - err = a.postKeys(ctx, WireguardPublicKeys{ - Public: wg.NodePrivateKey.Public(), - Disco: wg.DiscoPublicKey, - }) - if err != nil { - a.logger.Warn(ctx, "post keys", slog.Error(err)) - } - - go func() { - for { - ch, listenClose, err := a.listenWireguardPeers(ctx, a.logger) - if err != nil { - a.logger.Warn(ctx, "listen wireguard peers", slog.Error(err)) - return - } - - for { - peer, ok := <-ch - if !ok { - break - } - - err := wg.AddPeer(peer) - a.logger.Info(ctx, "added wireguard peer", slog.F("peer", peer.NodePublicKey.ShortString()), slog.Error(err)) - } - - listenClose() - } - }() - - a.startWireguardListeners(ctx, wg, []handlerPort{ - {port: 12212, handler: a.sshServer.HandleConn}, - }) - - a.network = wg - return nil -} - -type handlerPort struct { - handler func(conn net.Conn) - port uint16 -} - -func (a *agent) startWireguardListeners(ctx context.Context, network *peerwg.Network, handlers []handlerPort) { - for _, h := range handlers { - go func(h handlerPort) { - a.logger.Debug(ctx, "starting wireguard listener", slog.F("port", h.port)) - - listener, err := network.Listen("tcp", net.JoinHostPort("", strconv.Itoa(int(h.port)))) - if err != nil { - a.logger.Warn(ctx, "listen wireguard", slog.F("port", h.port), slog.Error(err)) - return - } - - for { - conn, err := listener.Accept() - if err != nil { - return - } - - go h.handler(conn) - } - }(h) - } -} diff --git a/cli/agent.go b/cli/agent.go index 7c9daa8653961..5e4bd5e548a35 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -22,6 +22,7 @@ import ( "github.com/coder/coder/agent/reaper" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/codersdk" + "github.com/coder/coder/tailnet" "github.com/coder/retry" ) @@ -177,9 +178,11 @@ func workspaceAgent() *cobra.Command { // shells so "gitssh" works! "CODER_AGENT_TOKEN": client.SessionToken, }, - EnableWireguard: wireguard, - UploadWireguardKeys: client.UploadWorkspaceAgentKeys, - ListenWireguardPeers: client.WireguardPeerListener, + EnableWireguard: wireguard, + UpdateTailscaleNode: func(ctx context.Context, node *tailnet.Node) error { + return client.UpdateTailscaleNode(ctx, "me", node) + }, + ListenTailscaleNodes: client.ListenTailscaleNodes, }) <-cmd.Context().Done() return closer.Close() diff --git a/coderd/coderd.go b/coderd/coderd.go index acf39fd8bbead..245e187163f92 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -19,6 +19,10 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" "golang.org/x/xerrors" "google.golang.org/api/idtoken" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/tailcfg" + "tailscale.com/types/key" "cdr.dev/slog" "github.com/coder/coder/buildinfo" @@ -34,6 +38,7 @@ import ( "github.com/coder/coder/coderd/wsconncache" "github.com/coder/coder/codersdk" "github.com/coder/coder/site" + "github.com/coder/coder/tailnet" ) // Options are requires parameters for Coder to start. @@ -63,6 +68,8 @@ type Options struct { Telemetry telemetry.Reporter TURNServer *turnconn.Server TracerProvider *sdktrace.TracerProvider + + DERPMap *tailcfg.DERPMap } // New constructs a Coder API handler. @@ -103,6 +110,7 @@ func New(options *Options) *API { siteHandler: site.Handler(site.FS(), binFS), } api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgent, 0) + api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) apiKeyMiddleware := httpmw.ExtractAPIKey(options.Database, &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, @@ -131,6 +139,8 @@ func New(options *Options) *API { // other applications might not as well. r.Route("/%40{user}/{workspacename}/apps/{workspaceapp}", apps) r.Route("/@{user}/{workspacename}/apps/{workspaceapp}", apps) + r.Get("/derp", derphttp.Handler(api.derpServer).ServeHTTP) + r.Get("/derpmap", api.derpMap) r.Route("/api/v2", func(r chi.Router) { r.NotFound(func(rw http.ResponseWriter, r *http.Request) { @@ -312,9 +322,14 @@ func New(options *Options) *API { r.Get("/gitsshkey", api.agentGitSSHKey) r.Get("/turn", api.workspaceAgentTurn) r.Get("/iceservers", api.workspaceAgentICEServers) - r.Get("/wireguardlisten", api.workspaceAgentWireguardListener) - r.Post("/keys", api.postWorkspaceAgentKeys) r.Get("/derp", api.derpMap) + + // Posting map under "me" sets the agents node map. + + // On the agent side, "map" is a WebSocket that sends + // updates via marshalled JSON and recieves networking + // messages on the other end. + r.Get("/netmap", api.workspaceAgentSelfNetmap) }) r.Route("/{workspaceagent}", func(r chi.Router) { r.Use( @@ -323,12 +338,15 @@ func New(options *Options) *API { httpmw.ExtractWorkspaceParam(options.Database), ) r.Get("/", api.workspaceAgent) - r.Post("/peer", api.postWorkspaceAgentWireguardPeer) r.Get("/dial", api.workspaceAgentDial) r.Get("/turn", api.workspaceAgentTurn) r.Get("/pty", api.workspaceAgentPTY) r.Get("/iceservers", api.workspaceAgentICEServers) r.Get("/derp", api.derpMap) + + // Specific to Tailscale networking: + r.Post("/netmap") + r.Get("/tail-dial", api.workspaceAgentTailnetDial) }) }) r.Route("/workspaceresources/{workspaceresource}", func(r chi.Router) { @@ -385,6 +403,8 @@ func New(options *Options) *API { type API struct { *Options + derpServer *derp.Server + Handler chi.Router siteHandler http.Handler websocketWaitMutex sync.Mutex diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index 1a211f6bd69b2..b22ee495ca326 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -14,6 +14,11 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/goleak" "golang.org/x/xerrors" + "inet.af/netaddr" + "tailscale.com/tailcfg" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/buildinfo" "github.com/coder/coder/coderd/coderdtest" @@ -21,6 +26,7 @@ import ( "github.com/coder/coder/codersdk" "github.com/coder/coder/provisioner/echo" "github.com/coder/coder/provisionersdk/proto" + "github.com/coder/coder/tailnet" ) func TestMain(m *testing.M) { @@ -474,3 +480,71 @@ func (f *fakeAuthorizer) ByRoleName(_ context.Context, subjectID string, roleNam func (f *fakeAuthorizer) reset() { f.Called = nil } + +func TestDERP(t *testing.T) { + t.Parallel() + client := coderdtest.New(t, nil) + + logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) + + derpPort, err := strconv.Atoi(client.URL.Port()) + require.NoError(t, err) + derpMap := &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "cdr", + RegionName: "Coder", + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 1, + HostName: client.URL.Hostname(), + DERPPort: derpPort, + STUNPort: -1, + HTTPForTests: true, + }}, + }, + }, + } + w1IP := tailnet.IP() + w1, err := tailnet.New(&tailnet.Options{ + Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(w1IP, 128)}, + Logger: logger.Named("w1"), + DERPMap: derpMap, + }) + require.NoError(t, err) + + w2, err := tailnet.New(&tailnet.Options{ + Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(tailnet.IP(), 128)}, + Logger: logger.Named("w2"), + DERPMap: derpMap, + }) + require.NoError(t, err) + w1.SetNodeCallback(func(node *tailnet.Node) { + w2.UpdateNodes([]*tailnet.Node{node}) + }) + w2.SetNodeCallback(func(node *tailnet.Node) { + w1.UpdateNodes([]*tailnet.Node{node}) + }) + + conn := make(chan struct{}) + go func() { + listener, err := w1.Listen("tcp", ":35565") + assert.NoError(t, err) + defer listener.Close() + conn <- struct{}{} + nc, err := listener.Accept() + assert.NoError(t, err) + _ = nc.Close() + conn <- struct{}{} + }() + + <-conn + nc, err := w2.DialContextTCP(context.Background(), netaddr.IPPortFrom(w1IP, 35565)) + require.NoError(t, err) + _ = nc.Close() + <-conn + + w1.Close() + w2.Close() +} diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 644d74b513436..5d822c8b0442c 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -1604,23 +1604,21 @@ func (q *fakeQuerier) InsertWorkspaceAgent(_ context.Context, arg database.Inser defer q.mutex.Unlock() agent := database.WorkspaceAgent{ - ID: arg.ID, - CreatedAt: arg.CreatedAt, - UpdatedAt: arg.UpdatedAt, - ResourceID: arg.ResourceID, - AuthToken: arg.AuthToken, - AuthInstanceID: arg.AuthInstanceID, - EnvironmentVariables: arg.EnvironmentVariables, - Name: arg.Name, - Architecture: arg.Architecture, - OperatingSystem: arg.OperatingSystem, - Directory: arg.Directory, - StartupScript: arg.StartupScript, - InstanceMetadata: arg.InstanceMetadata, - ResourceMetadata: arg.ResourceMetadata, - WireguardNodeIPv6: arg.WireguardNodeIPv6, - WireguardNodePublicKey: arg.WireguardNodePublicKey, - WireguardDiscoPublicKey: arg.WireguardDiscoPublicKey, + ID: arg.ID, + CreatedAt: arg.CreatedAt, + UpdatedAt: arg.UpdatedAt, + ResourceID: arg.ResourceID, + AuthToken: arg.AuthToken, + AuthInstanceID: arg.AuthInstanceID, + EnvironmentVariables: arg.EnvironmentVariables, + Name: arg.Name, + Architecture: arg.Architecture, + OperatingSystem: arg.OperatingSystem, + Directory: arg.Directory, + StartupScript: arg.StartupScript, + InstanceMetadata: arg.InstanceMetadata, + ResourceMetadata: arg.ResourceMetadata, + IPAddresses: arg.IPAddresses, } q.provisionerJobAgents = append(q.provisionerJobAgents, agent) @@ -1916,7 +1914,7 @@ func (q *fakeQuerier) UpdateWorkspaceAgentConnectionByID(_ context.Context, arg return sql.ErrNoRows } -func (q *fakeQuerier) UpdateWorkspaceAgentKeysByID(_ context.Context, arg database.UpdateWorkspaceAgentKeysByIDParams) error { +func (q *fakeQuerier) UpdateWorkspaceAgentNetworkByID(_ context.Context, arg database.UpdateWorkspaceAgentNetworkByIDParams) error { q.mutex.Lock() defer q.mutex.Unlock() @@ -1925,8 +1923,10 @@ func (q *fakeQuerier) UpdateWorkspaceAgentKeysByID(_ context.Context, arg databa continue } - agent.WireguardNodePublicKey = arg.WireguardNodePublicKey - agent.WireguardDiscoPublicKey = arg.WireguardDiscoPublicKey + agent.DiscoPublicKey = arg.DiscoPublicKey + agent.NodePublicKey = arg.NodePublicKey + agent.DERP = arg.DERP + agent.DERPLatency = arg.DERPLatency agent.UpdatedAt = database.Now() q.provisionerJobAgents[index] = agent return nil diff --git a/coderd/database/dbtypes/dbtypes.go b/coderd/database/dbtypes/dbtypes.go index 3653f4f37cb62..a158587269476 100644 --- a/coderd/database/dbtypes/dbtypes.go +++ b/coderd/database/dbtypes/dbtypes.go @@ -1,74 +1,6 @@ package dbtypes -import ( - "database/sql/driver" - - "golang.org/x/xerrors" - "tailscale.com/types/key" -) - -// NodePublic is a wrapper around a key.NodePublic which represents the -// Wireguard public key for an agent.. -type NodePublic key.NodePublic - -func (n NodePublic) String() string { - return key.NodePublic(n).String() -} - -// This is necessary so NodePublic can be serialized in JSON loggers. -func (n NodePublic) MarshalJSON() ([]byte, error) { - j, err := key.NodePublic(n).MarshalText() - // surround in quotes to make it a JSON string - j = append([]byte{'"'}, append(j, '"')...) - return j, err -} - -// Value is so NodePublic can be inserted into the database. -func (n NodePublic) Value() (driver.Value, error) { - return key.NodePublic(n).MarshalText() -} - -// Scan is so NodePublic can be read from the database. -func (n *NodePublic) Scan(value interface{}) error { - switch v := value.(type) { - case []byte: - return (*key.NodePublic)(n).UnmarshalText(v) - case string: - return (*key.NodePublic)(n).UnmarshalText([]byte(v)) - default: - return xerrors.Errorf("unexpected type: %T", v) - } -} - -// NodePublic is a wrapper around a key.NodePublic which represents the -// Tailscale disco key for an agent. -type DiscoPublic key.DiscoPublic - -func (n DiscoPublic) String() string { - return key.DiscoPublic(n).String() -} - -// This is necessary so DiscoPublic can be serialized in JSON loggers. -func (n DiscoPublic) MarshalJSON() ([]byte, error) { - j, err := key.DiscoPublic(n).MarshalText() - // surround in quotes to make it a JSON string - j = append([]byte{'"'}, append(j, '"')...) - return j, err -} - -// Value is so DiscoPublic can be inserted into the database. -func (n DiscoPublic) Value() (driver.Value, error) { - return key.DiscoPublic(n).MarshalText() -} - -// Scan is so DiscoPublic can be read from the database. -func (n *DiscoPublic) Scan(value interface{}) error { - switch v := value.(type) { - case []byte: - return (*key.DiscoPublic)(n).UnmarshalText(v) - case string: - return (*key.DiscoPublic)(n).UnmarshalText([]byte(v)) - default: - return xerrors.Errorf("unexpected type: %T", v) - } -} +// DERPLatency represents a KV mapping of latency to DERP servers. +// This type is only used for generation. sqlc doesn't support +// complex Go types. +type DERPLatency map[string]float64 diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index b9d6b57762609..1cac38cdb4713 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -293,9 +293,11 @@ CREATE TABLE workspace_agents ( instance_metadata jsonb, resource_metadata jsonb, directory character varying(4096) DEFAULT ''::character varying NOT NULL, - wireguard_node_ipv6 inet DEFAULT '::'::inet NOT NULL, - wireguard_node_public_key character varying(128) DEFAULT 'nodekey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, - wireguard_disco_public_key character varying(128) DEFAULT 'discokey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL + node_public_key character varying(128) DEFAULT 'nodekey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, + disco_public_key character varying(128) DEFAULT 'discokey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, + ip_addresses inet[] DEFAULT ARRAY[]::inet[] NOT NULL, + derp character varying(128) DEFAULT '127.3.3.40:0'::character varying NOT NULL, + derp_latency jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE workspace_apps ( diff --git a/coderd/database/dump/main.go b/coderd/database/dump/main.go index 802fcedc38b2e..9ee2ad917aafa 100644 --- a/coderd/database/dump/main.go +++ b/coderd/database/dump/main.go @@ -2,14 +2,12 @@ package main import ( "bytes" - "database/sql" "fmt" "os" "os/exec" "path/filepath" "runtime" - "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/database/postgres" ) @@ -20,16 +18,6 @@ func main() { } defer closeFn() - db, err := sql.Open("postgres", connection) - if err != nil { - panic(err) - } - - err = database.MigrateUp(db) - if err != nil { - panic(err) - } - cmd := exec.Command( "pg_dump", "--schema-only", diff --git a/coderd/database/migrations/000029_tailnet.down.sql b/coderd/database/migrations/000029_tailnet.down.sql new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/coderd/database/migrations/000029_tailnet.up.sql b/coderd/database/migrations/000029_tailnet.up.sql new file mode 100644 index 0000000000000..5ac9f8af06e20 --- /dev/null +++ b/coderd/database/migrations/000029_tailnet.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE workspace_agents DROP COLUMN wireguard_node_ipv6; +ALTER TABLE workspace_agents ADD COLUMN ip_addresses inet[] NOT NULL DEFAULT array[]::inet[]; +ALTER TABLE workspace_agents RENAME COLUMN wireguard_node_public_key TO node_public_key; +ALTER TABLE workspace_agents RENAME COLUMN wireguard_disco_public_key TO disco_public_key; +ALTER TABLE workspace_agents ADD COLUMN derp varchar(128) NOT NULL DEFAULT '127.3.3.40:0'; +ALTER TABLE workspace_agents ADD COLUMN derp_latency jsonb NOT NULL DEFAULT '{}'; diff --git a/coderd/database/models.go b/coderd/database/models.go index 660a7620df454..ac3a1f7d990c1 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -504,26 +504,28 @@ type Workspace struct { } type WorkspaceAgent struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Name string `db:"name" json:"name"` - FirstConnectedAt sql.NullTime `db:"first_connected_at" json:"first_connected_at"` - LastConnectedAt sql.NullTime `db:"last_connected_at" json:"last_connected_at"` - DisconnectedAt sql.NullTime `db:"disconnected_at" json:"disconnected_at"` - ResourceID uuid.UUID `db:"resource_id" json:"resource_id"` - AuthToken uuid.UUID `db:"auth_token" json:"auth_token"` - AuthInstanceID sql.NullString `db:"auth_instance_id" json:"auth_instance_id"` - Architecture string `db:"architecture" json:"architecture"` - EnvironmentVariables pqtype.NullRawMessage `db:"environment_variables" json:"environment_variables"` - OperatingSystem string `db:"operating_system" json:"operating_system"` - StartupScript sql.NullString `db:"startup_script" json:"startup_script"` - InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` - ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` - Directory string `db:"directory" json:"directory"` - WireguardNodeIPv6 pqtype.Inet `db:"wireguard_node_ipv6" json:"wireguard_node_ipv6"` - WireguardNodePublicKey dbtypes.NodePublic `db:"wireguard_node_public_key" json:"wireguard_node_public_key"` - WireguardDiscoPublicKey dbtypes.DiscoPublic `db:"wireguard_disco_public_key" json:"wireguard_disco_public_key"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Name string `db:"name" json:"name"` + FirstConnectedAt sql.NullTime `db:"first_connected_at" json:"first_connected_at"` + LastConnectedAt sql.NullTime `db:"last_connected_at" json:"last_connected_at"` + DisconnectedAt sql.NullTime `db:"disconnected_at" json:"disconnected_at"` + ResourceID uuid.UUID `db:"resource_id" json:"resource_id"` + AuthToken uuid.UUID `db:"auth_token" json:"auth_token"` + AuthInstanceID sql.NullString `db:"auth_instance_id" json:"auth_instance_id"` + Architecture string `db:"architecture" json:"architecture"` + EnvironmentVariables pqtype.NullRawMessage `db:"environment_variables" json:"environment_variables"` + OperatingSystem string `db:"operating_system" json:"operating_system"` + StartupScript sql.NullString `db:"startup_script" json:"startup_script"` + InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` + ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` + Directory string `db:"directory" json:"directory"` + NodePublicKey string `db:"node_public_key" json:"node_public_key"` + DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` + IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` + DERP string `db:"derp" json:"derp"` + DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } type WorkspaceApp struct { diff --git a/coderd/database/postgres/postgres.go b/coderd/database/postgres/postgres.go index d1ef7b3084197..53adbdb2431de 100644 --- a/coderd/database/postgres/postgres.go +++ b/coderd/database/postgres/postgres.go @@ -75,7 +75,7 @@ func Open() (string, func(), error) { resource, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "postgres", - Tag: "13", + Tag: "11", Env: []string{ "POSTGRES_PASSWORD=postgres", "POSTGRES_USER=postgres", @@ -134,16 +134,21 @@ func Open() (string, func(), error) { if err != nil { return xerrors.Errorf("ping postgres: %w", err) } - err = database.MigrateUp(db) - if err != nil { - return xerrors.Errorf("migrate db: %w", err) - } - return nil }) if err != nil { return "", nil, err } + db, err := sql.Open("postgres", dbURL) + if err != nil { + return "", nil, xerrors.Errorf("open postgres: %w", err) + } + defer db.Close() + err = database.MigrateUp(db) + if err != nil { + return "", nil, xerrors.Errorf("migrate db: %w", err) + } + return dbURL, func() { _ = pool.Purge(resource) _ = os.RemoveAll(tempDir) diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 1410c536e3db5..27edf31c3f4e8 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -127,7 +127,7 @@ type querier interface { UpdateUserRoles(ctx context.Context, arg UpdateUserRolesParams) (User, error) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusParams) (User, error) UpdateWorkspaceAgentConnectionByID(ctx context.Context, arg UpdateWorkspaceAgentConnectionByIDParams) error - UpdateWorkspaceAgentKeysByID(ctx context.Context, arg UpdateWorkspaceAgentKeysByIDParams) error + UpdateWorkspaceAgentNetworkByID(ctx context.Context, arg UpdateWorkspaceAgentNetworkByIDParams) error UpdateWorkspaceAutostart(ctx context.Context, arg UpdateWorkspaceAutostartParams) error UpdateWorkspaceBuildByID(ctx context.Context, arg UpdateWorkspaceBuildByIDParams) error UpdateWorkspaceDeletedByID(ctx context.Context, arg UpdateWorkspaceDeletedByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index e8589f4629937..6bc8fc8473e29 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2851,7 +2851,7 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP const getWorkspaceAgentByAuthToken = `-- name: GetWorkspaceAgentByAuthToken :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE @@ -2881,16 +2881,18 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ) return i, err } const getWorkspaceAgentByID = `-- name: GetWorkspaceAgentByID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE @@ -2918,16 +2920,18 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ) return i, err } const getWorkspaceAgentByInstanceID = `-- name: GetWorkspaceAgentByInstanceID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE @@ -2957,16 +2961,18 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ) return i, err } const getWorkspaceAgentsByResourceIDs = `-- name: GetWorkspaceAgentsByResourceIDs :many SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE @@ -3000,9 +3006,11 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ); err != nil { return nil, err } @@ -3018,7 +3026,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] } const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many -SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key FROM workspace_agents WHERE created_at > $1 +SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE created_at > $1 ` func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) { @@ -3048,9 +3056,11 @@ func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, created &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ); err != nil { return nil, err } @@ -3082,32 +3092,28 @@ INSERT INTO directory, instance_metadata, resource_metadata, - wireguard_node_ipv6, - wireguard_node_public_key, - wireguard_disco_public_key + ip_addresses ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, wireguard_node_ipv6, wireguard_node_public_key, wireguard_disco_public_key + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency ` type InsertWorkspaceAgentParams struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - Name string `db:"name" json:"name"` - ResourceID uuid.UUID `db:"resource_id" json:"resource_id"` - AuthToken uuid.UUID `db:"auth_token" json:"auth_token"` - AuthInstanceID sql.NullString `db:"auth_instance_id" json:"auth_instance_id"` - Architecture string `db:"architecture" json:"architecture"` - EnvironmentVariables pqtype.NullRawMessage `db:"environment_variables" json:"environment_variables"` - OperatingSystem string `db:"operating_system" json:"operating_system"` - StartupScript sql.NullString `db:"startup_script" json:"startup_script"` - Directory string `db:"directory" json:"directory"` - InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` - ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` - WireguardNodeIPv6 pqtype.Inet `db:"wireguard_node_ipv6" json:"wireguard_node_ipv6"` - WireguardNodePublicKey dbtypes.NodePublic `db:"wireguard_node_public_key" json:"wireguard_node_public_key"` - WireguardDiscoPublicKey dbtypes.DiscoPublic `db:"wireguard_disco_public_key" json:"wireguard_disco_public_key"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + Name string `db:"name" json:"name"` + ResourceID uuid.UUID `db:"resource_id" json:"resource_id"` + AuthToken uuid.UUID `db:"auth_token" json:"auth_token"` + AuthInstanceID sql.NullString `db:"auth_instance_id" json:"auth_instance_id"` + Architecture string `db:"architecture" json:"architecture"` + EnvironmentVariables pqtype.NullRawMessage `db:"environment_variables" json:"environment_variables"` + OperatingSystem string `db:"operating_system" json:"operating_system"` + StartupScript sql.NullString `db:"startup_script" json:"startup_script"` + Directory string `db:"directory" json:"directory"` + InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` + ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` + IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` } func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspaceAgentParams) (WorkspaceAgent, error) { @@ -3126,9 +3132,7 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa arg.Directory, arg.InstanceMetadata, arg.ResourceMetadata, - arg.WireguardNodeIPv6, - arg.WireguardNodePublicKey, - arg.WireguardDiscoPublicKey, + pq.Array(arg.IPAddresses), ) var i WorkspaceAgent err := row.Scan( @@ -3149,9 +3153,11 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - &i.WireguardNodeIPv6, - &i.WireguardNodePublicKey, - &i.WireguardDiscoPublicKey, + &i.NodePublicKey, + &i.DiscoPublicKey, + pq.Array(&i.IPAddresses), + &i.DERP, + &i.DERPLatency, ) return i, err } @@ -3185,25 +3191,37 @@ func (q *sqlQuerier) UpdateWorkspaceAgentConnectionByID(ctx context.Context, arg return err } -const updateWorkspaceAgentKeysByID = `-- name: UpdateWorkspaceAgentKeysByID :exec +const updateWorkspaceAgentNetworkByID = `-- name: UpdateWorkspaceAgentNetworkByID :exec UPDATE workspace_agents SET - updated_at = now(), - wireguard_node_public_key = $2, - wireguard_disco_public_key = $3 + updated_at = $2, + node_public_key = $3, + disco_public_key = $4, + derp = $5, + derp_latency = $6 WHERE id = $1 ` -type UpdateWorkspaceAgentKeysByIDParams struct { - ID uuid.UUID `db:"id" json:"id"` - WireguardNodePublicKey dbtypes.NodePublic `db:"wireguard_node_public_key" json:"wireguard_node_public_key"` - WireguardDiscoPublicKey dbtypes.DiscoPublic `db:"wireguard_disco_public_key" json:"wireguard_disco_public_key"` +type UpdateWorkspaceAgentNetworkByIDParams struct { + ID uuid.UUID `db:"id" json:"id"` + UpdatedAt time.Time `db:"updated_at" json:"updated_at"` + NodePublicKey string `db:"node_public_key" json:"node_public_key"` + DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` + DERP string `db:"derp" json:"derp"` + DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } -func (q *sqlQuerier) UpdateWorkspaceAgentKeysByID(ctx context.Context, arg UpdateWorkspaceAgentKeysByIDParams) error { - _, err := q.db.ExecContext(ctx, updateWorkspaceAgentKeysByID, arg.ID, arg.WireguardNodePublicKey, arg.WireguardDiscoPublicKey) +func (q *sqlQuerier) UpdateWorkspaceAgentNetworkByID(ctx context.Context, arg UpdateWorkspaceAgentNetworkByIDParams) error { + _, err := q.db.ExecContext(ctx, updateWorkspaceAgentNetworkByID, + arg.ID, + arg.UpdatedAt, + arg.NodePublicKey, + arg.DiscoPublicKey, + arg.DERP, + arg.DERPLatency, + ) return err } diff --git a/coderd/database/queries/workspaceagents.sql b/coderd/database/queries/workspaceagents.sql index 37e7754739311..23dbf4a776bca 100644 --- a/coderd/database/queries/workspaceagents.sql +++ b/coderd/database/queries/workspaceagents.sql @@ -54,12 +54,10 @@ INSERT INTO directory, instance_metadata, resource_metadata, - wireguard_node_ipv6, - wireguard_node_public_key, - wireguard_disco_public_key + ip_addresses ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *; -- name: UpdateWorkspaceAgentConnectionByID :exec UPDATE @@ -72,12 +70,14 @@ SET WHERE id = $1; --- name: UpdateWorkspaceAgentKeysByID :exec +-- name: UpdateWorkspaceAgentNetworkByID :exec UPDATE workspace_agents SET - updated_at = now(), - wireguard_node_public_key = $2, - wireguard_disco_public_key = $3 + updated_at = $2, + node_public_key = $3, + disco_public_key = $4, + derp = $5, + derp_latency = $6 WHERE id = $1; diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 6f42bbaa4bd2c..5c8dbf37f3173 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -17,10 +17,8 @@ packages: output_db_file_name: db_tmp.go overrides: - - column: workspace_agents.wireguard_node_public_key - go_type: github.com/coder/coder/coderd/database/dbtypes.NodePublic - - column: workspace_agents.wireguard_disco_public_key - go_type: github.com/coder/coder/coderd/database/dbtypes.DiscoPublic + - column: workspace_agents.derp_latency + go_type: github.com/coder/coder/coderd/database/dbtypes.DERPLatency rename: api_key: APIKey @@ -34,4 +32,6 @@ rename: gitsshkey: GitSSHKey rbac_roles: RBACRoles ip_address: IPAddress - wireguard_node_ipv6: WireguardNodeIPv6 + ip_addresses: IPAddresses + derp: DERP + derp_latency: DERPLatency diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index 7a013e15d755d..173da096924de 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "reflect" @@ -23,15 +24,14 @@ import ( "cdr.dev/slog" "github.com/coder/coder/coderd/database" - "github.com/coder/coder/coderd/database/dbtypes" "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/parameter" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/telemetry" - "github.com/coder/coder/peer/peerwg" "github.com/coder/coder/provisionerd/proto" "github.com/coder/coder/provisionersdk" sdkproto "github.com/coder/coder/provisionersdk/proto" + "github.com/coder/coder/tailnet" ) func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { @@ -744,6 +744,7 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. } agentID := uuid.New() + ip := tailnet.IP().As16() dbAgent, err := db.InsertWorkspaceAgent(ctx, database.InsertWorkspaceAgentParams{ ID: agentID, CreatedAt: database.Now(), @@ -760,9 +761,14 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. String: prAgent.StartupScript, Valid: prAgent.StartupScript != "", }, - WireguardNodeIPv6: peerwg.UUIDToInet(agentID), - WireguardNodePublicKey: dbtypes.NodePublic{}, - WireguardDiscoPublicKey: dbtypes.DiscoPublic{}, + // Generate a new random IP! + IPAddresses: []pqtype.Inet{{ + Valid: true, + IPNet: net.IPNet{ + IP: ip[:], + Mask: net.CIDRMask(128, 128), + }, + }}, }) if err != nil { return xerrors.Errorf("insert agent: %w", err) diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 0701c21483dfa..fe2f42a6757e5 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -1,8 +1,10 @@ package coderd import ( + "bytes" "context" "database/sql" + "encoding/base64" "encoding/json" "fmt" "io" @@ -14,25 +16,26 @@ import ( "github.com/google/uuid" "github.com/hashicorp/yamux" "github.com/tabbed/pqtype" + "go4.org/mem" "golang.org/x/xerrors" "inet.af/netaddr" "nhooyr.io/websocket" + "tailscale.com/tailcfg" "tailscale.com/types/key" "cdr.dev/slog" "github.com/coder/coder/agent" "github.com/coder/coder/coderd/database" - "github.com/coder/coder/coderd/database/dbtypes" "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/codersdk" "github.com/coder/coder/peer" - "github.com/coder/coder/peer/peerwg" "github.com/coder/coder/peerbroker" "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" + "github.com/coder/coder/tailnet" ) func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) { @@ -162,7 +165,7 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) return } - ipp, ok := netaddr.FromStdIPNet(&workspaceAgent.WireguardNodeIPv6.IPNet) + ipp, ok := netaddr.FromStdIPNet(&workspaceAgent.IPAddresses.IPNet) if !ok { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ Message: "Workspace agent has an invalid ipv6 address.", @@ -172,7 +175,8 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) } httpapi.Write(rw, http.StatusOK, agent.Metadata{ - WireguardAddresses: []netaddr.IPPrefix{ipp}, + TailscaleAddresses: []netaddr.IPPrefix{ipp}, + OwnerEmail: owner.Email, OwnerUsername: owner.Username, EnvironmentVariables: apiAgent.EnvironmentVariables, @@ -468,92 +472,51 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { _, _ = io.Copy(ptNetConn, wsNetConn) } -func (*API) derpMap(rw http.ResponseWriter, _ *http.Request) { - httpapi.Write(rw, http.StatusOK, peerwg.DerpMap) -} - -type WorkspaceKeysRequest struct { - Public key.NodePublic `json:"public"` - Disco key.DiscoPublic `json:"disco"` -} - -func (api *API) postWorkspaceAgentKeys(rw http.ResponseWriter, r *http.Request) { - var ( - ctx = r.Context() - workspaceAgent = httpmw.WorkspaceAgent(r) - keys WorkspaceKeysRequest - ) - if !httpapi.Read(rw, r, &keys) { - return +func (a *API) derpMap(rw http.ResponseWriter, _ *http.Request) { + var derpPort int + rawPort := a.AccessURL.Port() + if rawPort == "" { + if a.AccessURL.Scheme == "https" { + derpPort = 443 + } else { + derpPort = 80 + } + } else { + var err error + derpPort, err = strconv.Atoi(a.AccessURL.Port()) + if err != nil { + httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + Message: "Get ", + }) + return + } } - err := api.Database.UpdateWorkspaceAgentKeysByID(ctx, database.UpdateWorkspaceAgentKeysByIDParams{ - ID: workspaceAgent.ID, - WireguardNodePublicKey: dbtypes.NodePublic(keys.Public), - WireguardDiscoPublicKey: dbtypes.DiscoPublic(keys.Disco), + httpapi.Write(rw, http.StatusOK, &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: &tailcfg.DERPRegion{ + RegionID: 1, + RegionCode: "coder", + RegionName: "Coder", + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 1, + HostName: a.AccessURL.Hostname(), + DERPPort: derpPort, + STUNPort: -1, + }}, + }, + }, }) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Internal error setting agent keys.", - Detail: err.Error(), - }) - return - } - - rw.WriteHeader(http.StatusNoContent) } -func (api *API) postWorkspaceAgentWireguardPeer(rw http.ResponseWriter, r *http.Request) { - var ( - req peerwg.Handshake - workspaceAgent = httpmw.WorkspaceAgentParam(r) - workspace = httpmw.WorkspaceParam(r) - ) - - if !api.Authorize(r, rbac.ActionUpdate, workspace) { - httpapi.ResourceNotFound(rw) - return - } - - if !httpapi.Read(rw, r, &req) { - return - } - - if req.Recipient != workspaceAgent.ID { - httpapi.Write(rw, http.StatusBadRequest, httpapi.Response{ - Message: "Invalid recipient.", - }) - return - } - - raw, err := req.MarshalText() - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Internal error marshaling wireguard peer message.", - Detail: err.Error(), - }) - return - } - - err = api.Pubsub.Publish("wireguard_peers", raw) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Internal error publishing wireguard peer message.", - Detail: err.Error(), - }) - return - } - - rw.WriteHeader(http.StatusNoContent) -} - -func (api *API) workspaceAgentWireguardListener(rw http.ResponseWriter, r *http.Request) { +// workspaceAgentSelfNetmap accepts a WebSocket that reads +// node updates and sends new node connection info. +func (api *API) workspaceAgentSelfNetmap(rw http.ResponseWriter, r *http.Request) { api.websocketWaitMutex.Lock() api.websocketWaitGroup.Add(1) api.websocketWaitMutex.Unlock() defer api.websocketWaitGroup.Done() - - ctx := r.Context() workspaceAgent := httpmw.WorkspaceAgent(r) conn, err := websocket.Accept(rw, r, nil) @@ -565,24 +528,21 @@ func (api *API) workspaceAgentWireguardListener(rw http.ResponseWriter, r *http. return } defer conn.Close(websocket.StatusNormalClosure, "") - + ctx, nc := websocketNetConn(r.Context(), conn, websocket.MessageBinary) agentIDBytes, _ := workspaceAgent.ID.MarshalText() - subCancel, err := api.Pubsub.Subscribe("wireguard_peers", func(ctx context.Context, message []byte) { + subCancel, err := api.Pubsub.Subscribe("tailnet_dial", func(ctx context.Context, message []byte) { // Since we subscribe to all peer broadcasts, we do a light check to // make sure we're the intended recipient without fully decoding the // message. - hint, err := peerwg.HandshakeRecipientHint(agentIDBytes, message) - if err != nil { - api.Logger.Error(ctx, "invalid wireguard peer message", slog.Error(err)) + if len(message) < len(agentIDBytes) { + api.Logger.Error(ctx, "wireguard peer message too short", slog.F("got", len(message))) return } - // We aren't the intended recipient. - if !hint { + if !bytes.Equal(message[:len(agentIDBytes)-1], agentIDBytes) { return } - - _ = conn.Write(ctx, websocket.MessageBinary, message) + _, _ = nc.Write(message) }) if err != nil { api.Logger.Error(ctx, "pubsub listen", slog.Error(err)) @@ -590,9 +550,45 @@ func (api *API) workspaceAgentWireguardListener(rw http.ResponseWriter, r *http. } defer subCancel() - // Wait for the connection to close or the client to send a message. - //nolint:dogsled - _, _, _ = conn.Reader(ctx) + decoder := json.NewDecoder(nc) + for { + var node tailnet.Node + err = decoder.Decode(&node) + if err != nil { + return + } + err := api.Database.UpdateWorkspaceAgentNetworkByID(ctx, database.UpdateWorkspaceAgentNetworkByIDParams{ + ID: workspaceAgent.ID, + NodePublicKey: node.Key.String(), + DERPLatency: , + TailnetNodePublicKey: node.Key.String(), + TailnetDiscoPublicKey: node.DiscoKey.String(), + TailnetNodeDERP: node.DERP, + UpdatedAt: database.Now(), + }) + if err != nil { + httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + Message: "Internal error setting agent keys.", + Detail: err.Error(), + }) + return + } + nodeData, err := json.Marshal(node) + if err != nil { + return + } + id, _ := workspaceAgent.ID.MarshalText() + msg := base64.StdEncoding.EncodeToString(nodeData) + err = api.Pubsub.Publish("tailnet_listen", append(id, msg...)) + if err != nil { + conn.Close(websocket.StatusAbnormalClosure, err.Error()) + return + } + } +} + +func (api *API) workspaceAgentNetmap(r *http.Request, rw http.ResponseWriter) { + } // dialWorkspaceAgent connects to a workspace agent by ID. Only rely on @@ -697,7 +693,21 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work return codersdk.WorkspaceAgent{}, xerrors.Errorf("unmarshal: %w", err) } } - + nodePublicKey, err := key.ParseNodePublicUntyped(mem.S(dbAgent.NodePublicKey)) + if err != nil { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse node public key: %w", err) + } + var discoPublicKey key.DiscoPublic + err = discoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey)) + if err != nil { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse disco public key: %w", err) + } + ips := make([]netaddr.IP, 0) + for _, ip := range dbAgent.IPAddresses { + var ipData [16]byte + copy(ipData[:], []byte(ip.IPNet.IP)) + ips = append(ips, netaddr.IPFrom16(ipData)) + } workspaceAgent := codersdk.WorkspaceAgent{ ID: dbAgent.ID, CreatedAt: dbAgent.CreatedAt, @@ -711,9 +721,11 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work EnvironmentVariables: envs, Directory: dbAgent.Directory, Apps: apps, - IPv6: inetToNetaddr(dbAgent.WireguardNodeIPv6), - WireguardPublicKey: key.NodePublic(dbAgent.WireguardNodePublicKey), - DiscoPublicKey: key.DiscoPublic(dbAgent.WireguardDiscoPublicKey), + NodePublicKey: nodePublicKey, + DiscoPublicKey: discoPublicKey, + DERP: dbAgent.DERP, + DERPLatency: dbAgent.DERPLatency, + IPAddresses: ips, } if dbAgent.FirstConnectedAt.Valid { diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 499286d0c91a3..c50b444947cec 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -23,10 +23,10 @@ import ( "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/peer" - "github.com/coder/coder/peer/peerwg" "github.com/coder/coder/peerbroker" "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" + "github.com/coder/coder/tailnet" ) type GoogleInstanceIdentityToken struct { @@ -255,11 +255,10 @@ func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) ( // PostWireguardPeer announces your public keys and IPv6 address to the // specified recipient. -func (c *Client) PostWireguardPeer(ctx context.Context, workspaceID uuid.UUID, peerMsg peerwg.Handshake) error { - res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/peer?workspace=%s", - peerMsg.Recipient, - workspaceID.String(), - ), peerMsg) +func (c *Client) UpdateTailscaleNode(ctx context.Context, agentID string, node *tailnet.Node) error { + res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/peer", + agentID, + ), node) if err != nil { return err } @@ -272,10 +271,10 @@ func (c *Client) PostWireguardPeer(ctx context.Context, workspaceID uuid.UUID, p return nil } -// WireguardPeerListener listens for wireguard peer messages. Peer messages are +// ListenTailscaleNodes listens for Tailscale node updates. Peer messages are // sent when a new client wants to connect. Once receiving a peer message, the // peer should be added to the NetworkMap of the wireguard interface. -func (c *Client) WireguardPeerListener(ctx context.Context, logger slog.Logger) (<-chan peerwg.Handshake, func(), error) { +func (c *Client) ListenTailscaleNodes(ctx context.Context, logger slog.Logger) (<-chan *tailnet.Node, func(), error) { serverURL, err := c.URL.Parse("/api/v2/workspaceagents/me/wireguardlisten") if err != nil { return nil, nil, xerrors.Errorf("parse url: %w", err) @@ -304,46 +303,25 @@ func (c *Client) WireguardPeerListener(ctx context.Context, logger slog.Logger) return nil, nil, readBodyAsError(res) } - ch := make(chan peerwg.Handshake, 1) + ch := make(chan *tailnet.Node, 1) go func() { defer conn.Close(websocket.StatusGoingAway, "") defer close(ch) + decoder := json.NewDecoder(websocket.NetConn(ctx, conn, websocket.MessageBinary)) for { - _, message, err := conn.Read(ctx) + var node *tailnet.Node + err = decoder.Decode(node) if err != nil { break } - - var msg peerwg.Handshake - err = msg.UnmarshalText(message) - if err != nil { - logger.Error(ctx, "unmarshal wireguard peer message", slog.Error(err)) - continue - } - - ch <- msg + ch <- node } }() return ch, func() { _ = conn.Close(websocket.StatusGoingAway, "") }, nil } -// UploadWorkspaceAgentKeys uploads the public keys of the workspace agent that -// were generated on startup. These keys are used by clients to communicate with -// the workspace agent over the wireguard interface. -func (c *Client) UploadWorkspaceAgentKeys(ctx context.Context, keys agent.WireguardPublicKeys) error { - res, err := c.Request(ctx, http.MethodPost, "/api/v2/workspaceagents/me/keys", keys) - if err != nil { - return xerrors.Errorf("do request: %w", err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - return readBodyAsError(res) - } - return nil -} - // DialWorkspaceAgent creates a connection to the specified resource. func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *peer.ConnOptions) (*agent.Conn, error) { serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/dial", agentID.String())) diff --git a/codersdk/workspaceresources.go b/codersdk/workspaceresources.go index 10e83e5094bb7..24da0f38c8c32 100644 --- a/codersdk/workspaceresources.go +++ b/codersdk/workspaceresources.go @@ -47,9 +47,16 @@ type WorkspaceAgent struct { StartupScript string `json:"startup_script,omitempty"` Directory string `json:"directory,omitempty"` Apps []WorkspaceApp `json:"apps"` - WireguardPublicKey key.NodePublic `json:"wireguard_public_key"` - DiscoPublicKey key.DiscoPublic `json:"disco_public_key"` - IPv6 netaddr.IPPrefix `json:"ipv6"` + + // For internal routing only. + IPAddresses []netaddr.IP `json:"ip_addresses"` + NodePublicKey key.NodePublic `json:"node_public_key"` + DiscoPublicKey key.DiscoPublic `json:"disco_public_key"` + // DERP represents the connected region. + DERP string `json:"derp"` + // Maps DERP region to MS latency. + // Fetch the DERP mapping to extract region names! + DERPLatency map[string]float64 `json:"latency"` } type WorkspaceAgentResourceMetadata struct { diff --git a/go.mod b/go.mod index dfe8854ec68f6..ba63a41b5889d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,9 @@ replace github.com/chzyer/readline => github.com/kylecarbs/readline v0.0.0-20220 // Required until https://github.com/briandowns/spinner/pull/136 is merged. replace github.com/briandowns/spinner => github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e +// Required until is merged. +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae + // Required until https://github.com/storj/drpc/pull/31 is merged. replace storj.io/drpc => github.com/kylecarbs/drpc v0.0.31-0.20220424193521-8ebbaf48bdff @@ -130,11 +133,11 @@ require ( google.golang.org/protobuf v1.28.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 - inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 + inet.af/netaddr v0.0.0-20220617031823-097006376321 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 nhooyr.io/websocket v1.8.7 storj.io/drpc v0.0.30 - tailscale.com v1.26.0 + tailscale.com v1.1.1-0.20220628235937-06aa14163254 ) require github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect @@ -277,7 +280,7 @@ require ( go.opentelemetry.io/proto/otlp v0.16.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect - go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 // indirect + go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect golang.zx2c4.com/wireguard/windows v0.4.10 // indirect @@ -285,6 +288,6 @@ require ( google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad // indirect google.golang.org/grpc v1.47.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gvisor.dev/gvisor v0.0.0-20220407223209-21871174d445 // indirect + gvisor.dev/gvisor v0.0.0-20220407223209-21871174d445 howett.net/plist v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index 4173a31e928a3..fff59df6a7371 100644 --- a/go.sum +++ b/go.sum @@ -348,6 +348,8 @@ github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1 h1:UqBrPWSYvRI2s5RtOu github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= +github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae h1:YfsqHcNJqKh6raG8tmIlplV4kpUs7uFrKl6prsNC8Ps= +github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae/go.mod h1:8/bTk326TtqBzoJ4xuZPSSGllAZBtDkkVi1GYkciKTY= github.com/coder/wireguard-go/tun/netstack v0.0.0-20220614153727-d82b4ba8619f h1:wsrm7hB9cvvw8ybX41YjzXDMbpo3gjlesw7oHYhtZW4= github.com/coder/wireguard-go/tun/netstack v0.0.0-20220614153727-d82b4ba8619f/go.mod h1:PerNzwKlnUUbKSRrSghbyhE9wEl3xakvPY9muprxlv8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= @@ -1967,8 +1969,9 @@ go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3m go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/mem v0.0.0-20210711025021-927187094b94 h1:OAAkygi2Js191AJP1Ds42MhJRgeofeKGjuoUqNp1QC4= go4.org/mem v0.0.0-20210711025021-927187094b94/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37 h1:Tx9kY6yUkLge/pFG7IEMwDZy6CS2ajFc9TvQdPCW0uA= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2806,8 +2809,8 @@ honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83 h1:lZ9GIYaU+o5+X6ST702I/Ntyq9Y2oIMZ42rBQpem64A= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 h1:acCzuUSQ79tGsM/O50VRFySfMm19IoMKL+sZztZkCxw= -inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6/go.mod h1:y3MGhcFMlh0KZPMuXXow8mpjxxAk3yoDNsp4cQz54i8= +inet.af/netaddr v0.0.0-20220617031823-097006376321 h1:B4dC8ySKTQXasnjDTMsoCMf1sQG4WsMej0WXaHxunmU= +inet.af/netaddr v0.0.0-20220617031823-097006376321/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= @@ -2900,5 +2903,3 @@ sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZa sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= software.sslmate.com/src/go-pkcs12 v0.0.0-20210415151418-c5206de65a78 h1:SqYE5+A2qvRhErbsXFfUEUmpWEKxxRSMgGLkvRAFOV4= -tailscale.com v1.26.0 h1:P52fXdVSBjeMvlCSZew8roeFTzO+dv0MQwTrVnLEKdw= -tailscale.com v1.26.0/go.mod h1:fEHdIssBhv1Ax889L8TNPDlECCcDpqYm/M+wwVbFFvw= diff --git a/peer/peerwg/derp.go b/peer/peerwg/derp.go index 25c677fdfb2bd..afbbcbc5e9704 100644 --- a/peer/peerwg/derp.go +++ b/peer/peerwg/derp.go @@ -11,15 +11,15 @@ import ( // our own support for DERP servers. var DerpMap = &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ - 9: { - RegionID: 9, - RegionCode: "dfw", - RegionName: "Dallas", + 1: { + RegionID: 1, + RegionCode: "goog", + RegionName: "Google", Avoid: false, Nodes: []*tailcfg.DERPNode{ { Name: "9a", - RegionID: 9, + RegionID: 1, HostName: "derp9.tailscale.com", CertName: "", IPv4: "207.148.3.137", @@ -30,38 +30,45 @@ var DerpMap = &tailcfg.DERPMap{ InsecureForTests: false, STUNTestIP: "", }, + // { + // Name: "9c", + // RegionID: 9, + // HostName: "derp9c.tailscale.com", + // CertName: "", + // IPv4: "155.138.243.219", + // IPv6: "2001:19f0:6401:fe7:5400:3ff:fe8d:6d9c", + // STUNPort: 0, + // STUNOnly: false, + // DERPPort: 0, + // InsecureForTests: false, + // STUNTestIP: "", + // }, + // { + // Name: "9b", + // RegionID: 9, + // HostName: "derp9b.tailscale.com", + // CertName: "", + // IPv4: "144.202.67.195", + // IPv6: "2001:19f0:6401:eb5:5400:3ff:fe8d:6d9b", + // STUNPort: 0, + // STUNOnly: false, + // DERPPort: 0, + // InsecureForTests: false, + // STUNTestIP: "", + // }, { - Name: "9c", - RegionID: 9, - HostName: "derp9c.tailscale.com", - CertName: "", - IPv4: "155.138.243.219", - IPv6: "2001:19f0:6401:fe7:5400:3ff:fe8d:6d9c", - STUNPort: 0, - STUNOnly: false, - DERPPort: 0, - InsecureForTests: false, - STUNTestIP: "", - }, - { - Name: "9b", - RegionID: 9, - HostName: "derp9b.tailscale.com", - CertName: "", - IPv4: "144.202.67.195", - IPv6: "2001:19f0:6401:eb5:5400:3ff:fe8d:6d9b", - STUNPort: 0, - STUNOnly: false, - DERPPort: 0, - InsecureForTests: false, - STUNTestIP: "", + Name: "goog", + RegionID: 2, + HostName: "stun.l.google.com", + STUNPort: 19302, + STUNOnly: true, }, }, }, }, - OmitDefaultRegions: true, + // OmitDefaultRegions: true, } // DefaultDerpHome is the ipv4 representation of a DERP server. The port is the // DERP id. We only support using DERP 9 for now. -var DefaultDerpHome = net.JoinHostPort(magicsock.DerpMagicIP, "9") +var DefaultDerpHome = net.JoinHostPort(magicsock.DerpMagicIP, "1") diff --git a/peer/peerwg/wireguard.go b/peer/peerwg/wireguard.go index b210b2b70dadc..b29db41bd1cd3 100644 --- a/peer/peerwg/wireguard.go +++ b/peer/peerwg/wireguard.go @@ -56,7 +56,7 @@ func UUIDToInet(uid uuid.UUID) pqtype.Inet { } func UUIDToNetaddr(uid uuid.UUID) netaddr.IP { - return netaddr.IPFrom16(privateUUID(uid)) + return netaddr.IPFrom16(uuid.New()) } // privateUUID sets the uid to have the tailscale private ipv6 prefix. @@ -94,7 +94,7 @@ type Network struct { func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { nodePrivateKey := key.NewNode() nodePublicKey := nodePrivateKey.Public() - id, stableID := nodeIDs(nodePublicKey) + // id, stableID := nodeIDs(nodePublicKey) netMap := &netmap.NetworkMap{ NodeKey: nodePublicKey, @@ -128,15 +128,9 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { Caps: []filter.CapMatch{}, }}, } - // Identify itself as a node on the network with the addresses provided. netMap.SelfNode = &tailcfg.Node{ - ID: id, - StableID: stableID, - Key: nodePublicKey, - Addresses: netMap.Addresses, - AllowedIPs: append(netMap.Addresses, netaddr.MustParseIPPrefix("::/0")), - Endpoints: []string{}, - DERP: DefaultDerpHome, + Key: nodePublicKey, + Addresses: addresses, } wgMonitor, err := monitor.New(Logf) @@ -144,8 +138,9 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { return nil, xerrors.Errorf("create link monitor: %w", err) } - dialer := new(tsdial.Dialer) - dialer.Logf = Logf + dialer := &tsdial.Dialer{ + Logf: Logf, + } // Create a wireguard engine in userspace. engine, err := wgengine.NewUserspaceEngine(Logf, wgengine.Config{ LinkMonitor: wgMonitor, @@ -154,6 +149,10 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { if err != nil { return nil, xerrors.Errorf("create wgengine: %w", err) } + dialer.UseNetstackForIP = func(ip netaddr.IP) bool { + _, ok := engine.PeerForIP(ip) + return ok + } // This is taken from Tailscale: // https://github.com/tailscale/tailscale/blob/0f05b2c13ff0c305aa7a1655fa9c17ed969d65be/tsnet/tsnet.go#L247-L255 @@ -176,15 +175,10 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { if err != nil { return nil, xerrors.Errorf("create netstack: %w", err) } - netStack.ProcessLocalIPs = true - netStack.ProcessSubnets = true - dialer.UseNetstackForIP = func(ip netaddr.IP) bool { - _, ok := engine.PeerForIP(ip) - return ok - } dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) { return netStack.DialContextTCP(ctx, dst) } + netStack.ProcessLocalIPs = true err = netStack.Start() if err != nil { return nil, xerrors.Errorf("start netstack: %w", err) @@ -192,7 +186,7 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { engine = wgengine.NewWatchdog(engine) // Update the wireguard configuration to allow traffic to flow. - cfg, err := nmcfg.WGCfg(netMap, Logf, netmap.AllowSingleHosts|netmap.AllowSubnetRoutes, netMap.SelfNode.StableID) + cfg, err := nmcfg.WGCfg(netMap, Logf, netmap.AllowSingleHosts, "") if err != nil { return nil, xerrors.Errorf("create wgcfg: %w", err) } @@ -206,7 +200,9 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { } engine.SetDERPMap(DerpMap) - engine.SetNetworkMap(copyNetMap(netMap)) + netMapCopy := *netMap + netMapCopy.SelfNode = &tailcfg.Node{} + engine.SetNetworkMap(&netMapCopy) ipb := netaddr.IPSetBuilder{} for _, addr := range netMap.Addresses { @@ -304,17 +300,17 @@ func (n *Network) AddPeer(handshake Handshake) error { // modifications. peers := append(([]*tailcfg.Node)(nil), n.netMap.Peers...) - id, stableID := nodeIDs(handshake.NodePublicKey) + // id, stableID := nodeIDs(handshake.NodePublicKey) peers = append(peers, &tailcfg.Node{ - ID: id, - StableID: stableID, - Name: handshake.NodePublicKey.String() + ".com", + // ID: id, + // StableID: stableID, + // Name: handshake.NodePublicKey.String() + ".com", Key: handshake.NodePublicKey, DiscoKey: handshake.DiscoPublicKey, Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(handshake.IPv6, 128)}, AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(handshake.IPv6, 128)}, DERP: DefaultDerpHome, - Endpoints: []string{DefaultDerpHome}, + // Endpoints: []string{DefaultDerpHome}, }) n.netMap.Peers = peers diff --git a/peer/peerwg/wireguard_test.go b/peer/peerwg/wireguard_test.go new file mode 100644 index 0000000000000..cc801bbde33cb --- /dev/null +++ b/peer/peerwg/wireguard_test.go @@ -0,0 +1,116 @@ +package peerwg_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "inet.af/netaddr" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/net/stun/stuntest" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + "tailscale.com/types/logger" + "tailscale.com/types/nettype" + + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/peer/peerwg" +) + +func TestConnect(t *testing.T) { + t.Parallel() + + logger := slogtest.Make(t, nil) + c1IPv6 := peerwg.UUIDToNetaddr(uuid.New()) + wgn1, err := peerwg.New(logger.Named("c1"), []netaddr.IPPrefix{ + netaddr.IPPrefixFrom(c1IPv6, 128), + }) + require.NoError(t, err) + + c2IPv6 := peerwg.UUIDToNetaddr(uuid.New()) + wgn2, err := peerwg.New(logger.Named("c2"), []netaddr.IPPrefix{ + netaddr.IPPrefixFrom(c2IPv6, 128), + }) + require.NoError(t, err) + err = wgn1.AddPeer(peerwg.Handshake{ + DiscoPublicKey: wgn2.DiscoPublicKey, + NodePublicKey: wgn2.NodePrivateKey.Public(), + IPv6: c2IPv6, + }) + require.NoError(t, err) + + conn := make(chan struct{}) + go func() { + listener, err := wgn1.Listen("tcp", ":35565") + require.NoError(t, err) + conn <- struct{}{} + fmt.Printf("Started listening...\n") + _, err = listener.Accept() + fmt.Printf("Got connection!\n") + require.NoError(t, err) + conn <- struct{}{} + }() + + err = wgn2.AddPeer(peerwg.Handshake{ + DiscoPublicKey: wgn1.DiscoPublicKey, + NodePublicKey: wgn1.NodePrivateKey.Public(), + IPv6: c1IPv6, + }) + require.NoError(t, err) + <-conn + time.Sleep(100 * time.Millisecond) + fmt.Printf("\n\n\n\n\nDIALING TCP\n\n\n\n\n") + _, err = wgn2.Netstack.DialContextTCP(context.Background(), netaddr.IPPortFrom(c1IPv6, 35565)) + require.NoError(t, err) + <-conn +} + +func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, stunIP netaddr.IP) (derpMap *tailcfg.DERPMap, cleanup func()) { + d := derp.NewServer(key.NewNode(), logf) + + httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d)) + httpsrv.Config.ErrorLog = logger.StdLogger(logf) + httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + httpsrv.StartTLS() + + stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, l) + + m := &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "test", + Nodes: []*tailcfg.DERPNode{ + { + Name: "t1", + RegionID: 1, + HostName: "test-node.unused", + IPv4: "127.0.0.1", + IPv6: "none", + STUNPort: stunAddr.Port, + DERPPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port, + InsecureForTests: true, + STUNTestIP: stunIP.String(), + }, + }, + }, + }, + } + + cleanup = func() { + httpsrv.CloseClientConnections() + httpsrv.Close() + d.Close() + stunCleanup() + } + + return m, cleanup +} diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 5721d1d50a93e..1edaf731999e1 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -399,15 +399,17 @@ export interface WorkspaceAgent { readonly startup_script?: string readonly directory?: string readonly apps: WorkspaceApp[] + // Named type "inet.af/netaddr.IP" unknown, using "any" + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readonly ip_addresses: any[] // Named type "tailscale.com/types/key.NodePublic" unknown, using "any" // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly wireguard_public_key: any + readonly node_public_key: any // Named type "tailscale.com/types/key.DiscoPublic" unknown, using "any" // eslint-disable-next-line @typescript-eslint/no-explicit-any readonly disco_public_key: any - // Named type "inet.af/netaddr.IPPrefix" unknown, using "any" - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly ipv6: any + readonly derp: string + readonly latency: Record } // From codersdk/workspaceagents.go:48:6 @@ -415,7 +417,7 @@ export interface WorkspaceAgentAuthenticateResponse { readonly session_token: string } -// From codersdk/workspaceresources.go:63:6 +// From codersdk/workspaceresources.go:70:6 export interface WorkspaceAgentInstanceMetadata { readonly jail_orchestrator: string readonly operating_system: string @@ -428,7 +430,7 @@ export interface WorkspaceAgentInstanceMetadata { readonly vnc: boolean } -// From codersdk/workspaceresources.go:55:6 +// From codersdk/workspaceresources.go:62:6 export interface WorkspaceAgentResourceMetadata { readonly memory_total: number readonly disk_total: number diff --git a/tailnet/tailnet.go b/tailnet/tailnet.go new file mode 100644 index 0000000000000..ce60755ea5ca6 --- /dev/null +++ b/tailnet/tailnet.go @@ -0,0 +1,439 @@ +package tailnet + +import ( + "context" + "fmt" + "net" + "sync" + "time" + + "github.com/google/uuid" + "golang.org/x/xerrors" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "inet.af/netaddr" + "tailscale.com/net/dns" + "tailscale.com/net/netns" + "tailscale.com/net/tsdial" + "tailscale.com/net/tstun" + "tailscale.com/tailcfg" + "tailscale.com/types/ipproto" + "tailscale.com/types/key" + tslogger "tailscale.com/types/logger" + "tailscale.com/types/netmap" + "tailscale.com/wgengine" + "tailscale.com/wgengine/filter" + "tailscale.com/wgengine/magicsock" + "tailscale.com/wgengine/monitor" + "tailscale.com/wgengine/netstack" + "tailscale.com/wgengine/router" + "tailscale.com/wgengine/wgcfg/nmcfg" + + "github.com/coder/coder/cryptorand" + + "cdr.dev/slog" +) + +func init() { + // Globally disable network namespacing. + // All networking happens in userspace. + netns.SetEnabled(false) +} + +type Options struct { + Addresses []netaddr.IPPrefix + DERPMap *tailcfg.DERPMap + + Logger slog.Logger +} + +// New constructs a new Wireguard server that will accept connections from the addresses provided. +func New(options *Options) (*Server, error) { + if options == nil { + options = &Options{} + } + if len(options.Addresses) == 0 { + return nil, xerrors.New("At least one IP range must be provided") + } + if options.DERPMap == nil { + return nil, xerrors.New("DERPMap must be provided") + } + nodePrivateKey := key.NewNode() + nodePublicKey := nodePrivateKey.Public() + + netMap := &netmap.NetworkMap{ + NodeKey: nodePublicKey, + PrivateKey: nodePrivateKey, + Addresses: options.Addresses, + PacketFilter: []filter.Match{{ + // Allow any protocol! + IPProto: []ipproto.Proto{ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, ipproto.SCTP}, + // Allow traffic sourced from anywhere. + Srcs: []netaddr.IPPrefix{ + netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), + netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + }, + // Allow traffic to route anywhere. + Dsts: []filter.NetPortRange{ + { + Net: netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), + Ports: filter.PortRange{ + First: 0, + Last: 65535, + }, + }, + { + Net: netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + Ports: filter.PortRange{ + First: 0, + Last: 65535, + }, + }, + }, + Caps: []filter.CapMatch{}, + }}, + } + nodeID, err := cryptorand.Int63() + if err != nil { + return nil, xerrors.Errorf("generate node id: %w", err) + } + // This is used by functions below to identify the node via key + netMap.SelfNode = &tailcfg.Node{ + ID: tailcfg.NodeID(nodeID), + Key: nodePublicKey, + Addresses: options.Addresses, + AllowedIPs: options.Addresses, + } + + wireguardMonitor, err := monitor.New(Logger(options.Logger.Named("wgmonitor"))) + if err != nil { + return nil, xerrors.Errorf("create wireguard link monitor: %w", err) + } + + dialer := &tsdial.Dialer{ + Logf: Logger(options.Logger), + } + wireguardEngine, err := wgengine.NewUserspaceEngine(Logger(options.Logger.Named("wgengine")), wgengine.Config{ + LinkMonitor: wireguardMonitor, + Dialer: dialer, + }) + if err != nil { + return nil, xerrors.Errorf("create wgengine: %w", err) + } + dialer.UseNetstackForIP = func(ip netaddr.IP) bool { + _, ok := wireguardEngine.PeerForIP(ip) + return ok + } + + // This is taken from Tailscale: + // https://github.com/tailscale/tailscale/blob/0f05b2c13ff0c305aa7a1655fa9c17ed969d65be/tsnet/tsnet.go#L247-L255 + wireguardInternals, ok := wireguardEngine.(wgengine.InternalsGetter) + if !ok { + return nil, xerrors.Errorf("wireguard engine isn't the correct type %T", wireguardEngine) + } + tunDevice, magicConn, dnsManager, ok := wireguardInternals.GetInternals() + if !ok { + return nil, xerrors.New("failed to get wireguard internals") + } + + // Update the keys for the magic connection! + err = magicConn.SetPrivateKey(nodePrivateKey) + if err != nil { + return nil, xerrors.Errorf("set node private key: %w", err) + } + netMap.SelfNode.DiscoKey = magicConn.DiscoPublicKey() + + netStack, err := netstack.Create( + Logger(options.Logger.Named("netstack")), tunDevice, wireguardEngine, magicConn, dialer, dnsManager) + if err != nil { + return nil, xerrors.Errorf("create netstack: %w", err) + } + dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) { + return netStack.DialContextTCP(ctx, dst) + } + netStack.ProcessLocalIPs = true + err = netStack.Start() + if err != nil { + return nil, xerrors.Errorf("start netstack: %w", err) + } + wireguardEngine = wgengine.NewWatchdog(wireguardEngine) + + // Update the wireguard configuration to allow traffic to flow. + wireguardConfig, err := nmcfg.WGCfg(netMap, Logger(options.Logger.Named("wgconfig")), netmap.AllowSingleHosts, "") + if err != nil { + return nil, xerrors.Errorf("create wgcfg: %w", err) + } + + wireguardRouter := &router.Config{ + LocalAddrs: wireguardConfig.Addresses, + } + err = wireguardEngine.Reconfig(wireguardConfig, wireguardRouter, &dns.Config{}, &tailcfg.Debug{}) + if err != nil { + return nil, xerrors.Errorf("reconfig: %w", err) + } + + wireguardEngine.SetDERPMap(options.DERPMap) + netMapCopy := *netMap + wireguardEngine.SetNetworkMap(&netMapCopy) + + localIPSet := netaddr.IPSetBuilder{} + for _, addr := range netMap.Addresses { + localIPSet.AddPrefix(addr) + } + localIPs, _ := localIPSet.IPSet() + logIPSet := netaddr.IPSetBuilder{} + logIPs, _ := logIPSet.IPSet() + wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter")))) + server := &Server{ + logger: options.Logger, + magicConn: magicConn, + dialer: dialer, + listeners: map[listenKey]*listener{}, + tunDevice: tunDevice, + netMap: netMap, + netStack: netStack, + wireguardMonitor: wireguardMonitor, + wireguardRouter: wireguardRouter, + wireguardEngine: wireguardEngine, + } + netStack.ForwardTCPIn = server.forwardTCP + return server, nil +} + +// IP generates a new IP with a static service prefix. +func IP() netaddr.IP { + // This is Tailscale's ephemeral service prefix. + // This can be changed easily later-on, because + // all of our nodes are ephemeral. + // fd7a:115c:a1e0 + uid := uuid.New() + uid[0] = 0xfd + uid[1] = 0x7a + uid[2] = 0x11 + uid[3] = 0x5c + uid[4] = 0xa1 + uid[5] = 0xe0 + return netaddr.IPFrom16(uid) +} + +// Server is an actively listening Wireguard connection. +type Server struct { + mutex sync.Mutex + logger slog.Logger + + dialer *tsdial.Dialer + tunDevice *tstun.Wrapper + netMap *netmap.NetworkMap + netStack *netstack.Impl + magicConn *magicsock.Conn + wireguardMonitor *monitor.Mon + wireguardRouter *router.Config + wireguardEngine wgengine.Engine + listeners map[listenKey]*listener +} + +// SetNodeCallback is triggered when a network change occurs and peer +// renegotiation may be required. Clients should constantly be emitting +// node changes. +func (s *Server) SetNodeCallback(callback func(node *Node)) { + s.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { + fmt.Printf("\n\n\n\nNetwork status: %+v %s\n\n\n\n", s, err) + }) + + s.wireguardEngine.AddNetworkMapCallback(func(nm *netmap.NetworkMap) { + fmt.Printf("\n\n\n\nNetwork map: %+v\n\n\n\n", nm) + }) + s.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { + fmt.Printf("\n\n\n\n\nUpdating network information: %+v\n\n\n\n\n", ni) + callback(&Node{ + ID: s.netMap.SelfNode.ID, + Key: s.netMap.SelfNode.Key, + Addresses: s.netMap.SelfNode.Addresses, + AllowedIPs: s.netMap.SelfNode.AllowedIPs, + DiscoKey: s.magicConn.DiscoPublicKey(), + PreferredDERP: ni.PreferredDERP, + DERPLatency: ni.DERPLatency, + }) + }) +} + +// Once the client disconnects, why not just reset the connection? +// When the agent reconnects it will update itself, in which case +// we should do a `SetNetInfoCallback` to get the personal node info +// again to send on the wire. +// +// On disconnect, we should continually poll the workspace agent to +// update the node to reestablish a connection. +// +// How do we know if we're disconnected? + +// UpdateNodes connects with a set of peers. This can be constantly updated, +// and peers will continually be reconnected as necessary. +func (s *Server) UpdateNodes(nodes []*Node) error { + s.mutex.Lock() + defer s.mutex.Unlock() + peerMap := map[tailcfg.NodeID]*tailcfg.Node{} + for _, peer := range s.netMap.Peers { + peerMap[peer.ID] = peer + } + for _, node := range nodes { + peerMap[node.ID] = &tailcfg.Node{ + ID: node.ID, + Key: node.Key, + DiscoKey: node.DiscoKey, + Addresses: node.Addresses, + AllowedIPs: node.AllowedIPs, + DERP: fmt.Sprintf("%s:%d", magicsock.DerpMagicIP, node.PreferredDERP), + } + } + s.netMap.Peers = make([]*tailcfg.Node, 0, len(peerMap)) + for _, peer := range peerMap { + s.netMap.Peers = append(s.netMap.Peers, peer) + } + cfg, err := nmcfg.WGCfg(s.netMap, Logger(s.logger.Named("wgconfig")), netmap.AllowSingleHosts, "") + if err != nil { + return xerrors.Errorf("update wireguard config: %w", err) + } + err = s.wireguardEngine.Reconfig(cfg, s.wireguardRouter, &dns.Config{}, &tailcfg.Debug{}) + if err != nil { + return xerrors.Errorf("reconfig: %w", err) + } + netMapCopy := *s.netMap + s.wireguardEngine.SetNetworkMap(&netMapCopy) + return nil +} + +// Close shuts down the Wireguard connection. +func (s *Server) Close() error { + s.mutex.Lock() + defer s.mutex.Unlock() + for _, l := range s.listeners { + _ = l.Close() + } + _ = s.dialer.Close() + _ = s.magicConn.Close() + _ = s.netStack.Close() + _ = s.wireguardMonitor.Close() + _ = s.tunDevice.Close() + s.wireguardEngine.Close() + return nil +} + +// Node represents a node in the network. +type Node struct { + ID tailcfg.NodeID `json:"id"` + Key key.NodePublic `json:"key"` + DiscoKey key.DiscoPublic `json:"disco"` + PreferredDERP int `json:"preferred_derp"` + DERPLatency map[string]float64 `json:"derp_latency"` + Addresses []netaddr.IPPrefix `json:"addresses"` + AllowedIPs []netaddr.IPPrefix `json:"allowed_ips"` +} + +// This and below is taken _mostly_ verbatim from Tailscale: +// https://github.com/tailscale/tailscale/blob/c88bd53b1b7b2fcf7ba302f2e53dd1ce8c32dad4/tsnet/tsnet.go#L459-L494 + +// Listen announces only on the Tailscale network. +// It will start the server if it has not been started yet. +func (s *Server) Listen(network, addr string) (net.Listener, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, xerrors.Errorf("wgnet: %w", err) + } + lk := listenKey{network, host, port} + ln := &listener{ + s: s, + key: lk, + addr: addr, + + conn: make(chan net.Conn), + } + s.mutex.Lock() + if s.listeners == nil { + s.listeners = map[listenKey]*listener{} + } + if _, ok := s.listeners[lk]; ok { + s.mutex.Unlock() + return nil, xerrors.Errorf("wgnet: listener already open for %s, %s", network, addr) + } + s.listeners[lk] = ln + s.mutex.Unlock() + return ln, nil +} + +func (s *Server) DialContextTCP(ctx context.Context, ipp netaddr.IPPort) (*gonet.TCPConn, error) { + return s.netStack.DialContextTCP(ctx, ipp) +} + +func (s *Server) DialContextUDP(ctx context.Context, ipp netaddr.IPPort) (*gonet.UDPConn, error) { + return s.netStack.DialContextUDP(ctx, ipp) +} + +func (s *Server) forwardTCP(c net.Conn, port uint16) { + s.mutex.Lock() + ln, ok := s.listeners[listenKey{"tcp", "", fmt.Sprint(port)}] + s.mutex.Unlock() + if !ok { + _ = c.Close() + return + } + t := time.NewTimer(time.Second) + defer t.Stop() + select { + case ln.conn <- c: + case <-t.C: + _ = c.Close() + } +} + +type listenKey struct { + network string + host string + port string +} + +type listener struct { + s *Server + key listenKey + addr string + conn chan net.Conn +} + +func (ln *listener) Accept() (net.Conn, error) { + c, ok := <-ln.conn + if !ok { + return nil, xerrors.Errorf("wgnet: %w", net.ErrClosed) + } + return c, nil +} + +func (ln *listener) Addr() net.Addr { return addr{ln} } +func (ln *listener) Close() error { + ln.s.mutex.Lock() + defer ln.s.mutex.Unlock() + if v, ok := ln.s.listeners[ln.key]; ok && v == ln { + delete(ln.s.listeners, ln.key) + close(ln.conn) + } + return nil +} + +type addr struct{ ln *listener } + +func (a addr) Network() string { return a.ln.key.network } +func (a addr) String() string { return a.ln.addr } + +// Logger converts the Tailscale logging function to use slog. +func Logger(logger slog.Logger) tslogger.Logf { + return tslogger.Logf(func(format string, args ...any) { + logger.Debug(context.Background(), fmt.Sprintf(format, args...)) + }) +} + +// The exchanger is entirely in-memory and works based on connected nodes. +// It uses a PubSub system to dynamically add/remove nodes from the network +// and build a netmap based on connection ID. +// +// Each node is allocated it's own internal connection ID. +// +// The connecting node *just* requires information about the other node. +// The other node needs connection information of all the others. diff --git a/tailnet/tailnet_test.go b/tailnet/tailnet_test.go new file mode 100644 index 0000000000000..f8800467fedb3 --- /dev/null +++ b/tailnet/tailnet_test.go @@ -0,0 +1,147 @@ +package tailnet_test + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "inet.af/netaddr" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + tslogger "tailscale.com/types/logger" + + "github.com/coder/coder/tailnet" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/slogtest" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestTailnet(t *testing.T) { + t.Parallel() + logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) + derpMap := runDERPAndStun(t, tailnet.Logger(logger.Named("derp"))) + + w1IP := tailnet.IP() + w1, err := tailnet.New(&tailnet.Options{ + Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(w1IP, 128)}, + Logger: logger.Named("w1"), + DERPMap: derpMap, + }) + require.NoError(t, err) + + // When a new connection occurs, we want those nodes to exist for the lifetime of the connection. + // As soon as the connection ends, the nodes can be removed. + + // The workspace agent creates a Tailnet on start. It updates keys and + // begins listening for connection messages. + // + // A new connection starts by concurrently sending a POST request with + // it's keys, and using a GET request on the workspace agent. + // + // Internally, the agent WebSocket listens for these messages. + // If the agent dies and comes back to life, + + w2, err := tailnet.New(&tailnet.Options{ + Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(tailnet.IP(), 128)}, + Logger: logger.Named("w2"), + DERPMap: derpMap, + }) + require.NoError(t, err) + t.Cleanup(func() { + _ = w1.Close() + _ = w2.Close() + }) + w1.SetNodeCallback(func(node *tailnet.Node) { + w2.UpdateNodes([]*tailnet.Node{node}) + }) + w2.SetNodeCallback(func(node *tailnet.Node) { + w1.UpdateNodes([]*tailnet.Node{node}) + }) + + conn := make(chan struct{}) + go func() { + listener, err := w1.Listen("tcp", ":35565") + assert.NoError(t, err) + defer listener.Close() + nc, err := listener.Accept() + assert.NoError(t, err) + _ = nc.Close() + conn <- struct{}{} + }() + + nc, err := w2.DialContextTCP(context.Background(), netaddr.IPPortFrom(w1IP, 35565)) + require.NoError(t, err) + _ = nc.Close() + <-conn + + time.Sleep(time.Minute) + + w1.Close() + w2.Close() +} + +func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) { + d := derp.NewServer(key.NewNode(), logf) + server := httptest.NewUnstartedServer(derphttp.Handler(d)) + server.Config.ErrorLog = tslogger.StdLogger(logf) + server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + server.StartTLS() + + go func() { + time.Sleep(5 * time.Second) + fmt.Printf("\n\n\n\n\nSHUTTING IT DOWN\n\n\n\n\n") + server.CloseClientConnections() + server.Close() + d.Close() + }() + + // stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + t.Cleanup(func() { + server.CloseClientConnections() + server.Close() + d.Close() + // stunCleanup() + }) + + tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) + if !ok { + t.FailNow() + } + + return &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "test", + RegionName: "Testlandia", + Nodes: []*tailcfg.DERPNode{ + { + Name: "t1", + RegionID: 1, + HostName: "test-node.dns", + IPv4: "127.0.0.1", + IPv6: "none", + STUNPort: -1, + DERPPort: tcpAddr.Port, + InsecureForTests: true, + STUNTestIP: "127.0.0.1", + }, + }, + }, + }, + } +} From 844071dcac78c7c0db65042edaf3bc1b9fc4039b Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sat, 9 Jul 2022 02:07:07 +0000 Subject: [PATCH 03/54] Add ping --- tailnet/tailnet.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/tailnet/tailnet.go b/tailnet/tailnet.go index ce60755ea5ca6..6eb99f24fe739 100644 --- a/tailnet/tailnet.go +++ b/tailnet/tailnet.go @@ -11,6 +11,8 @@ import ( "golang.org/x/xerrors" "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" "inet.af/netaddr" + "tailscale.com/hostinfo" + "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" "tailscale.com/net/netns" "tailscale.com/net/tsdial" @@ -239,11 +241,11 @@ func (s *Server) SetNodeCallback(callback func(node *Node)) { fmt.Printf("\n\n\n\nNetwork status: %+v %s\n\n\n\n", s, err) }) - s.wireguardEngine.AddNetworkMapCallback(func(nm *netmap.NetworkMap) { - fmt.Printf("\n\n\n\nNetwork map: %+v\n\n\n\n", nm) - }) + // s.wireguardEngine.AddNetworkMapCallback(func(nm *netmap.NetworkMap) { + // fmt.Printf("\n\n\n\nNetwork map: %+v\n\n\n\n", nm) + // }) s.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { - fmt.Printf("\n\n\n\n\nUpdating network information: %+v\n\n\n\n\n", ni) + // fmt.Printf("\n\n\n\n\nUpdating network information: %+v\n\n\n\n\n", ni) callback(&Node{ ID: s.netMap.SelfNode.ID, Key: s.netMap.SelfNode.Key, @@ -256,16 +258,6 @@ func (s *Server) SetNodeCallback(callback func(node *Node)) { }) } -// Once the client disconnects, why not just reset the connection? -// When the agent reconnects it will update itself, in which case -// we should do a `SetNetInfoCallback` to get the personal node info -// again to send on the wire. -// -// On disconnect, we should continually poll the workspace agent to -// update the node to reestablish a connection. -// -// How do we know if we're disconnected? - // UpdateNodes connects with a set of peers. This can be constantly updated, // and peers will continually be reconnected as necessary. func (s *Server) UpdateNodes(nodes []*Node) error { @@ -283,6 +275,7 @@ func (s *Server) UpdateNodes(nodes []*Node) error { Addresses: node.Addresses, AllowedIPs: node.AllowedIPs, DERP: fmt.Sprintf("%s:%d", magicsock.DerpMagicIP, node.PreferredDERP), + Hostinfo: hostinfo.New().View(), } } s.netMap.Peers = make([]*tailcfg.Node, 0, len(peerMap)) @@ -302,6 +295,10 @@ func (s *Server) UpdateNodes(nodes []*Node) error { return nil } +func (s *Server) Ping(ip netaddr.IP, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { + s.wireguardEngine.Ping(ip, pingType, cb) +} + // Close shuts down the Wireguard connection. func (s *Server) Close() error { s.mutex.Lock() From 36655f0ce8d2662d39c1fa3b75b3ce722a443bd7 Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sat, 9 Jul 2022 03:54:29 +0000 Subject: [PATCH 04/54] Add listening --- .vscode/settings.json | 1 + cli/ssh.go | 111 ++++----- cli/wireguardtunnel.go | 157 ++++++------ coderd/coderd.go | 16 +- coderd/database/databasefake/databasefake.go | 2 +- coderd/database/dump.sql | 2 +- .../database/migrations/000029_tailnet.up.sql | 2 +- coderd/database/models.go | 2 +- coderd/database/queries.sql.go | 30 +-- coderd/database/queries/workspaceagents.sql | 2 +- coderd/database/sqlc.yaml | 2 +- coderd/workspaceagents.go | 235 ++++++++++-------- codersdk/workspaceagents.go | 21 ++ codersdk/workspaceresources.go | 4 +- tailnet/tailnet.go | 9 +- tailnet/tailnet_test.go | 32 +-- 16 files changed, 316 insertions(+), 312 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index ccb4a61733cb7..52b06ca59886c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,7 @@ "cronstrue", "DERP", "derphttp", + "derpmap", "devel", "drpc", "drpcconn", diff --git a/cli/ssh.go b/cli/ssh.go index 81eacc33f0899..96dc8a7450a6c 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -18,18 +18,13 @@ import ( gosshagent "golang.org/x/crypto/ssh/agent" "golang.org/x/term" "golang.org/x/xerrors" - "inet.af/netaddr" - tslogger "tailscale.com/types/logger" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" "github.com/coder/coder/coderd/autobuild/notify" "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" - "github.com/coder/coder/peer/peerwg" ) var workspacePollInterval = time.Minute @@ -122,59 +117,59 @@ func ssh() *cobra.Command { } } else { // TODO: more granual control of Tailscale logging. - peerwg.Logf = tslogger.Discard - - ipv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn, err := peerwg.New( - slog.Make(sloghuman.Sink(os.Stderr)), - []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, - ) - if err != nil { - return xerrors.Errorf("create wireguard network: %w", err) - } - - err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ - Recipient: workspaceAgent.ID, - NodePublicKey: wgn.NodePrivateKey.Public(), - DiscoPublicKey: wgn.DiscoPublicKey, - IPv6: ipv6, - }) - if err != nil { - return xerrors.Errorf("post wireguard peer: %w", err) - } - - err = wgn.AddPeer(peerwg.Handshake{ - Recipient: workspaceAgent.ID, - DiscoPublicKey: workspaceAgent.DiscoPublicKey, - NodePublicKey: workspaceAgent.WireguardPublicKey, - IPv6: workspaceAgent.IPv6.IP(), - }) - if err != nil { - return xerrors.Errorf("add workspace agent as peer: %w", err) - } - - if stdio { - rawSSH, err := wgn.SSH(cmd.Context(), workspaceAgent.IPv6.IP()) - if err != nil { - return err - } - - go func() { - _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) - }() - _, _ = io.Copy(rawSSH, cmd.InOrStdin()) - return nil - } - - sshClient, err = wgn.SSHClient(cmd.Context(), workspaceAgent.IPv6.IP()) - if err != nil { - return err - } - - sshSession, err = sshClient.NewSession() - if err != nil { - return err - } + // peerwg.Logf = tslogger.Discard + + // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) + // wgn, err := peerwg.New( + // slog.Make(sloghuman.Sink(os.Stderr)), + // []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, + // ) + // if err != nil { + // return xerrors.Errorf("create wireguard network: %w", err) + // } + + // err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ + // Recipient: workspaceAgent.ID, + // NodePublicKey: wgn.NodePrivateKey.Public(), + // DiscoPublicKey: wgn.DiscoPublicKey, + // IPv6: ipv6, + // }) + // if err != nil { + // return xerrors.Errorf("post wireguard peer: %w", err) + // } + + // err = wgn.AddPeer(peerwg.Handshake{ + // Recipient: workspaceAgent.ID, + // DiscoPublicKey: workspaceAgent.DiscoPublicKey, + // NodePublicKey: workspaceAgent.WireguardPublicKey, + // IPv6: workspaceAgent.IPv6.IP(), + // }) + // if err != nil { + // return xerrors.Errorf("add workspace agent as peer: %w", err) + // } + + // if stdio { + // rawSSH, err := wgn.SSH(cmd.Context(), workspaceAgent.IPv6.IP()) + // if err != nil { + // return err + // } + + // go func() { + // _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) + // }() + // _, _ = io.Copy(rawSSH, cmd.InOrStdin()) + // return nil + // } + + // sshClient, err = wgn.SSHClient(cmd.Context(), workspaceAgent.IPv6.IP()) + // if err != nil { + // return err + // } + + // sshSession, err = sshClient.NewSession() + // if err != nil { + // return err + // } } if identityAgent == "" { diff --git a/cli/wireguardtunnel.go b/cli/wireguardtunnel.go index 6a44f17d9ec58..2276317e9b4a8 100644 --- a/cli/wireguardtunnel.go +++ b/cli/wireguardtunnel.go @@ -4,20 +4,14 @@ import ( "context" "fmt" "net" - "os" - "os/signal" "strconv" "sync" - "syscall" - "github.com/google/uuid" "github.com/pion/udp" "github.com/spf13/cobra" "golang.org/x/xerrors" "inet.af/netaddr" - "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" coderagent "github.com/coder/coder/agent" "github.com/coder/coder/cli/cliui" "github.com/coder/coder/codersdk" @@ -94,81 +88,82 @@ func wireguardPortForward() *cobra.Command { return xerrors.Errorf("await agent: %w", err) } - ipv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn, err := peerwg.New( - slog.Make(sloghuman.Sink(os.Stderr)), - []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, - ) - if err != nil { - return xerrors.Errorf("create wireguard network: %w", err) - } - - err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ - Recipient: workspaceAgent.ID, - NodePublicKey: wgn.NodePrivateKey.Public(), - DiscoPublicKey: wgn.DiscoPublicKey, - IPv6: ipv6, - }) - if err != nil { - return xerrors.Errorf("post wireguard peer: %w", err) - } - - err = wgn.AddPeer(peerwg.Handshake{ - Recipient: workspaceAgent.ID, - DiscoPublicKey: workspaceAgent.DiscoPublicKey, - NodePublicKey: workspaceAgent.WireguardPublicKey, - IPv6: workspaceAgent.IPv6.IP(), - }) - if err != nil { - return xerrors.Errorf("add workspace agent as peer: %w", err) - } - - // Start all listeners. - var ( - ctx, cancel = context.WithCancel(cmd.Context()) - wg = new(sync.WaitGroup) - listeners = make([]net.Listener, len(specs)) - closeAllListeners = func() { - for _, l := range listeners { - if l == nil { - continue - } - _ = l.Close() - } - } - ) - defer cancel() - for i, spec := range specs { - l, err := listenAndPortForwardWireguard(ctx, cmd, wgn, wg, spec, workspaceAgent.IPv6.IP()) - if err != nil { - closeAllListeners() - return err - } - listeners[i] = l - } - - // Wait for the context to be canceled or for a signal and close - // all listeners. - var closeErr error - go func() { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - select { - case <-ctx.Done(): - closeErr = ctx.Err() - case <-sigs: - _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Received signal, closing all listeners and active connections") - closeErr = xerrors.New("signal received") - } - - cancel() - closeAllListeners() - }() - - _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Ready!") - wg.Wait() - return closeErr + // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) + // wgn, err := peerwg.New( + // slog.Make(sloghuman.Sink(os.Stderr)), + // []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, + // ) + // if err != nil { + // return xerrors.Errorf("create wireguard network: %w", err) + // } + + // err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ + // Recipient: workspaceAgent.ID, + // NodePublicKey: wgn.NodePrivateKey.Public(), + // DiscoPublicKey: wgn.DiscoPublicKey, + // IPv6: ipv6, + // }) + // if err != nil { + // return xerrors.Errorf("post wireguard peer: %w", err) + // } + + // err = wgn.AddPeer(peerwg.Handshake{ + // Recipient: workspaceAgent.ID, + // DiscoPublicKey: workspaceAgent.DiscoPublicKey, + // NodePublicKey: workspaceAgent.WireguardPublicKey, + // IPv6: workspaceAgent.IPv6.IP(), + // }) + // if err != nil { + // return xerrors.Errorf("add workspace agent as peer: %w", err) + // } + + // // Start all listeners. + // var ( + // ctx, cancel = context.WithCancel(cmd.Context()) + // wg = new(sync.WaitGroup) + // listeners = make([]net.Listener, len(specs)) + // closeAllListeners = func() { + // for _, l := range listeners { + // if l == nil { + // continue + // } + // _ = l.Close() + // } + // } + // ) + // defer cancel() + // for i, spec := range specs { + // l, err := listenAndPortForwardWireguard(ctx, cmd, wgn, wg, spec, workspaceAgent.IPv6.IP()) + // if err != nil { + // closeAllListeners() + // return err + // } + // listeners[i] = l + // } + + // // Wait for the context to be canceled or for a signal and close + // // all listeners. + // var closeErr error + // go func() { + // sigs := make(chan os.Signal, 1) + // signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + + // select { + // case <-ctx.Done(): + // closeErr = ctx.Err() + // case <-sigs: + // _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Received signal, closing all listeners and active connections") + // closeErr = xerrors.New("signal received") + // } + + // cancel() + // closeAllListeners() + // }() + + // _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Ready!") + // wg.Wait() + // return closeErr + return nil }, } diff --git a/coderd/coderd.go b/coderd/coderd.go index 245e187163f92..050f5539b2cf6 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -322,14 +322,10 @@ func New(options *Options) *API { r.Get("/gitsshkey", api.agentGitSSHKey) r.Get("/turn", api.workspaceAgentTurn) r.Get("/iceservers", api.workspaceAgentICEServers) - r.Get("/derp", api.derpMap) - // Posting map under "me" sets the agents node map. - - // On the agent side, "map" is a WebSocket that sends - // updates via marshalled JSON and recieves networking - // messages on the other end. - r.Get("/netmap", api.workspaceAgentSelfNetmap) + // Everything below this is Tailnet. + r.Get("/node", api.workspaceAgentNode) + r.Get("/derpmap", api.derpMap) }) r.Route("/{workspaceagent}", func(r chi.Router) { r.Use( @@ -342,11 +338,9 @@ func New(options *Options) *API { r.Get("/turn", api.workspaceAgentTurn) r.Get("/pty", api.workspaceAgentPTY) r.Get("/iceservers", api.workspaceAgentICEServers) - r.Get("/derp", api.derpMap) - // Specific to Tailscale networking: - r.Post("/netmap") - r.Get("/tail-dial", api.workspaceAgentTailnetDial) + r.Get("/derpmap", api.derpMap) + r.Post("/node", api.postWorkspaceAgentNode) }) }) r.Route("/workspaceresources/{workspaceresource}", func(r chi.Router) { diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 5d822c8b0442c..29b37b6a3b82f 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -1925,7 +1925,7 @@ func (q *fakeQuerier) UpdateWorkspaceAgentNetworkByID(_ context.Context, arg dat agent.DiscoPublicKey = arg.DiscoPublicKey agent.NodePublicKey = arg.NodePublicKey - agent.DERP = arg.DERP + agent.PreferredDERP = arg.PreferredDERP agent.DERPLatency = arg.DERPLatency agent.UpdatedAt = database.Now() q.provisionerJobAgents[index] = agent diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 1cac38cdb4713..3d1c017463794 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -296,7 +296,7 @@ CREATE TABLE workspace_agents ( node_public_key character varying(128) DEFAULT 'nodekey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, disco_public_key character varying(128) DEFAULT 'discokey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, ip_addresses inet[] DEFAULT ARRAY[]::inet[] NOT NULL, - derp character varying(128) DEFAULT '127.3.3.40:0'::character varying NOT NULL, + preferred_derp integer DEFAULT 0 NOT NULL, derp_latency jsonb DEFAULT '{}'::jsonb NOT NULL ); diff --git a/coderd/database/migrations/000029_tailnet.up.sql b/coderd/database/migrations/000029_tailnet.up.sql index 5ac9f8af06e20..1a90ee5c19fb4 100644 --- a/coderd/database/migrations/000029_tailnet.up.sql +++ b/coderd/database/migrations/000029_tailnet.up.sql @@ -2,5 +2,5 @@ ALTER TABLE workspace_agents DROP COLUMN wireguard_node_ipv6; ALTER TABLE workspace_agents ADD COLUMN ip_addresses inet[] NOT NULL DEFAULT array[]::inet[]; ALTER TABLE workspace_agents RENAME COLUMN wireguard_node_public_key TO node_public_key; ALTER TABLE workspace_agents RENAME COLUMN wireguard_disco_public_key TO disco_public_key; -ALTER TABLE workspace_agents ADD COLUMN derp varchar(128) NOT NULL DEFAULT '127.3.3.40:0'; +ALTER TABLE workspace_agents ADD COLUMN preferred_derp integer NOT NULL DEFAULT 0; ALTER TABLE workspace_agents ADD COLUMN derp_latency jsonb NOT NULL DEFAULT '{}'; diff --git a/coderd/database/models.go b/coderd/database/models.go index ac3a1f7d990c1..e52b6726ea682 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -524,7 +524,7 @@ type WorkspaceAgent struct { NodePublicKey string `db:"node_public_key" json:"node_public_key"` DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` - DERP string `db:"derp" json:"derp"` + PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 6bc8fc8473e29..0a694d8a25c13 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2851,7 +2851,7 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP const getWorkspaceAgentByAuthToken = `-- name: GetWorkspaceAgentByAuthToken :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2884,7 +2884,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ) return i, err @@ -2892,7 +2892,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken const getWorkspaceAgentByID = `-- name: GetWorkspaceAgentByID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2923,7 +2923,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ) return i, err @@ -2931,7 +2931,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W const getWorkspaceAgentByInstanceID = `-- name: GetWorkspaceAgentByInstanceID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2964,7 +2964,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ) return i, err @@ -2972,7 +2972,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst const getWorkspaceAgentsByResourceIDs = `-- name: GetWorkspaceAgentsByResourceIDs :many SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -3009,7 +3009,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ); err != nil { return nil, err @@ -3026,7 +3026,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] } const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many -SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency FROM workspace_agents WHERE created_at > $1 +SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE created_at > $1 ` func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) { @@ -3059,7 +3059,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, created &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ); err != nil { return nil, err @@ -3095,7 +3095,7 @@ INSERT INTO ip_addresses ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, derp, derp_latency + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency ` type InsertWorkspaceAgentParams struct { @@ -3156,7 +3156,7 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa &i.NodePublicKey, &i.DiscoPublicKey, pq.Array(&i.IPAddresses), - &i.DERP, + &i.PreferredDERP, &i.DERPLatency, ) return i, err @@ -3198,7 +3198,7 @@ SET updated_at = $2, node_public_key = $3, disco_public_key = $4, - derp = $5, + preferred_derp = $5, derp_latency = $6 WHERE id = $1 @@ -3209,7 +3209,7 @@ type UpdateWorkspaceAgentNetworkByIDParams struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at"` NodePublicKey string `db:"node_public_key" json:"node_public_key"` DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` - DERP string `db:"derp" json:"derp"` + PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } @@ -3219,7 +3219,7 @@ func (q *sqlQuerier) UpdateWorkspaceAgentNetworkByID(ctx context.Context, arg Up arg.UpdatedAt, arg.NodePublicKey, arg.DiscoPublicKey, - arg.DERP, + arg.PreferredDERP, arg.DERPLatency, ) return err diff --git a/coderd/database/queries/workspaceagents.sql b/coderd/database/queries/workspaceagents.sql index 23dbf4a776bca..69b23c51dc715 100644 --- a/coderd/database/queries/workspaceagents.sql +++ b/coderd/database/queries/workspaceagents.sql @@ -77,7 +77,7 @@ SET updated_at = $2, node_public_key = $3, disco_public_key = $4, - derp = $5, + preferred_derp = $5, derp_latency = $6 WHERE id = $1; diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 5c8dbf37f3173..39e5384a38858 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -33,5 +33,5 @@ rename: rbac_roles: RBACRoles ip_address: IPAddress ip_addresses: IPAddresses - derp: DERP + preferred_derp: PreferredDERP derp_latency: DERPLatency diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index fe2f42a6757e5..7651601710e88 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "database/sql" - "encoding/base64" "encoding/json" "fmt" "io" @@ -165,17 +164,17 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) return } - ipp, ok := netaddr.FromStdIPNet(&workspaceAgent.IPAddresses.IPNet) - if !ok { - httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Workspace agent has an invalid ipv6 address.", - Detail: workspaceAgent.WireguardNodeIPv6.IPNet.String(), - }) - return - } + // ipp, ok := netaddr.FromStdIPNet(&workspaceAgent.IPAddresses.IPNet) + // if !ok { + // httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + // Message: "Workspace agent has an invalid ipv6 address.", + // Detail: workspaceAgent.WireguardNodeIPv6.IPNet.String(), + // }) + // return + // } httpapi.Write(rw, http.StatusOK, agent.Metadata{ - TailscaleAddresses: []netaddr.IPPrefix{ipp}, + TailscaleAddresses: []netaddr.IPPrefix{}, OwnerEmail: owner.Email, OwnerUsername: owner.Username, @@ -472,6 +471,74 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { _, _ = io.Copy(ptNetConn, wsNetConn) } +// dialWorkspaceAgent connects to a workspace agent by ID. Only rely on +// r.Context() for cancellation if it's use is safe or r.Hijack() has +// not been performed. +func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (*agent.Conn, error) { + client, server := provisionersdk.TransportPipe() + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + _ = peerbroker.ProxyListen(ctx, server, peerbroker.ProxyOptions{ + ChannelID: agentID.String(), + Logger: api.Logger.Named("peerbroker-proxy-dial"), + Pubsub: api.Pubsub, + }) + _ = client.Close() + _ = server.Close() + }() + + peerClient := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) + stream, err := peerClient.NegotiateConnection(ctx) + if err != nil { + cancelFunc() + return nil, xerrors.Errorf("negotiate: %w", err) + } + options := &peer.ConnOptions{ + Logger: api.Logger.Named("agent-dialer"), + } + options.SettingEngine.SetSrflxAcceptanceMinWait(0) + options.SettingEngine.SetRelayAcceptanceMinWait(0) + // Use the ProxyDialer for the TURN server. + // This is required for connections where P2P is not enabled. + options.SettingEngine.SetICEProxyDialer(turnconn.ProxyDialer(func() (c net.Conn, err error) { + clientPipe, serverPipe := net.Pipe() + go func() { + <-ctx.Done() + _ = clientPipe.Close() + _ = serverPipe.Close() + }() + localAddress, _ := r.Context().Value(http.LocalAddrContextKey).(*net.TCPAddr) + remoteAddress := &net.TCPAddr{ + IP: net.ParseIP(r.RemoteAddr), + } + // By default requests have the remote address and port. + host, port, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return nil, xerrors.Errorf("split remote address: %w", err) + } + remoteAddress.IP = net.ParseIP(host) + remoteAddress.Port, err = strconv.Atoi(port) + if err != nil { + return nil, xerrors.Errorf("convert remote port: %w", err) + } + api.TURNServer.Accept(clientPipe, remoteAddress, localAddress) + return serverPipe, nil + })) + peerConn, err := peerbroker.Dial(stream, append(api.ICEServers, turnconn.Proxy), options) + if err != nil { + cancelFunc() + return nil, xerrors.Errorf("dial: %w", err) + } + go func() { + <-peerConn.Closed() + cancelFunc() + }() + return &agent.Conn{ + Negotiator: peerClient, + Conn: peerConn, + }, nil +} + func (a *API) derpMap(rw http.ResponseWriter, _ *http.Request) { var derpPort int rawPort := a.AccessURL.Port() @@ -486,7 +553,8 @@ func (a *API) derpMap(rw http.ResponseWriter, _ *http.Request) { derpPort, err = strconv.Atoi(a.AccessURL.Port()) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Get ", + Message: "Failed to convert access URL port.", + Detail: err.Error(), }) return } @@ -494,25 +562,33 @@ func (a *API) derpMap(rw http.ResponseWriter, _ *http.Request) { httpapi.Write(rw, http.StatusOK, &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ - 1: &tailcfg.DERPRegion{ + 1: { RegionID: 1, RegionCode: "coder", RegionName: "Coder", Nodes: []*tailcfg.DERPNode{{ - Name: "1a", + Name: "1a", + RegionID: 1, + HostName: a.AccessURL.Hostname(), + DERPPort: derpPort, + STUNPort: -1, + HTTPForTests: a.AccessURL.Scheme == "http:", + }, { + Name: "1b", RegionID: 1, - HostName: a.AccessURL.Hostname(), - DERPPort: derpPort, - STUNPort: -1, + HostName: "stun.l.google.com", + STUNOnly: true, + STUNPort: 19302, }}, }, }, }) } -// workspaceAgentSelfNetmap accepts a WebSocket that reads -// node updates and sends new node connection info. -func (api *API) workspaceAgentSelfNetmap(rw http.ResponseWriter, r *http.Request) { +// workspaceAgentNode accepts a WebSocket that reads node network updates. +// After accept a PubSub starts listening for new connection node updates +// which are written to the WebSocket. +func (api *API) workspaceAgentNode(rw http.ResponseWriter, r *http.Request) { api.websocketWaitMutex.Lock() api.websocketWaitGroup.Add(1) api.websocketWaitMutex.Unlock() @@ -530,7 +606,7 @@ func (api *API) workspaceAgentSelfNetmap(rw http.ResponseWriter, r *http.Request defer conn.Close(websocket.StatusNormalClosure, "") ctx, nc := websocketNetConn(r.Context(), conn, websocket.MessageBinary) agentIDBytes, _ := workspaceAgent.ID.MarshalText() - subCancel, err := api.Pubsub.Subscribe("tailnet_dial", func(ctx context.Context, message []byte) { + subCancel, err := api.Pubsub.Subscribe("tailnet", func(ctx context.Context, message []byte) { // Since we subscribe to all peer broadcasts, we do a light check to // make sure we're the intended recipient without fully decoding the // message. @@ -558,13 +634,12 @@ func (api *API) workspaceAgentSelfNetmap(rw http.ResponseWriter, r *http.Request return } err := api.Database.UpdateWorkspaceAgentNetworkByID(ctx, database.UpdateWorkspaceAgentNetworkByIDParams{ - ID: workspaceAgent.ID, - NodePublicKey: node.Key.String(), - DERPLatency: , - TailnetNodePublicKey: node.Key.String(), - TailnetDiscoPublicKey: node.DiscoKey.String(), - TailnetNodeDERP: node.DERP, - UpdatedAt: database.Now(), + ID: workspaceAgent.ID, + NodePublicKey: node.Key.String(), + DERPLatency: node.DERPLatency, + DiscoPublicKey: node.DiscoKey.String(), + PreferredDERP: int32(node.PreferredDERP), + UpdatedAt: database.Now(), }) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ @@ -573,90 +648,42 @@ func (api *API) workspaceAgentSelfNetmap(rw http.ResponseWriter, r *http.Request }) return } - nodeData, err := json.Marshal(node) - if err != nil { - return - } - id, _ := workspaceAgent.ID.MarshalText() - msg := base64.StdEncoding.EncodeToString(nodeData) - err = api.Pubsub.Publish("tailnet_listen", append(id, msg...)) - if err != nil { - conn.Close(websocket.StatusAbnormalClosure, err.Error()) - return - } } } -func (api *API) workspaceAgentNetmap(r *http.Request, rw http.ResponseWriter) { - -} - -// dialWorkspaceAgent connects to a workspace agent by ID. Only rely on -// r.Context() for cancellation if it's use is safe or r.Hijack() has -// not been performed. -func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (*agent.Conn, error) { - client, server := provisionersdk.TransportPipe() - ctx, cancelFunc := context.WithCancel(context.Background()) - go func() { - _ = peerbroker.ProxyListen(ctx, server, peerbroker.ProxyOptions{ - ChannelID: agentID.String(), - Logger: api.Logger.Named("peerbroker-proxy-dial"), - Pubsub: api.Pubsub, - }) - _ = client.Close() - _ = server.Close() - }() +// postWorkspaceAgentNode sends networking information to a workspace agent node. +func (api *API) postWorkspaceAgentNode(rw http.ResponseWriter, r *http.Request) { + workspaceAgent := httpmw.WorkspaceAgentParam(r) + workspace := httpmw.WorkspaceParam(r) + if !api.Authorize(r, rbac.ActionUpdate, workspace) { + httpapi.ResourceNotFound(rw) + return + } - peerClient := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) - stream, err := peerClient.NegotiateConnection(ctx) - if err != nil { - cancelFunc() - return nil, xerrors.Errorf("negotiate: %w", err) + var node tailnet.Node + if !httpapi.Read(rw, r, &node) { + return } - options := &peer.ConnOptions{ - Logger: api.Logger.Named("agent-dialer"), + data, err := json.Marshal(node) + if err != nil { + httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + Message: "Failed to marshal node data.", + Detail: err.Error(), + }) + return } - options.SettingEngine.SetSrflxAcceptanceMinWait(0) - options.SettingEngine.SetRelayAcceptanceMinWait(0) - // Use the ProxyDialer for the TURN server. - // This is required for connections where P2P is not enabled. - options.SettingEngine.SetICEProxyDialer(turnconn.ProxyDialer(func() (c net.Conn, err error) { - clientPipe, serverPipe := net.Pipe() - go func() { - <-ctx.Done() - _ = clientPipe.Close() - _ = serverPipe.Close() - }() - localAddress, _ := r.Context().Value(http.LocalAddrContextKey).(*net.TCPAddr) - remoteAddress := &net.TCPAddr{ - IP: net.ParseIP(r.RemoteAddr), - } - // By default requests have the remote address and port. - host, port, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return nil, xerrors.Errorf("split remote address: %w", err) - } - remoteAddress.IP = net.ParseIP(host) - remoteAddress.Port, err = strconv.Atoi(port) - if err != nil { - return nil, xerrors.Errorf("convert remote port: %w", err) - } - api.TURNServer.Accept(clientPipe, remoteAddress, localAddress) - return serverPipe, nil - })) - peerConn, err := peerbroker.Dial(stream, append(api.ICEServers, turnconn.Proxy), options) + data = append(workspaceAgent.ID[:], data...) + err = api.Pubsub.Publish("tailnet", data) if err != nil { - cancelFunc() - return nil, xerrors.Errorf("dial: %w", err) + httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ + Message: "Publish node data.", + Detail: err.Error(), + }) + return } - go func() { - <-peerConn.Closed() - cancelFunc() - }() - return &agent.Conn{ - Negotiator: peerClient, - Conn: peerConn, - }, nil + httpapi.Write(rw, http.StatusOK, httpapi.Response{ + Message: "Published!", + }) } func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { @@ -723,7 +750,7 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work Apps: apps, NodePublicKey: nodePublicKey, DiscoPublicKey: discoPublicKey, - DERP: dbAgent.DERP, + PreferredDERP: int(dbAgent.PreferredDERP), DERPLatency: dbAgent.DERPLatency, IPAddresses: ips, } diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index c50b444947cec..5a26e1661865f 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -271,6 +271,27 @@ func (c *Client) UpdateTailscaleNode(ctx context.Context, agentID string, node * return nil } +// UpdateWorkspaceAgentNode publishes a node update for the provided agent. +// This should be used to negotiate a connection. +func (c *Client) UpdateWorkspaceAgentNode(ctx context.Context, agentID uuid.UUID, node *tailnet.Node) error { + res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/node", + agentID, + ), node) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return readBodyAsError(res) + } + return nil +} + +// +func (c *Client) ListenWorkspaceAgentNode(ctx context.Context, onNode func(node *tailnet.Node)) { + +} + // ListenTailscaleNodes listens for Tailscale node updates. Peer messages are // sent when a new client wants to connect. Once receiving a peer message, the // peer should be added to the NetworkMap of the wireguard interface. diff --git a/codersdk/workspaceresources.go b/codersdk/workspaceresources.go index 24da0f38c8c32..fbf0404a2fa25 100644 --- a/codersdk/workspaceresources.go +++ b/codersdk/workspaceresources.go @@ -52,8 +52,8 @@ type WorkspaceAgent struct { IPAddresses []netaddr.IP `json:"ip_addresses"` NodePublicKey key.NodePublic `json:"node_public_key"` DiscoPublicKey key.DiscoPublic `json:"disco_public_key"` - // DERP represents the connected region. - DERP string `json:"derp"` + // PreferredDERP represents the connected region. + PreferredDERP int `json:"preferred_derp"` // Maps DERP region to MS latency. // Fetch the DERP mapping to extract region names! DERPLatency map[string]float64 `json:"latency"` diff --git a/tailnet/tailnet.go b/tailnet/tailnet.go index 6eb99f24fe739..f8e36f77b77c5 100644 --- a/tailnet/tailnet.go +++ b/tailnet/tailnet.go @@ -237,15 +237,7 @@ type Server struct { // renegotiation may be required. Clients should constantly be emitting // node changes. func (s *Server) SetNodeCallback(callback func(node *Node)) { - s.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { - fmt.Printf("\n\n\n\nNetwork status: %+v %s\n\n\n\n", s, err) - }) - - // s.wireguardEngine.AddNetworkMapCallback(func(nm *netmap.NetworkMap) { - // fmt.Printf("\n\n\n\nNetwork map: %+v\n\n\n\n", nm) - // }) s.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { - // fmt.Printf("\n\n\n\n\nUpdating network information: %+v\n\n\n\n\n", ni) callback(&Node{ ID: s.netMap.SelfNode.ID, Key: s.netMap.SelfNode.Key, @@ -295,6 +287,7 @@ func (s *Server) UpdateNodes(nodes []*Node) error { return nil } +// Ping sends a ping to the Wireguard engine. func (s *Server) Ping(ip netaddr.IP, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { s.wireguardEngine.Ping(ip, pingType, cb) } diff --git a/tailnet/tailnet_test.go b/tailnet/tailnet_test.go index f8800467fedb3..01b99d16ce090 100644 --- a/tailnet/tailnet_test.go +++ b/tailnet/tailnet_test.go @@ -3,12 +3,10 @@ package tailnet_test import ( "context" "crypto/tls" - "fmt" "net" "net/http" "net/http/httptest" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,9 +14,11 @@ import ( "inet.af/netaddr" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" "tailscale.com/types/key" tslogger "tailscale.com/types/logger" + "tailscale.com/types/nettype" "github.com/coder/coder/tailnet" @@ -43,18 +43,6 @@ func TestTailnet(t *testing.T) { }) require.NoError(t, err) - // When a new connection occurs, we want those nodes to exist for the lifetime of the connection. - // As soon as the connection ends, the nodes can be removed. - - // The workspace agent creates a Tailnet on start. It updates keys and - // begins listening for connection messages. - // - // A new connection starts by concurrently sending a POST request with - // it's keys, and using a GET request on the workspace agent. - // - // Internally, the agent WebSocket listens for these messages. - // If the agent dies and comes back to life, - w2, err := tailnet.New(&tailnet.Options{ Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(tailnet.IP(), 128)}, Logger: logger.Named("w2"), @@ -88,8 +76,6 @@ func TestTailnet(t *testing.T) { _ = nc.Close() <-conn - time.Sleep(time.Minute) - w1.Close() w2.Close() } @@ -101,20 +87,12 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) server.StartTLS() - go func() { - time.Sleep(5 * time.Second) - fmt.Printf("\n\n\n\n\nSHUTTING IT DOWN\n\n\n\n\n") - server.CloseClientConnections() - server.Close() - d.Close() - }() - - // stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) t.Cleanup(func() { server.CloseClientConnections() server.Close() d.Close() - // stunCleanup() + stunCleanup() }) tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) @@ -135,7 +113,7 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) HostName: "test-node.dns", IPv4: "127.0.0.1", IPv6: "none", - STUNPort: -1, + STUNPort: stunAddr.Port, DERPPort: tcpAddr.Port, InsecureForTests: true, STUNTestIP: "127.0.0.1", From 38f40044e03353a318e3f88509df0ea4952e2cd0 Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sat, 9 Jul 2022 06:16:09 +0000 Subject: [PATCH 05/54] Finish refactor to make this work --- .vscode/settings.json | 2 + agent/agent.go | 257 +++++++++++------- agent/agent_test.go | 12 +- cli/agent.go | 14 +- cli/configssh_test.go | 6 +- cli/ssh_test.go | 19 +- coderd/coderd.go | 6 +- coderd/coderd_test.go | 7 +- coderd/coderdtest/coderdtest.go | 23 ++ coderd/database/dump.sql | 4 +- .../database/migrations/000029_tailnet.up.sql | 6 +- coderd/database/models.go | 4 +- coderd/database/queries.sql.go | 28 +- coderd/workspaceagents.go | 103 ++----- coderd/workspaceagents_test.go | 20 +- coderd/workspaceapps_test.go | 6 +- coderd/wsconncache/wsconncache_test.go | 19 +- codersdk/workspaceagents.go | 119 ++++---- site/src/api/typesGenerated.ts | 17 +- 19 files changed, 355 insertions(+), 317 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 52b06ca59886c..e6c5d5279753a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -75,6 +75,7 @@ "sdkproto", "sdktrace", "Signup", + "slogtest", "sourcemapped", "Srcs", "stretchr", @@ -110,6 +111,7 @@ "wgmonitor", "wgnet", "workspaceagent", + "workspaceagents", "workspaceapp", "workspaceapps", "workspacebuilds", diff --git a/agent/agent.go b/agent/agent.go index e6b7caeecc006..ae4aca3e612fd 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -46,17 +46,19 @@ const ( ) type Options struct { - EnableWireguard bool - UpdateTailscaleNode UpdateTailscaleNode - ListenTailscaleNodes ListenTailscaleNodes + EnableTailnet bool + NodeDialer NodeDialer + WebRTCDialer WebRTCDialer + FetchMetadata FetchMetadata + ReconnectingPTYTimeout time.Duration EnvironmentVariables map[string]string Logger slog.Logger } type Metadata struct { - TailscaleAddresses []netaddr.IPPrefix `json:"tailscale_addresses"` - TailscaleDERPMap *tailcfg.DERPMap `json:"tailscale_derpmap"` + IPAddresses []netaddr.IP `json:"ip_addresses"` + DERPMap *tailcfg.DERPMap `json:"derpmap"` OwnerEmail string `json:"owner_email"` OwnerUsername string `json:"owner_username"` @@ -65,37 +67,47 @@ type Metadata struct { Directory string `json:"directory"` } -type Dialer func(ctx context.Context, logger slog.Logger) (Metadata, *peerbroker.Listener, error) +type WebRTCDialer func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) -type UpdateTailscaleNode func(ctx context.Context, node *tailnet.Node) error -type ListenTailscaleNodes func(ctx context.Context, logger slog.Logger) (<-chan *tailnet.Node, func(), error) +// NodeBroker handles the exchange of node information. +type NodeBroker interface { + io.Closer + // Read will be a constant stream of incoming connection requests. + Read(ctx context.Context) (*tailnet.Node, error) + // Write should be called with the listening agent node information. + Write(ctx context.Context, node *tailnet.Node) error +} -func New(dialer Dialer, options *Options) io.Closer { - if options == nil { - options = &Options{} - } +// NodeDialer is a function that constructs a new broker. +// A dialer must be passed in to allow for reconnects. +type NodeDialer func(ctx context.Context) (NodeBroker, error) + +// FetchMetadata is a function to obtain metadata for the agent. +type FetchMetadata func(ctx context.Context) (Metadata, error) + +func New(options Options) io.Closer { if options.ReconnectingPTYTimeout == 0 { options.ReconnectingPTYTimeout = 5 * time.Minute } ctx, cancelFunc := context.WithCancel(context.Background()) server := &agent{ - dialer: dialer, + webrtcDialer: options.WebRTCDialer, reconnectingPTYTimeout: options.ReconnectingPTYTimeout, logger: options.Logger, closeCancel: cancelFunc, closed: make(chan struct{}), envVars: options.EnvironmentVariables, - enableWireguard: options.EnableWireguard, - updateTailscaleNode: options.UpdateTailscaleNode, - listenTailscaleNodes: options.ListenTailscaleNodes, + enableTailnet: options.EnableTailnet, + nodeDialer: options.NodeDialer, + fetchMetadata: options.FetchMetadata, } server.init(ctx) return server } type agent struct { - dialer Dialer - logger slog.Logger + webrtcDialer WebRTCDialer + logger slog.Logger reconnectingPTYs sync.Map reconnectingPTYTimeout time.Duration @@ -108,23 +120,21 @@ type agent struct { envVars map[string]string // metadata is atomic because values can change after reconnection. metadata atomic.Value - startupScript atomic.Bool + fetchMetadata FetchMetadata sshServer *ssh.Server - enableWireguard bool - network *tailnet.Server - updateTailscaleNode UpdateTailscaleNode - listenTailscaleNodes ListenTailscaleNodes + enableTailnet bool + network *tailnet.Server + nodeDialer NodeDialer } func (a *agent) run(ctx context.Context) { var metadata Metadata - var peerListener *peerbroker.Listener var err error // An exponential back-off occurs when the connection is failing to dial. // This is to prevent server spam in case of a coderd outage. for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { - metadata, peerListener, err = a.dialer(ctx, a.logger) + metadata, err = a.fetchMetadata(ctx) if err != nil { if errors.Is(err, context.Canceled) { return @@ -135,7 +145,7 @@ func (a *agent) run(ctx context.Context) { a.logger.Warn(context.Background(), "failed to dial", slog.Error(err)) continue } - a.logger.Info(context.Background(), "connected") + a.logger.Info(context.Background(), "fetched metadata") break } select { @@ -145,25 +155,131 @@ func (a *agent) run(ctx context.Context) { } a.metadata.Store(metadata) - if a.startupScript.CAS(false, true) { - // The startup script has not ran yet! - go func() { - err := a.runStartupScript(ctx, metadata.StartupScript) + // The startup script has not ran yet! + go func() { + err := a.runStartupScript(ctx, metadata.StartupScript) + if errors.Is(err, context.Canceled) { + return + } + if err != nil { + a.logger.Warn(ctx, "agent script failed", slog.Error(err)) + } + }() + + go a.runWebRTCNetworking(ctx) + if a.enableTailnet { + go a.runTailnet(ctx, metadata.IPAddresses, metadata.DERPMap) + } +} + +func (a *agent) runTailnet(ctx context.Context, addresses []netaddr.IP, derpMap *tailcfg.DERPMap) { + ipRanges := make([]netaddr.IPPrefix, 0, len(addresses)) + for _, address := range addresses { + ipRanges = append(ipRanges, netaddr.IPPrefixFrom(address, 128)) + } + var err error + a.network, err = tailnet.New(&tailnet.Options{ + Addresses: ipRanges, + DERPMap: derpMap, + Logger: a.logger.Named("tailnet"), + }) + if err != nil { + a.logger.Critical(ctx, "create tailnet", slog.Error(err)) + return + } + go a.runNodeBroker(ctx) + + sshListener, err := a.network.Listen("tcp", ":12212") + if err != nil { + a.logger.Critical(ctx, "listen for ssh", slog.Error(err)) + return + } + go func() { + for { + conn, err := sshListener.Accept() + if err != nil { + return + } + go a.sshServer.HandleConn(conn) + } + }() +} + +// runNodeBroker listens for nodes and updates the self-node as it changes. +func (a *agent) runNodeBroker(ctx context.Context) { + var nodeBroker NodeBroker + var err error + // An exponential back-off occurs when the connection is failing to dial. + // This is to prevent server spam in case of a coderd outage. + for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { + nodeBroker, err = a.nodeDialer(ctx) + if err != nil { if errors.Is(err, context.Canceled) { return } - if err != nil { - a.logger.Warn(ctx, "agent script failed", slog.Error(err)) + if a.isClosed() { + return } - }() + a.logger.Warn(context.Background(), "failed to dial", slog.Error(err)) + continue + } + a.logger.Info(context.Background(), "connected to node broker") + break + } + select { + case <-ctx.Done(): + return + default: + } + + a.network.SetNodeCallback(func(node *tailnet.Node) { + err := nodeBroker.Write(ctx, node) + if err != nil { + a.logger.Warn(context.Background(), "write node", slog.Error(err), slog.F("node", node)) + } + }) + + for { + node, err := nodeBroker.Read(ctx) + if err != nil { + if a.isClosed() { + return + } + a.logger.Debug(ctx, "node broker accept exited; restarting connection", slog.Error(err)) + a.runNodeBroker(ctx) + return + } + err = a.network.UpdateNodes([]*tailnet.Node{node}) + if err != nil { + a.logger.Error(ctx, "update tailnet nodes", slog.Error(err), slog.F("node", node)) + } } +} - // We don't want to reinitialize the network if it already exists. - if a.enableWireguard && a.network == nil { - err = a.startWireguard(ctx, metadata.TailscaleAddresses, metadata.TailscaleDERPMap) +func (a *agent) runWebRTCNetworking(ctx context.Context) { + var peerListener *peerbroker.Listener + var err error + // An exponential back-off occurs when the connection is failing to dial. + // This is to prevent server spam in case of a coderd outage. + for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { + peerListener, err = a.webrtcDialer(ctx, a.logger) if err != nil { - a.logger.Error(ctx, "start wireguard", slog.Error(err)) + if errors.Is(err, context.Canceled) { + return + } + if a.isClosed() { + return + } + a.logger.Warn(context.Background(), "failed to dial", slog.Error(err)) + continue } + a.logger.Info(context.Background(), "connected to webrtc broker") + break + } + select { + case <-ctx.Done(): + return + default: } for { @@ -173,7 +289,7 @@ func (a *agent) run(ctx context.Context) { return } a.logger.Debug(ctx, "peer listener accept exited; restarting connection", slog.Error(err)) - a.run(ctx) + a.runWebRTCNetworking(ctx) return } a.closeMutex.Lock() @@ -667,71 +783,6 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne } } -func (a *agent) startWireguard(ctx context.Context, addresses []netaddr.IPPrefix, derpMap *tailcfg.DERPMap) error { - var err error - a.network, err = tailnet.New(&tailnet.Options{ - Addresses: addresses, - DERPMap: derpMap, - Logger: a.logger.Named("tailnet"), - }) - if err != nil { - return err - } - a.network.SetNodeCallback(func(node *tailnet.Node) { - err := a.updateTailscaleNode(ctx, node) - if err != nil { - a.logger.Error(ctx, "update tailscale node", slog.Error(err)) - } - }) - go func() { - for { - var nodes <-chan *tailnet.Node - var err error - var listenClose func() - for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { - nodes, listenClose, err = a.listenTailscaleNodes(ctx, a.logger) - if err != nil { - if errors.Is(err, context.Canceled) { - return - } - a.logger.Warn(ctx, "listen for tailscale nodes", slog.Error(err)) - continue - } - defer listenClose() - a.logger.Info(context.Background(), "listening for tailscale nodes") - break - } - for { - var node *tailnet.Node - select { - case <-ctx.Done(): - case node = <-nodes: - } - if node == nil { - // The channel ended! - break - } - a.network.UpdateNodes([]*tailnet.Node{node}) - } - } - }() - - sshListener, err := a.network.Listen("tcp", ":12212") - if err != nil { - return xerrors.Errorf("listen for ssh: %w", err) - } - go func() { - for { - conn, err := sshListener.Accept() - if err != nil { - return - } - go a.sshServer.HandleConn(conn) - } - }() - return nil -} - // dialResponse is written to datachannels with protocol "dial" by the agent as // the first packet to signify whether the dial succeeded or failed. type dialResponse struct { diff --git a/agent/agent_test.go b/agent/agent_test.go index 55c53c20a0503..b292ed9b06d1e 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -441,10 +441,14 @@ func setupSSHSession(t *testing.T, options agent.Metadata) *ssh.Session { func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) *agent.Conn { client, server := provisionersdk.TransportPipe() - closer := agent.New(func(ctx context.Context, logger slog.Logger) (agent.Metadata, *peerbroker.Listener, error) { - listener, err := peerbroker.Listen(server, nil) - return metadata, listener, err - }, &agent.Options{ + closer := agent.New(agent.Options{ + FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { + return metadata, nil + }, + WebRTCDialer: func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) { + listener, err := peerbroker.Listen(server, nil) + return listener, err + }, Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), ReconnectingPTYTimeout: ptyTimeout, }) diff --git a/cli/agent.go b/cli/agent.go index 5e4bd5e548a35..0a74ae6a51e0e 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -22,7 +22,6 @@ import ( "github.com/coder/coder/agent/reaper" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/codersdk" - "github.com/coder/coder/tailnet" "github.com/coder/retry" ) @@ -171,18 +170,17 @@ func workspaceAgent() *cobra.Command { return xerrors.Errorf("add executable to $PATH: %w", err) } - closer := agent.New(client.ListenWorkspaceAgent, &agent.Options{ - Logger: logger, + closer := agent.New(agent.Options{ + FetchMetadata: client.WorkspaceAgentMetadata, + WebRTCDialer: client.ListenWorkspaceAgent, + Logger: logger, EnvironmentVariables: map[string]string{ // Override the "CODER_AGENT_TOKEN" variable in all // shells so "gitssh" works! "CODER_AGENT_TOKEN": client.SessionToken, }, - EnableWireguard: wireguard, - UpdateTailscaleNode: func(ctx context.Context, node *tailnet.Node) error { - return client.UpdateTailscaleNode(ctx, "me", node) - }, - ListenTailscaleNodes: client.ListenTailscaleNodes, + EnableTailnet: wireguard, + NodeDialer: client.WorkspaceAgentNodeBroker, }) <-cmd.Context().Done() return closer.Close() diff --git a/cli/configssh_test.go b/cli/configssh_test.go index df3aa0b99f872..0e181aee9be64 100644 --- a/cli/configssh_test.go +++ b/cli/configssh_test.go @@ -104,8 +104,10 @@ func TestConfigSSH(t *testing.T) { coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/cli/ssh_test.go b/cli/ssh_test.go index 171907ee06155..e9bc3a12af9a8 100644 --- a/cli/ssh_test.go +++ b/cli/ssh_test.go @@ -19,7 +19,6 @@ import ( "golang.org/x/crypto/ssh" gosshagent "golang.org/x/crypto/ssh/agent" - "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/agent" @@ -82,8 +81,10 @@ func TestSSH(t *testing.T) { pty.ExpectMatch("Waiting") agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) t.Cleanup(func() { _ = agentCloser.Close() @@ -101,8 +102,10 @@ func TestSSH(t *testing.T) { // the build and agent to connect! agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) <-ctx.Done() _ = agentCloser.Close() @@ -157,8 +160,10 @@ func TestSSH(t *testing.T) { // the build and agent to connect! agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) <-ctx.Done() _ = agentCloser.Close() diff --git a/coderd/coderd.go b/coderd/coderd.go index 050f5539b2cf6..6457b215e14b0 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -140,7 +140,6 @@ func New(options *Options) *API { r.Route("/%40{user}/{workspacename}/apps/{workspaceapp}", apps) r.Route("/@{user}/{workspacename}/apps/{workspaceapp}", apps) r.Get("/derp", derphttp.Handler(api.derpServer).ServeHTTP) - r.Get("/derpmap", api.derpMap) r.Route("/api/v2", func(r chi.Router) { r.NotFound(func(rw http.ResponseWriter, r *http.Request) { @@ -325,7 +324,6 @@ func New(options *Options) *API { // Everything below this is Tailnet. r.Get("/node", api.workspaceAgentNode) - r.Get("/derpmap", api.derpMap) }) r.Route("/{workspaceagent}", func(r chi.Router) { r.Use( @@ -339,7 +337,9 @@ func New(options *Options) *API { r.Get("/pty", api.workspaceAgentPTY) r.Get("/iceservers", api.workspaceAgentICEServers) - r.Get("/derpmap", api.derpMap) + r.Get("/derpmap", func(w http.ResponseWriter, r *http.Request) { + httpapi.Write(w, http.StatusOK, options.DERPMap) + }) r.Post("/node", api.postWorkspaceAgentNode) }) }) diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index b22ee495ca326..750caf3b1f413 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -131,6 +131,7 @@ func TestAuthorizeAllEndpoints(t *testing.T) { } assertRoute := map[string]routeCheck{ // These endpoints do not require auth + "GET:/derp": {NoAuthorize: true}, "GET:/api/v2": {NoAuthorize: true}, "GET:/api/v2/buildinfo": {NoAuthorize: true}, "GET:/api/v2/users/first": {NoAuthorize: true}, @@ -160,12 +161,10 @@ func TestAuthorizeAllEndpoints(t *testing.T) { "GET:/api/v2/workspaceagents/me/listen": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/metadata": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/turn": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/me/derp": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/me/wireguardlisten": {NoAuthorize: true}, - "POST:/api/v2/workspaceagents/me/keys": {NoAuthorize: true}, + "GET:/api/v2/workspaceagents/me/node": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/{workspaceagent}/iceservers": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/{workspaceagent}/turn": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/{workspaceagent}/derp": {NoAuthorize: true}, + "GET:/api/v2/workspaceagents/{workspaceagent}/derpmap": {NoAuthorize: true}, // These endpoints have more assertions. This is good, add more endpoints to assert if you can! "GET:/api/v2/organizations/{organization}": {AssertObject: rbac.ResourceOrganization.InOrg(admin.OrganizationID)}, diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 6a25395688508..85eca24bdc7cd 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -21,6 +21,7 @@ import ( "net/http/httptest" "net/url" "os" + "strconv" "strings" "testing" "time" @@ -35,6 +36,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/api/idtoken" "google.golang.org/api/option" + "tailscale.com/tailcfg" "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" @@ -139,6 +141,9 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API) serverURL, err := url.Parse(srv.URL) require.NoError(t, err) + derpPort, err := strconv.Atoi(serverURL.Port()) + require.NoError(t, err) + // match default with cli default if options.SSHKeygenAlgorithm == "" { options.SSHKeygenAlgorithm = gitsshkey.AlgorithmEd25519 @@ -167,6 +172,24 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API) APIRateLimit: options.APIRateLimit, Authorizer: options.Authorizer, Telemetry: telemetry.NewNoop(), + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "coder", + RegionName: "Coder", + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 1, + HostName: serverURL.Host, + DERPPort: derpPort, + STUNPort: -1, + InsecureForTests: true, + HTTPForTests: true, + }}, + }, + }, + }, }) srv.Config.Handler = coderAPI.Handler if options.IncludeProvisionerD { diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 3d1c017463794..8f6c5c6fdf136 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -293,9 +293,9 @@ CREATE TABLE workspace_agents ( instance_metadata jsonb, resource_metadata jsonb, directory character varying(4096) DEFAULT ''::character varying NOT NULL, - node_public_key character varying(128) DEFAULT 'nodekey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, - disco_public_key character varying(128) DEFAULT 'discokey:0000000000000000000000000000000000000000000000000000000000000000'::character varying NOT NULL, ip_addresses inet[] DEFAULT ARRAY[]::inet[] NOT NULL, + node_public_key character varying(128), + disco_public_key character varying(128), preferred_derp integer DEFAULT 0 NOT NULL, derp_latency jsonb DEFAULT '{}'::jsonb NOT NULL ); diff --git a/coderd/database/migrations/000029_tailnet.up.sql b/coderd/database/migrations/000029_tailnet.up.sql index 1a90ee5c19fb4..2b03666d713bd 100644 --- a/coderd/database/migrations/000029_tailnet.up.sql +++ b/coderd/database/migrations/000029_tailnet.up.sql @@ -1,6 +1,8 @@ ALTER TABLE workspace_agents DROP COLUMN wireguard_node_ipv6; +ALTER TABLE workspace_agents DROP COLUMN wireguard_node_public_key; +ALTER TABLE workspace_agents DROP COLUMN wireguard_disco_public_key; ALTER TABLE workspace_agents ADD COLUMN ip_addresses inet[] NOT NULL DEFAULT array[]::inet[]; -ALTER TABLE workspace_agents RENAME COLUMN wireguard_node_public_key TO node_public_key; -ALTER TABLE workspace_agents RENAME COLUMN wireguard_disco_public_key TO disco_public_key; +ALTER TABLE workspace_agents ADD COLUMN node_public_key varchar(128); +ALTER TABLE workspace_agents ADD COLUMN disco_public_key varchar(128); ALTER TABLE workspace_agents ADD COLUMN preferred_derp integer NOT NULL DEFAULT 0; ALTER TABLE workspace_agents ADD COLUMN derp_latency jsonb NOT NULL DEFAULT '{}'; diff --git a/coderd/database/models.go b/coderd/database/models.go index e52b6726ea682..4513584f9192f 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -521,9 +521,9 @@ type WorkspaceAgent struct { InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` Directory string `db:"directory" json:"directory"` - NodePublicKey string `db:"node_public_key" json:"node_public_key"` - DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` + NodePublicKey sql.NullString `db:"node_public_key" json:"node_public_key"` + DiscoPublicKey sql.NullString `db:"disco_public_key" json:"disco_public_key"` PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0a694d8a25c13..831c63d4f794a 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2851,7 +2851,7 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP const getWorkspaceAgentByAuthToken = `-- name: GetWorkspaceAgentByAuthToken :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2881,9 +2881,9 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ) @@ -2892,7 +2892,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken const getWorkspaceAgentByID = `-- name: GetWorkspaceAgentByID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2920,9 +2920,9 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ) @@ -2931,7 +2931,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W const getWorkspaceAgentByInstanceID = `-- name: GetWorkspaceAgentByInstanceID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -2961,9 +2961,9 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ) @@ -2972,7 +2972,7 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst const getWorkspaceAgentsByResourceIDs = `-- name: GetWorkspaceAgentsByResourceIDs :many SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE @@ -3006,9 +3006,9 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ); err != nil { @@ -3026,7 +3026,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] } const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many -SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency FROM workspace_agents WHERE created_at > $1 +SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE created_at > $1 ` func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) { @@ -3056,9 +3056,9 @@ func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, created &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ); err != nil { @@ -3095,7 +3095,7 @@ INSERT INTO ip_addresses ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, node_public_key, disco_public_key, ip_addresses, preferred_derp, derp_latency + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency ` type InsertWorkspaceAgentParams struct { @@ -3153,9 +3153,9 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, + pq.Array(&i.IPAddresses), &i.NodePublicKey, &i.DiscoPublicKey, - pq.Array(&i.IPAddresses), &i.PreferredDERP, &i.DERPLatency, ) @@ -3207,8 +3207,8 @@ WHERE type UpdateWorkspaceAgentNetworkByIDParams struct { ID uuid.UUID `db:"id" json:"id"` UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - NodePublicKey string `db:"node_public_key" json:"node_public_key"` - DiscoPublicKey string `db:"disco_public_key" json:"disco_public_key"` + NodePublicKey sql.NullString `db:"node_public_key" json:"node_public_key"` + DiscoPublicKey sql.NullString `db:"disco_public_key" json:"disco_public_key"` PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 7651601710e88..b434cab032599 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -19,7 +19,6 @@ import ( "golang.org/x/xerrors" "inet.af/netaddr" "nhooyr.io/websocket" - "tailscale.com/tailcfg" "tailscale.com/types/key" "cdr.dev/slog" @@ -164,17 +163,9 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) return } - // ipp, ok := netaddr.FromStdIPNet(&workspaceAgent.IPAddresses.IPNet) - // if !ok { - // httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - // Message: "Workspace agent has an invalid ipv6 address.", - // Detail: workspaceAgent.WireguardNodeIPv6.IPNet.String(), - // }) - // return - // } - httpapi.Write(rw, http.StatusOK, agent.Metadata{ - TailscaleAddresses: []netaddr.IPPrefix{}, + IPAddresses: apiAgent.IPAddresses, + DERPMap: api.DERPMap, OwnerEmail: owner.Email, OwnerUsername: owner.Username, @@ -539,52 +530,6 @@ func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (*agent.C }, nil } -func (a *API) derpMap(rw http.ResponseWriter, _ *http.Request) { - var derpPort int - rawPort := a.AccessURL.Port() - if rawPort == "" { - if a.AccessURL.Scheme == "https" { - derpPort = 443 - } else { - derpPort = 80 - } - } else { - var err error - derpPort, err = strconv.Atoi(a.AccessURL.Port()) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ - Message: "Failed to convert access URL port.", - Detail: err.Error(), - }) - return - } - } - - httpapi.Write(rw, http.StatusOK, &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, - RegionCode: "coder", - RegionName: "Coder", - Nodes: []*tailcfg.DERPNode{{ - Name: "1a", - RegionID: 1, - HostName: a.AccessURL.Hostname(), - DERPPort: derpPort, - STUNPort: -1, - HTTPForTests: a.AccessURL.Scheme == "http:", - }, { - Name: "1b", - RegionID: 1, - HostName: "stun.l.google.com", - STUNOnly: true, - STUNPort: 19302, - }}, - }, - }, - }) -} - // workspaceAgentNode accepts a WebSocket that reads node network updates. // After accept a PubSub starts listening for new connection node updates // which are written to the WebSocket. @@ -634,12 +579,18 @@ func (api *API) workspaceAgentNode(rw http.ResponseWriter, r *http.Request) { return } err := api.Database.UpdateWorkspaceAgentNetworkByID(ctx, database.UpdateWorkspaceAgentNetworkByIDParams{ - ID: workspaceAgent.ID, - NodePublicKey: node.Key.String(), - DERPLatency: node.DERPLatency, - DiscoPublicKey: node.DiscoKey.String(), - PreferredDERP: int32(node.PreferredDERP), - UpdatedAt: database.Now(), + ID: workspaceAgent.ID, + NodePublicKey: sql.NullString{ + String: node.Key.String(), + Valid: true, + }, + DERPLatency: node.DERPLatency, + DiscoPublicKey: sql.NullString{ + String: node.DiscoKey.String(), + Valid: true, + }, + PreferredDERP: int32(node.PreferredDERP), + UpdatedAt: database.Now(), }) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ @@ -720,15 +671,6 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work return codersdk.WorkspaceAgent{}, xerrors.Errorf("unmarshal: %w", err) } } - nodePublicKey, err := key.ParseNodePublicUntyped(mem.S(dbAgent.NodePublicKey)) - if err != nil { - return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse node public key: %w", err) - } - var discoPublicKey key.DiscoPublic - err = discoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey)) - if err != nil { - return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse disco public key: %w", err) - } ips := make([]netaddr.IP, 0) for _, ip := range dbAgent.IPAddresses { var ipData [16]byte @@ -748,13 +690,26 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work EnvironmentVariables: envs, Directory: dbAgent.Directory, Apps: apps, - NodePublicKey: nodePublicKey, - DiscoPublicKey: discoPublicKey, PreferredDERP: int(dbAgent.PreferredDERP), DERPLatency: dbAgent.DERPLatency, IPAddresses: ips, } + if dbAgent.NodePublicKey.Valid { + var err error + workspaceAgent.NodePublicKey, err = key.ParseNodePublicUntyped(mem.S(dbAgent.NodePublicKey.String)) + if err != nil { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse node public key: %w", err) + } + } + if dbAgent.DiscoPublicKey.Valid { + var err error + err = workspaceAgent.DiscoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey.String)) + if err != nil { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse disco public key: %w", err) + } + } + if dbAgent.FirstConnectedAt.Valid { workspaceAgent.FirstConnectedAt = &dbAgent.FirstConnectedAt.Time } diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index c658791f93941..9aca0db5b6b27 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -104,8 +104,10 @@ func TestWorkspaceAgentListen(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) t.Cleanup(func() { _ = agentCloser.Close() @@ -188,7 +190,7 @@ func TestWorkspaceAgentListen(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - _, _, err = agentClient.ListenWorkspaceAgent(ctx, slogtest.Make(t, nil)) + _, err = agentClient.ListenWorkspaceAgent(ctx, slogtest.Make(t, nil)) require.Error(t, err) require.ErrorContains(t, err, "build is outdated") }) @@ -228,8 +230,10 @@ func TestWorkspaceAgentTURN(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) t.Cleanup(func() { _ = agentCloser.Close() @@ -289,8 +293,10 @@ func TestWorkspaceAgentPTY(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 399b1874dc6aa..300515d378863 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -74,8 +74,10 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken - agentCloser := agent.New(agentClient.ListenWorkspaceAgent, &agent.Options{ - Logger: slogtest.Make(t, nil), + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index 34ce39e20b86d..01d38859bfb82 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -141,14 +141,17 @@ func TestCache(t *testing.T) { func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) *agent.Conn { client, server := provisionersdk.TransportPipe() - closer := agent.New(func(ctx context.Context, logger slog.Logger) (agent.Metadata, *peerbroker.Listener, error) { - listener, err := peerbroker.Listen(server, func(ctx context.Context) ([]webrtc.ICEServer, *peer.ConnOptions, error) { - return nil, &peer.ConnOptions{ - Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), - }, nil - }) - return metadata, listener, err - }, &agent.Options{ + closer := agent.New(agent.Options{ + FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { + return metadata, nil + }, + WebRTCDialer: func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) { + return peerbroker.Listen(server, func(ctx context.Context) ([]webrtc.ICEServer, *peer.ConnOptions, error) { + return nil, &peer.ConnOptions{ + Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), + }, nil + }) + }, Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), ReconnectingPTYTimeout: ptyTimeout, }) diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 5a26e1661865f..0261b611b9ff9 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -16,6 +16,7 @@ import ( "golang.org/x/net/proxy" "golang.org/x/xerrors" "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" "cdr.dev/slog" @@ -177,16 +178,30 @@ func (c *Client) AuthWorkspaceAzureInstanceIdentity(ctx context.Context) (Worksp return resp, json.NewDecoder(res.Body).Decode(&resp) } +// WorkspaceAgentMetadata fetches metadata for the currently authenticated workspace agent. +func (c *Client) WorkspaceAgentMetadata(ctx context.Context) (agent.Metadata, error) { + res, err := c.Request(ctx, http.MethodGet, "/api/v2/workspaceagents/me/metadata", nil) + if err != nil { + return agent.Metadata{}, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return agent.Metadata{}, readBodyAsError(res) + } + var agentMetadata agent.Metadata + return agentMetadata, json.NewDecoder(res.Body).Decode(&agentMetadata) +} + // ListenWorkspaceAgent connects as a workspace agent identifying with the session token. // On each inbound connection request, connection info is fetched. -func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) (agent.Metadata, *peerbroker.Listener, error) { +func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) { serverURL, err := c.URL.Parse("/api/v2/workspaceagents/me/listen") if err != nil { - return agent.Metadata{}, nil, xerrors.Errorf("parse url: %w", err) + return nil, xerrors.Errorf("parse url: %w", err) } jar, err := cookiejar.New(nil) if err != nil { - return agent.Metadata{}, nil, xerrors.Errorf("create cookie jar: %w", err) + return nil, xerrors.Errorf("create cookie jar: %w", err) } jar.SetCookies(serverURL, []*http.Cookie{{ Name: httpmw.SessionTokenKey, @@ -202,17 +217,17 @@ func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) ( }) if err != nil { if res == nil { - return agent.Metadata{}, nil, err + return nil, err } - return agent.Metadata{}, nil, readBodyAsError(res) + return nil, readBodyAsError(res) } config := yamux.DefaultConfig() config.LogOutput = io.Discard session, err := yamux.Client(websocket.NetConn(ctx, conn, websocket.MessageBinary), config) if err != nil { - return agent.Metadata{}, nil, xerrors.Errorf("multiplex client: %w", err) + return nil, xerrors.Errorf("multiplex client: %w", err) } - listener, err := peerbroker.Listen(session, func(ctx context.Context) ([]webrtc.ICEServer, *peer.ConnOptions, error) { + return peerbroker.Listen(session, func(ctx context.Context) ([]webrtc.ICEServer, *peer.ConnOptions, error) { // This can be cached if it adds to latency too much. res, err := c.Request(ctx, http.MethodGet, "/api/v2/workspaceagents/me/iceservers", nil) if err != nil { @@ -238,37 +253,6 @@ func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) ( Logger: logger, }, nil }) - if err != nil { - return agent.Metadata{}, nil, xerrors.Errorf("listen peerbroker: %w", err) - } - res, err = c.Request(ctx, http.MethodGet, "/api/v2/workspaceagents/me/metadata", nil) - if err != nil { - return agent.Metadata{}, nil, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return agent.Metadata{}, nil, readBodyAsError(res) - } - var agentMetadata agent.Metadata - return agentMetadata, listener, json.NewDecoder(res.Body).Decode(&agentMetadata) -} - -// PostWireguardPeer announces your public keys and IPv6 address to the -// specified recipient. -func (c *Client) UpdateTailscaleNode(ctx context.Context, agentID string, node *tailnet.Node) error { - res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/peer", - agentID, - ), node) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusNoContent { - return readBodyAsError(res) - } - - _, _ = io.Copy(io.Discard, res.Body) - return nil } // UpdateWorkspaceAgentNode publishes a node update for the provided agent. @@ -287,22 +271,14 @@ func (c *Client) UpdateWorkspaceAgentNode(ctx context.Context, agentID uuid.UUID return nil } -// -func (c *Client) ListenWorkspaceAgentNode(ctx context.Context, onNode func(node *tailnet.Node)) { - -} - -// ListenTailscaleNodes listens for Tailscale node updates. Peer messages are -// sent when a new client wants to connect. Once receiving a peer message, the -// peer should be added to the NetworkMap of the wireguard interface. -func (c *Client) ListenTailscaleNodes(ctx context.Context, logger slog.Logger) (<-chan *tailnet.Node, func(), error) { - serverURL, err := c.URL.Parse("/api/v2/workspaceagents/me/wireguardlisten") +func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker, error) { + serverURL, err := c.URL.Parse("/api/v2/workspaceagents/me/node") if err != nil { - return nil, nil, xerrors.Errorf("parse url: %w", err) + return nil, xerrors.Errorf("parse url: %w", err) } jar, err := cookiejar.New(nil) if err != nil { - return nil, nil, xerrors.Errorf("create cookie jar: %w", err) + return nil, xerrors.Errorf("create cookie jar: %w", err) } jar.SetCookies(serverURL, []*http.Cookie{{ Name: httpmw.SessionTokenKey, @@ -319,28 +295,11 @@ func (c *Client) ListenTailscaleNodes(ctx context.Context, logger slog.Logger) ( }) if err != nil { if res == nil { - return nil, nil, xerrors.Errorf("websocket dial: %w", err) + return nil, xerrors.Errorf("websocket dial: %w", err) } - return nil, nil, readBodyAsError(res) + return nil, readBodyAsError(res) } - - ch := make(chan *tailnet.Node, 1) - go func() { - defer conn.Close(websocket.StatusGoingAway, "") - defer close(ch) - - decoder := json.NewDecoder(websocket.NetConn(ctx, conn, websocket.MessageBinary)) - for { - var node *tailnet.Node - err = decoder.Decode(node) - if err != nil { - break - } - ch <- node - } - }() - - return ch, func() { _ = conn.Close(websocket.StatusGoingAway, "") }, nil + return &workspaceAgentNodeBroker{conn}, nil } // DialWorkspaceAgent creates a connection to the specified resource. @@ -480,3 +439,23 @@ func (c *Client) turnProxyDialer(ctx context.Context, httpClient *http.Client, p return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil }) } + +// workspaceAgentNodeBroker is used to listen for node updates +// and write them. +type workspaceAgentNodeBroker struct { + conn *websocket.Conn +} + +func (w *workspaceAgentNodeBroker) Read(ctx context.Context) (*tailnet.Node, error) { + var node *tailnet.Node + err := wsjson.Read(ctx, w.conn, node) + return node, err +} + +func (w *workspaceAgentNodeBroker) Write(ctx context.Context, node *tailnet.Node) error { + return wsjson.Write(ctx, w.conn, node) +} + +func (w *workspaceAgentNodeBroker) Close() error { + return w.conn.Close(websocket.StatusGoingAway, "") +} diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 1edaf731999e1..535ba2b46b57d 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -1,6 +1,6 @@ // Code generated by 'make coder/scripts/apitypings/main.go'. DO NOT EDIT. -// From codersdk/workspaceagents.go:36:6 +// From codersdk/workspaceagents.go:37:6 export interface AWSInstanceIdentityToken { readonly signature: string readonly document: string @@ -18,7 +18,7 @@ export interface AuthMethods { readonly github: boolean } -// From codersdk/workspaceagents.go:41:6 +// From codersdk/workspaceagents.go:42:6 export interface AzureInstanceIdentityToken { readonly signature: string readonly encoding: string @@ -129,7 +129,7 @@ export interface GitSSHKey { readonly public_key: string } -// From codersdk/workspaceagents.go:32:6 +// From codersdk/workspaceagents.go:33:6 export interface GoogleInstanceIdentityToken { readonly json_web_token: string } @@ -408,11 +408,11 @@ export interface WorkspaceAgent { // Named type "tailscale.com/types/key.DiscoPublic" unknown, using "any" // eslint-disable-next-line @typescript-eslint/no-explicit-any readonly disco_public_key: any - readonly derp: string + readonly preferred_derp: number readonly latency: Record } -// From codersdk/workspaceagents.go:48:6 +// From codersdk/workspaceagents.go:49:6 export interface WorkspaceAgentAuthenticateResponse { readonly session_token: string } @@ -493,6 +493,13 @@ export interface WorkspaceResource { readonly agents?: WorkspaceAgent[] } +// From codersdk/workspaceagents.go:445:6 +export interface workspaceAgentNodeBroker { + // Named type "nhooyr.io/websocket.Conn" unknown, using "any" + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readonly conn?: any +} + // From codersdk/workspacebuilds.go:22:6 export type BuildReason = "autostart" | "autostop" | "initiator" From 5ba96b5c38ce3a830e19c2975b7eba81f0ac32bb Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sat, 9 Jul 2022 06:21:50 +0000 Subject: [PATCH 06/54] Add interface for swapping --- agent/agent_test.go | 4 ++-- agent/conn.go | 26 ++++++++++++++++++++------ cli/portforward.go | 2 +- coderd/workspaceagents.go | 4 ++-- coderd/wsconncache/wsconncache_test.go | 12 ++++++------ codersdk/workspaceagents.go | 4 ++-- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index b292ed9b06d1e..189f0774ebd72 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -439,7 +439,7 @@ func setupSSHSession(t *testing.T, options agent.Metadata) *ssh.Session { return session } -func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) *agent.Conn { +func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) agent.Conn { client, server := provisionersdk.TransportPipe() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { @@ -468,7 +468,7 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) _ = conn.Close() }) - return &agent.Conn{ + return &agent.WebRTCConn{ Negotiator: api, Conn: conn, } diff --git a/agent/conn.go b/agent/conn.go index 0be45bc05c33e..0fce303b64961 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "fmt" + "io" "net" "net/url" "strings" + "time" "golang.org/x/crypto/ssh" "golang.org/x/xerrors" @@ -23,9 +25,21 @@ type ReconnectingPTYRequest struct { Width uint16 `json:"width"` } +// Conn is a temporary interface while we switch from WebRTC to Wireguard networking. +type Conn interface { + io.Closer + Closed() <-chan struct{} + Ping() (time.Duration, error) + CloseWithError(err error) error + ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) + SSH() (net.Conn, error) + SSHClient() (*ssh.Client, error) + DialContext(ctx context.Context, network string, addr string) (net.Conn, error) +} + // Conn wraps a peer connection with helper functions to // communicate with the agent. -type Conn struct { +type WebRTCConn struct { // Negotiator is responsible for exchanging messages. Negotiator proto.DRPCPeerBrokerClient @@ -36,7 +50,7 @@ type Conn struct { // be reconnected to via ID. // // The command is optional and defaults to start a shell. -func (c *Conn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { +func (c *WebRTCConn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { channel, err := c.CreateChannel(context.Background(), fmt.Sprintf("%s:%d:%d:%s", id, height, width, command), &peer.ChannelOptions{ Protocol: ProtocolReconnectingPTY, }) @@ -47,7 +61,7 @@ func (c *Conn) ReconnectingPTY(id string, height, width uint16, command string) } // SSH dials the built-in SSH server. -func (c *Conn) SSH() (net.Conn, error) { +func (c *WebRTCConn) SSH() (net.Conn, error) { channel, err := c.CreateChannel(context.Background(), "ssh", &peer.ChannelOptions{ Protocol: ProtocolSSH, }) @@ -59,7 +73,7 @@ func (c *Conn) SSH() (net.Conn, error) { // SSHClient calls SSH to create a client that uses a weak cipher // for high throughput. -func (c *Conn) SSHClient() (*ssh.Client, error) { +func (c *WebRTCConn) SSHClient() (*ssh.Client, error) { netConn, err := c.SSH() if err != nil { return nil, xerrors.Errorf("ssh: %w", err) @@ -78,7 +92,7 @@ func (c *Conn) SSHClient() (*ssh.Client, error) { // DialContext dials an arbitrary protocol+address from inside the workspace and // proxies it through the provided net.Conn. -func (c *Conn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { +func (c *WebRTCConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { u := &url.URL{ Scheme: network, } @@ -112,7 +126,7 @@ func (c *Conn) DialContext(ctx context.Context, network string, addr string) (ne return channel.NetConn(), nil } -func (c *Conn) Close() error { +func (c *WebRTCConn) Close() error { _ = c.Negotiator.DRPCConn().Close() return c.Conn.Close() } diff --git a/cli/portforward.go b/cli/portforward.go index de8f81ea2321b..22ccc34841697 100644 --- a/cli/portforward.go +++ b/cli/portforward.go @@ -157,7 +157,7 @@ func portForward() *cobra.Command { return cmd } -func listenAndPortForward(ctx context.Context, cmd *cobra.Command, conn *coderagent.Conn, wg *sync.WaitGroup, spec portForwardSpec) (net.Listener, error) { +func listenAndPortForward(ctx context.Context, cmd *cobra.Command, conn coderagent.Conn, wg *sync.WaitGroup, spec portForwardSpec) (net.Listener, error) { _, _ = fmt.Fprintf(cmd.OutOrStderr(), "Forwarding '%v://%v' locally to '%v://%v' in the workspace\n", spec.listenNetwork, spec.listenAddress, spec.dialNetwork, spec.dialAddress) var ( diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index b434cab032599..491586a0a5b33 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -465,7 +465,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { // dialWorkspaceAgent connects to a workspace agent by ID. Only rely on // r.Context() for cancellation if it's use is safe or r.Hijack() has // not been performed. -func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (*agent.Conn, error) { +func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Conn, error) { client, server := provisionersdk.TransportPipe() ctx, cancelFunc := context.WithCancel(context.Background()) go func() { @@ -524,7 +524,7 @@ func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (*agent.C <-peerConn.Closed() cancelFunc() }() - return &agent.Conn{ + return &agent.WebRTCConn{ Negotiator: peerClient, Conn: peerConn, }, nil diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index 01d38859bfb82..5f4ce2bb7b257 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -37,7 +37,7 @@ func TestCache(t *testing.T) { t.Parallel() t.Run("Same", func(t *testing.T) { t.Parallel() - cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (*agent.Conn, error) { + cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (agent.Conn, error) { return setupAgent(t, agent.Metadata{}, 0), nil }, 0) t.Cleanup(func() { @@ -52,7 +52,7 @@ func TestCache(t *testing.T) { t.Run("Expire", func(t *testing.T) { t.Parallel() called := atomic.NewInt32(0) - cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (*agent.Conn, error) { + cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (agent.Conn, error) { called.Add(1) return setupAgent(t, agent.Metadata{}, 0), nil }, time.Microsecond) @@ -71,7 +71,7 @@ func TestCache(t *testing.T) { }) t.Run("NoExpireWhenLocked", func(t *testing.T) { t.Parallel() - cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (*agent.Conn, error) { + cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (agent.Conn, error) { return setupAgent(t, agent.Metadata{}, 0), nil }, time.Microsecond) t.Cleanup(func() { @@ -103,7 +103,7 @@ func TestCache(t *testing.T) { }) go server.Serve(random) - cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (*agent.Conn, error) { + cache := wsconncache.New(func(r *http.Request, id uuid.UUID) (agent.Conn, error) { return setupAgent(t, agent.Metadata{}, 0), nil }, time.Microsecond) t.Cleanup(func() { @@ -139,7 +139,7 @@ func TestCache(t *testing.T) { }) } -func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) *agent.Conn { +func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) agent.Conn { client, server := provisionersdk.TransportPipe() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { @@ -171,7 +171,7 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) _ = conn.Close() }) - return &agent.Conn{ + return &agent.WebRTCConn{ Negotiator: api, Conn: conn, } diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 0261b611b9ff9..eb2cebfed1e36 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -303,7 +303,7 @@ func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker } // DialWorkspaceAgent creates a connection to the specified resource. -func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *peer.ConnOptions) (*agent.Conn, error) { +func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *peer.ConnOptions) (agent.Conn, error) { serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/dial", agentID.String())) if err != nil { return nil, xerrors.Errorf("parse url: %w", err) @@ -368,7 +368,7 @@ func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, opti if err != nil { return nil, xerrors.Errorf("dial peer: %w", err) } - return &agent.Conn{ + return &agent.WebRTCConn{ Negotiator: client, Conn: peerConn, }, nil From 7fe91a2a018af90aade1e7e8392972ef8ea8b68b Mon Sep 17 00:00:00 2001 From: kylecarbs Date: Sat, 9 Jul 2022 20:16:55 +0000 Subject: [PATCH 07/54] Fix conncache with interface --- agent/agent.go | 3 ++ agent/conn.go | 53 ++++++++++++++++++++++++ coderd/coderdtest/coderdtest.go | 8 +++- coderd/workspaceagents.go | 24 +++++------ coderd/workspaceagents_test.go | 63 ++++++++++++++++++++++++++++ coderd/wsconncache/wsconncache.go | 4 +- codersdk/workspaceagents.go | 69 +++++++++++++++++++++++++++++-- 7 files changed, 204 insertions(+), 20 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index ae4aca3e612fd..d3dd1b57c1760 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -865,6 +865,9 @@ func (a *agent) Close() error { } close(a.closed) a.closeCancel() + if a.network != nil { + _ = a.network.Close() + } _ = a.sshServer.Close() a.connCloseWait.Wait() return nil diff --git a/agent/conn.go b/agent/conn.go index 0fce303b64961..555854d5dd413 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -7,14 +7,17 @@ import ( "io" "net" "net/url" + "strconv" "strings" "time" "golang.org/x/crypto/ssh" "golang.org/x/xerrors" + "inet.af/netaddr" "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker/proto" + "github.com/coder/coder/tailnet" ) // ReconnectingPTYRequest is sent from the client to the server @@ -130,3 +133,53 @@ func (c *WebRTCConn) Close() error { _ = c.Negotiator.DRPCConn().Close() return c.Conn.Close() } + +type TailnetConn struct { + Target netaddr.IP + *tailnet.Server +} + +func (c *TailnetConn) Closed() <-chan struct{} { + return nil +} + +func (c *TailnetConn) Ping() (time.Duration, error) { + return 0, nil +} + +func (c *TailnetConn) CloseWithError(err error) error { + return c.Close() +} + +func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { + return nil, xerrors.New("not implemented") +} + +func (c *TailnetConn) SSH() (net.Conn, error) { + return c.DialContextTCP(context.Background(), netaddr.IPPortFrom(c.Target, 12212)) +} + +// SSHClient calls SSH to create a client that uses a weak cipher +// for high throughput. +func (c *TailnetConn) SSHClient() (*ssh.Client, error) { + netConn, err := c.SSH() + if err != nil { + return nil, xerrors.Errorf("ssh: %w", err) + } + sshConn, channels, requests, err := ssh.NewClientConn(netConn, "localhost:22", &ssh.ClientConfig{ + // SSH host validation isn't helpful, because obtaining a peer + // connection already signifies user-intent to dial a workspace. + // #nosec + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + }) + if err != nil { + return nil, xerrors.Errorf("ssh conn: %w", err) + } + return ssh.NewClient(sshConn, channels, requests), nil +} + +func (c *TailnetConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { + _, rawPort, _ := net.SplitHostPort(addr) + port, _ := strconv.Atoi(rawPort) + return c.Server.DialContextTCP(ctx, netaddr.IPPortFrom(c.Target, uint16(port))) +} diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 85eca24bdc7cd..c5b8428b6c152 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -181,11 +181,17 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API) Nodes: []*tailcfg.DERPNode{{ Name: "1a", RegionID: 1, - HostName: serverURL.Host, + IPv4: "127.0.0.1", DERPPort: derpPort, STUNPort: -1, InsecureForTests: true, HTTPForTests: true, + }, { + Name: "1b", + RegionID: 1, + STUNOnly: true, + HostName: "stun.l.google.com", + STUNPort: 19302, }}, }, }, diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 491586a0a5b33..4cf1ce2fbe873 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -15,11 +15,10 @@ import ( "github.com/google/uuid" "github.com/hashicorp/yamux" "github.com/tabbed/pqtype" - "go4.org/mem" "golang.org/x/xerrors" "inet.af/netaddr" "nhooyr.io/websocket" - "tailscale.com/types/key" + "nhooyr.io/websocket/wsjson" "cdr.dev/slog" "github.com/coder/coder/agent" @@ -549,7 +548,6 @@ func (api *API) workspaceAgentNode(rw http.ResponseWriter, r *http.Request) { return } defer conn.Close(websocket.StatusNormalClosure, "") - ctx, nc := websocketNetConn(r.Context(), conn, websocket.MessageBinary) agentIDBytes, _ := workspaceAgent.ID.MarshalText() subCancel, err := api.Pubsub.Subscribe("tailnet", func(ctx context.Context, message []byte) { // Since we subscribe to all peer broadcasts, we do a light check to @@ -560,25 +558,24 @@ func (api *API) workspaceAgentNode(rw http.ResponseWriter, r *http.Request) { return } // We aren't the intended recipient. - if !bytes.Equal(message[:len(agentIDBytes)-1], agentIDBytes) { + if !bytes.Equal(message[:len(agentIDBytes)], agentIDBytes) { return } - _, _ = nc.Write(message) + _ = conn.Write(ctx, websocket.MessageText, message[len(agentIDBytes):]) }) if err != nil { - api.Logger.Error(ctx, "pubsub listen", slog.Error(err)) + api.Logger.Error(context.Background(), "pubsub listen", slog.Error(err)) return } defer subCancel() - decoder := json.NewDecoder(nc) for { var node tailnet.Node - err = decoder.Decode(&node) + err = wsjson.Read(r.Context(), conn, &node) if err != nil { return } - err := api.Database.UpdateWorkspaceAgentNetworkByID(ctx, database.UpdateWorkspaceAgentNetworkByIDParams{ + err := api.Database.UpdateWorkspaceAgentNetworkByID(r.Context(), database.UpdateWorkspaceAgentNetworkByIDParams{ ID: workspaceAgent.ID, NodePublicKey: sql.NullString{ String: node.Key.String(), @@ -623,7 +620,8 @@ func (api *API) postWorkspaceAgentNode(rw http.ResponseWriter, r *http.Request) }) return } - data = append(workspaceAgent.ID[:], data...) + agentIDBytes, _ := workspaceAgent.ID.MarshalText() + data = append(agentIDBytes, data...) err = api.Pubsub.Publish("tailnet", data) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{ @@ -696,15 +694,13 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work } if dbAgent.NodePublicKey.Valid { - var err error - workspaceAgent.NodePublicKey, err = key.ParseNodePublicUntyped(mem.S(dbAgent.NodePublicKey.String)) + err := workspaceAgent.NodePublicKey.UnmarshalText([]byte(dbAgent.NodePublicKey.String)) if err != nil { return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse node public key: %w", err) } } if dbAgent.DiscoPublicKey.Valid { - var err error - err = workspaceAgent.DiscoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey.String)) + err := workspaceAgent.DiscoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey.String)) if err != nil { return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse disco public key: %w", err) } diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 9aca0db5b6b27..582db328c7d33 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "encoding/json" + "fmt" "runtime" "strings" "testing" @@ -253,6 +254,68 @@ func TestWorkspaceAgentTURN(t *testing.T) { require.NoError(t, err) } +func TestWorkspaceAgentTailnet(t *testing.T) { + t.Parallel() + client, coderAPI := coderdtest.NewWithAPI(t, nil) + user := coderdtest.CreateFirstUser(t, client) + daemonCloser := coderdtest.NewProvisionerDaemon(t, coderAPI) + authToken := uuid.NewString() + version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionDryRun: echo.ProvisionComplete, + Provision: []*proto.Provision_Response{{ + Type: &proto.Provision_Response_Complete{ + Complete: &proto.Provision_Complete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Auth: &proto.Agent_Token{ + Token: authToken, + }, + }}, + }}, + }, + }, + }}, + }) + template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) + coderdtest.AwaitTemplateVersionJob(t, client, version.ID) + workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID) + coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) + daemonCloser.Close() + + agentClient := codersdk.New(client.URL) + agentClient.SessionToken = authToken + agentCloser := agent.New(agent.Options{ + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + EnableTailnet: true, + NodeDialer: agentClient.WorkspaceAgentNodeBroker, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + }) + t.Cleanup(func() { + _ = agentCloser.Close() + }) + resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) + + time.Sleep(3 * time.Second) + + conn, err := client.DialWorkspaceAgentTailnet(context.Background(), resources[0].Agents[0].ID, slogtest.Make(t, nil).Named("tailnet").Leveled(slog.LevelDebug)) + require.NoError(t, err) + t.Cleanup(func() { + _ = conn.Close() + }) + sshClient, err := conn.SSHClient() + require.NoError(t, err) + session, err := sshClient.NewSession() + require.NoError(t, err) + output, err := session.CombinedOutput("echo test") + require.NoError(t, err) + fmt.Printf("Output: %s\n", output) +} + func TestWorkspaceAgentPTY(t *testing.T) { t.Parallel() if runtime.GOOS == "windows" { diff --git a/coderd/wsconncache/wsconncache.go b/coderd/wsconncache/wsconncache.go index 7d3b741a63b7e..698f467a40790 100644 --- a/coderd/wsconncache/wsconncache.go +++ b/coderd/wsconncache/wsconncache.go @@ -32,11 +32,11 @@ func New(dialer Dialer, inactiveTimeout time.Duration) *Cache { } // Dialer creates a new agent connection by ID. -type Dialer func(r *http.Request, id uuid.UUID) (*agent.Conn, error) +type Dialer func(r *http.Request, id uuid.UUID) (agent.Conn, error) // Conn wraps an agent connection with a reusable HTTP transport. type Conn struct { - *agent.Conn + agent.Conn locks atomic.Uint64 timeoutMutex sync.Mutex diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index eb2cebfed1e36..0889d6b5bc9c4 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -15,8 +15,10 @@ import ( "github.com/pion/webrtc/v3" "golang.org/x/net/proxy" "golang.org/x/xerrors" + "inet.af/netaddr" "nhooyr.io/websocket" "nhooyr.io/websocket/wsjson" + "tailscale.com/tailcfg" "cdr.dev/slog" @@ -302,6 +304,67 @@ func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker return &workspaceAgentNodeBroker{conn}, nil } +func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUID, logger slog.Logger) (agent.Conn, error) { + res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/derpmap", agentID), nil) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nil, readBodyAsError(res) + } + var derpMap tailcfg.DERPMap + err = json.NewDecoder(res.Body).Decode(&derpMap) + if err != nil { + return nil, xerrors.Errorf("decode derpmap: %w", err) + } + ip := tailnet.IP() + + server, err := tailnet.New(&tailnet.Options{ + Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(ip, 128)}, + DERPMap: &derpMap, + Logger: logger, + }) + if err != nil { + return nil, xerrors.Errorf("create tailnet: %w", err) + } + server.SetNodeCallback(func(node *tailnet.Node) { + res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/node", agentID), node) + if err != nil { + logger.Error(ctx, "update node", slog.Error(err), slog.F("node", node)) + return + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + logger.Error(ctx, "update node", slog.F("status_code", res.StatusCode), slog.F("node", node)) + } + }) + workspaceAgent, err := c.WorkspaceAgent(ctx, agentID) + if err != nil { + return nil, xerrors.Errorf("get workspace agent: %w", err) + } + ipRanges := make([]netaddr.IPPrefix, 0, len(workspaceAgent.IPAddresses)) + for _, address := range workspaceAgent.IPAddresses { + ipRanges = append(ipRanges, netaddr.IPPrefixFrom(address, 128)) + } + agentNode := &tailnet.Node{ + Key: workspaceAgent.NodePublicKey, + DiscoKey: workspaceAgent.DiscoPublicKey, + PreferredDERP: workspaceAgent.PreferredDERP, + Addresses: ipRanges, + AllowedIPs: ipRanges, + } + logger.Debug(ctx, "adding agent node", slog.F("node", agentNode)) + err = server.UpdateNodes([]*tailnet.Node{agentNode}) + if err != nil { + return nil, xerrors.Errorf("update nodes: %w", err) + } + return &agent.TailnetConn{ + Target: workspaceAgent.IPAddresses[0], + Server: server, + }, nil +} + // DialWorkspaceAgent creates a connection to the specified resource. func (c *Client) DialWorkspaceAgent(ctx context.Context, agentID uuid.UUID, options *peer.ConnOptions) (agent.Conn, error) { serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/dial", agentID.String())) @@ -447,9 +510,9 @@ type workspaceAgentNodeBroker struct { } func (w *workspaceAgentNodeBroker) Read(ctx context.Context) (*tailnet.Node, error) { - var node *tailnet.Node - err := wsjson.Read(ctx, w.conn, node) - return node, err + var node tailnet.Node + err := wsjson.Read(ctx, w.conn, &node) + return &node, err } func (w *workspaceAgentNodeBroker) Write(ctx context.Context, node *tailnet.Node) error { From 4348b88bae1033023331eced8552b252e41e26bf Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 8 Aug 2022 13:15:35 -0500 Subject: [PATCH 08/54] chore: update gvisor --- go.mod | 34 ++++++++++++++++++---------------- go.sum | 22 ++++++++++++++++------ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 1b5cad9060707..4b9edbd063cdf 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ replace github.com/chzyer/readline => github.com/kylecarbs/readline v0.0.0-20220 replace github.com/briandowns/spinner => github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e // Required until is merged. -replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 // Required until https://github.com/fergusstrange/embedded-postgres/pull/75 is merged. replace github.com/fergusstrange/embedded-postgres => github.com/kylecarbs/embedded-postgres v1.17.1-0.20220615202325-461532cecd3a @@ -39,7 +39,7 @@ replace github.com/golang/glog => github.com/coder/glog v1.0.1-0.20220322161911- // https://github.com/coder/kcp-go/commit/83c0904cec69dcf21ec10c54ea666bda18ada831 replace github.com/fatedier/kcp-go => github.com/coder/kcp-go v2.0.4-0.20220409183554-83c0904cec69+incompatible -replace golang.zx2c4.com/wireguard/tun/netstack => github.com/coder/wireguard-go/tun/netstack v0.0.0-20220614153727-d82b4ba8619f +replace golang.zx2c4.com/wireguard/tun/netstack => github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c require ( cdr.dev/slog v1.4.2-0.20220525200111-18dce5c2cd5f @@ -65,6 +65,7 @@ require ( github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa github.com/gen2brain/beeep v0.0.0-20220402123239-6a3042f4b71a github.com/gliderlabs/ssh v0.3.4 + github.com/go-chi/chi v1.5.4 github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/httprate v0.6.0 github.com/go-chi/render v1.0.1 @@ -74,6 +75,7 @@ require ( github.com/gohugoio/hugo v0.101.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang-migrate/migrate/v4 v4.15.2 + github.com/golang/protobuf v1.5.2 github.com/google/go-github/v43 v43.0.1-0.20220414155304-00e42332e405 github.com/google/uuid v1.3.0 github.com/hashicorp/go-reap v0.0.0-20170704170343-bf58d8a43e7b @@ -86,6 +88,7 @@ require ( github.com/jedib0t/go-pretty/v6 v6.3.5 github.com/justinas/nosurf v1.1.1 github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f + github.com/klauspost/compress v1.15.9 github.com/lib/pq v1.10.6 github.com/mattn/go-isatty v0.0.14 github.com/mitchellh/go-wordwrap v1.0.1 @@ -105,13 +108,18 @@ require ( github.com/prometheus/client_golang v1.12.2 github.com/quasilyte/go-ruleguard/dsl v0.3.21 github.com/robfig/cron/v3 v3.0.1 + github.com/spf13/afero v1.9.2 github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.0 github.com/tabbed/pqtype v0.1.1 github.com/unrolled/secure v1.12.0 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 + go.opentelemetry.io/otel v1.8.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.8.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 + go.opentelemetry.io/otel/sdk v1.8.0 + go.opentelemetry.io/otel/trace v1.8.0 go.uber.org/atomic v1.9.0 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 @@ -132,6 +140,7 @@ require ( google.golang.org/protobuf v1.28.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 + gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444 inet.af/netaddr v0.0.0-20220617031823-097006376321 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 nhooyr.io/websocket v1.8.7 @@ -140,11 +149,7 @@ require ( ) require ( - github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect - gopkg.in/square/go-jose.v2 v2.6.0 // indirect -) - -require ( + filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect @@ -174,9 +179,9 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/elastic/go-windows v1.0.0 // indirect + github.com/fxamacker/cbor/v2 v2.4.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gin-gonic/gin v1.7.0 // indirect - github.com/go-chi/chi v1.5.4 github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -188,16 +193,17 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/insomniacslk/dhcp v0.0.0-20211209223715-7d93572ebe8e // indirect @@ -205,7 +211,6 @@ require ( github.com/josharian/native v1.0.0 // indirect github.com/jsimonetti/rtnetlink v1.1.2-0.20220408201609-d380b505068b // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.9 github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.1 // indirect @@ -250,7 +255,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - github.com/spf13/afero v1.9.2 github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect @@ -264,6 +268,7 @@ require ( github.com/vektah/gqlparser/v2 v2.4.4 // indirect github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 // indirect github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -273,15 +278,12 @@ require ( github.com/zclconf/go-cty v1.10.0 // indirect github.com/zeebo/errs v1.3.0 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/otel v1.8.0 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 go.opentelemetry.io/otel/metric v0.31.0 // indirect - go.opentelemetry.io/otel/sdk v1.8.0 - go.opentelemetry.io/otel/trace v1.8.0 go.opentelemetry.io/proto/otlp v0.18.0 // indirect go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect + go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf // indirect go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect @@ -289,7 +291,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f // indirect google.golang.org/grpc v1.47.0 // indirect + gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gvisor.dev/gvisor v0.0.0-20220407223209-21871174d445 howett.net/plist v1.0.0 // indirect ) diff --git a/go.sum b/go.sum index eb36ba37d21c8..6d3f2da808e56 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= contrib.go.opencensus.io/exporter/stackdriver v0.13.4/go.mod h1:aXENhDJ1Y4lIg4EUaVTwzvYETVNZk10Pu26tevFKLUc= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +filippo.io/edwards25519 v1.0.0-rc.1 h1:m0VOOB23frXZvAOK44usCgLWvtsxIoMCTBGJZlpmGfU= +filippo.io/edwards25519 v1.0.0-rc.1/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= filippo.io/mkcert v1.4.3 h1:axpnmtrZMM8u5Hf4N3UXxboGemMOV+Tn+e+pkHM6E3o= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= @@ -348,10 +350,10 @@ github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1 h1:UqBrPWSYvRI2s5RtOu github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= -github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae h1:YfsqHcNJqKh6raG8tmIlplV4kpUs7uFrKl6prsNC8Ps= -github.com/coder/tailscale v1.1.1-0.20220630165752-6bb3c86a84ae/go.mod h1:8/bTk326TtqBzoJ4xuZPSSGllAZBtDkkVi1GYkciKTY= -github.com/coder/wireguard-go/tun/netstack v0.0.0-20220614153727-d82b4ba8619f h1:wsrm7hB9cvvw8ybX41YjzXDMbpo3gjlesw7oHYhtZW4= -github.com/coder/wireguard-go/tun/netstack v0.0.0-20220614153727-d82b4ba8619f/go.mod h1:PerNzwKlnUUbKSRrSghbyhE9wEl3xakvPY9muprxlv8= +github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 h1:Lv081uuydkOPNFwOSGv8fIWZeMXcIjQI1Wr8aZpLejE= +github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1/go.mod h1:1AAccn2hLv0wT6q/MZ/bPIki+B3csbjq+P+nM/Xm2Oo= +github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c h1:EWRwdW6sGsNFiwDqQpwCgJDt4ilUwT11ZQcWV5WBNH8= +github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c/go.mod h1:TCJ66NtXh3urJotTdoYQOHHkyE899vOQl5TuF+WLSes= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -615,6 +617,8 @@ github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa h1:RDBNVkRviHZtvDvId8XSGPu3rmpmSe+wKRcEWNgsfWU= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/fullstorydev/grpcurl v1.6.0/go.mod h1:ZQ+ayqbKMJNhzLmbpCiurTVlaK2M/3nqZCxaQ2Ze/sM= +github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= +github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/gabriel-vasile/mimetype v1.3.1/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= @@ -1028,6 +1032,8 @@ github.com/hashicorp/terraform-json v0.14.0 h1:sh9iZ1Y8IFJLx+xQiKHGud6/TSUCM0N8e github.com/hashicorp/terraform-json v0.14.0/go.mod h1:5A9HIWPkk4e5aeeXIBbkcOvaZbIYnAIkEyqP2pNSckM= github.com/hashicorp/yamux v0.0.0-20220718163420-dd80a7ee44ce h1:7FO+LmZwiG/eDsBWo50ZeqV5PoH0gwiM1mxFajXAkas= github.com/hashicorp/yamux v0.0.0-20220718163420-dd80a7ee44ce/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 h1:aSVUgRRRtOrZOC1fYmY9gV0e9z/Iu+xNVSASWjsuyGU= +github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3/go.mod h1:5PC6ZNPde8bBqU/ewGZig35+UIZtw9Ytxez8/q5ZyFE= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -1839,6 +1845,8 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -1979,6 +1987,8 @@ go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3m go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/mem v0.0.0-20210711025021-927187094b94 h1:OAAkygi2Js191AJP1Ds42MhJRgeofeKGjuoUqNp1QC4= go4.org/mem v0.0.0-20210711025021-927187094b94/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= +go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf h1:IdwJUzqoIo5lkr2EOyKoe5qipUaEjbOKKY5+fzPBZ3A= +go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf/go.mod h1:+QXzaoURFd0rGDIjDNpyIkv+F9R7EmeKorvlKRnhqgA= go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= @@ -2814,8 +2824,8 @@ gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= -gvisor.dev/gvisor v0.0.0-20220407223209-21871174d445 h1:pLNQCtMzh4O6rdhoUeWHuutt4yMft+B9Cgw/bezWchE= -gvisor.dev/gvisor v0.0.0-20220407223209-21871174d445/go.mod h1:tWwEcFvJavs154OdjFCw78axNrsDlz4Zh8jvPqwcpGI= +gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444 h1:0d3ygmOM5RgQB8rmsZNeAY/7Q98fKt1HrGO2XIp4pDI= +gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444/go.mod h1:TIvkJD0sxe8pIob3p6T8IzxXunlp6yfgktvTNp+DGNM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 47af2b755ff3c72f21a20611b2702fd606d1464a Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 8 Aug 2022 16:01:20 -0500 Subject: [PATCH 09/54] fix tailscale types --- agent/agent.go | 10 ++++----- agent/conn.go | 8 +++---- cli/ssh.go | 26 ++++++++++++---------- cli/wireguardtunnel.go | 9 ++++---- coderd/coderd_test.go | 8 +++---- coderd/database/queries.sql.go | 12 ++++++++++ coderd/workspaceagents.go | 20 +++-------------- codersdk/workspaceagents.go | 10 ++++----- codersdk/workspaceresources.go | 4 ++-- go.mod | 5 +---- go.sum | 6 ----- peer/peerwg/derp.go | 3 +-- peer/peerwg/handshake.go | 6 ++--- peer/peerwg/ssh.go | 8 +++---- peer/peerwg/wireguard.go | 40 +++++++++++++++++----------------- peer/peerwg/wireguard_test.go | 14 ++++++------ tailnet/tailnet.go | 39 +++++++++++++++++---------------- tailnet/tailnet_test.go | 11 +++++----- 18 files changed, 115 insertions(+), 124 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index e4e27c923e06e..160c87ee48a4f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net" + "net/netip" "net/url" "os" "os/exec" @@ -27,7 +28,6 @@ import ( "go.uber.org/atomic" gossh "golang.org/x/crypto/ssh" "golang.org/x/xerrors" - "inet.af/netaddr" "tailscale.com/tailcfg" "cdr.dev/slog" @@ -62,7 +62,7 @@ type Options struct { } type Metadata struct { - IPAddresses []netaddr.IP `json:"ip_addresses"` + IPAddresses []netip.Addr `json:"ip_addresses"` DERPMap *tailcfg.DERPMap `json:"derpmap"` EnvironmentVariables map[string]string `json:"environment_variables"` StartupScript string `json:"startup_script"` @@ -174,10 +174,10 @@ func (a *agent) run(ctx context.Context) { } } -func (a *agent) runTailnet(ctx context.Context, addresses []netaddr.IP, derpMap *tailcfg.DERPMap) { - ipRanges := make([]netaddr.IPPrefix, 0, len(addresses)) +func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap *tailcfg.DERPMap) { + ipRanges := make([]netip.Prefix, 0, len(addresses)) for _, address := range addresses { - ipRanges = append(ipRanges, netaddr.IPPrefixFrom(address, 128)) + ipRanges = append(ipRanges, netip.PrefixFrom(address, 128)) } var err error a.network, err = tailnet.New(&tailnet.Options{ diff --git a/agent/conn.go b/agent/conn.go index 555854d5dd413..55f09a33c247b 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "net/netip" "net/url" "strconv" "strings" @@ -13,7 +14,6 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/xerrors" - "inet.af/netaddr" "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker/proto" @@ -135,7 +135,7 @@ func (c *WebRTCConn) Close() error { } type TailnetConn struct { - Target netaddr.IP + Target netip.Addr *tailnet.Server } @@ -156,7 +156,7 @@ func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command s } func (c *TailnetConn) SSH() (net.Conn, error) { - return c.DialContextTCP(context.Background(), netaddr.IPPortFrom(c.Target, 12212)) + return c.DialContextTCP(context.Background(), netip.AddrPortFrom(c.Target, 12212)) } // SSHClient calls SSH to create a client that uses a weak cipher @@ -181,5 +181,5 @@ func (c *TailnetConn) SSHClient() (*ssh.Client, error) { func (c *TailnetConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { _, rawPort, _ := net.SplitHostPort(addr) port, _ := strconv.Atoi(rawPort) - return c.Server.DialContextTCP(ctx, netaddr.IPPortFrom(c.Target, uint16(port))) + return c.Server.DialContextTCP(ctx, netip.AddrPortFrom(c.Target, uint16(port))) } diff --git a/cli/ssh.go b/cli/ssh.go index 6e53c8bd65d60..89cf3cbfcbf76 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -19,6 +19,7 @@ import ( gosshagent "golang.org/x/crypto/ssh/agent" "golang.org/x/term" "golang.org/x/xerrors" + tslogger "tailscale.com/types/logger" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" @@ -26,6 +27,7 @@ import ( "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" + "github.com/coder/coder/peer/peerwg" ) var workspacePollInterval = time.Minute @@ -112,12 +114,12 @@ func ssh() *cobra.Command { newSSHClient = conn.SSHClient } else { // TODO: more granual control of Tailscale logging. - // peerwg.Logf = tslogger.Discard + peerwg.Logf = tslogger.Discard // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) // wgn, err := peerwg.New( // slog.Make(sloghuman.Sink(os.Stderr)), - // []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, + // []netip.Prefix{netip.PrefixFrom(ipv6, 128)}, // ) // if err != nil { // return xerrors.Errorf("create wireguard network: %w", err) @@ -136,18 +138,19 @@ func ssh() *cobra.Command { // err = wgn.AddPeer(peerwg.Handshake{ // Recipient: workspaceAgent.ID, // DiscoPublicKey: workspaceAgent.DiscoPublicKey, - // NodePublicKey: workspaceAgent.WireguardPublicKey, - // IPv6: workspaceAgent.IPv6.IP(), + // NodePublicKey: workspaceAgent.NodePublicKey, + // IPv6: workspaceAgent.IPAddresses[0], // TODO: fix? // }) // if err != nil { // return xerrors.Errorf("add workspace agent as peer: %w", err) // } // if stdio { - // rawSSH, err := wgn.SSH(cmd.Context(), workspaceAgent.IPv6.IP()) + // rawSSH, err := wgn.SSH(cmd.Context(), workspaceAgent.IPAddresses[0]) // TODO: fix? // if err != nil { // return err // } + // defer rawSSH.Close() // go func() { // _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) @@ -156,15 +159,14 @@ func ssh() *cobra.Command { // return nil // } - // sshClient, err = wgn.SSHClient(cmd.Context(), workspaceAgent.IPv6.IP()) - // if err != nil { - // return err + // newSSHClient = func() (*gossh.Client, error) { + // return wgn.SSHClient(ctx, workspaceAgent.IPAddresses[0]) // TODO: fix? // } + } - // sshSession, err = sshClient.NewSession() - // if err != nil { - // return err - // } + sshClient, err := newSSHClient() + if err != nil { + return err } defer sshClient.Close() diff --git a/cli/wireguardtunnel.go b/cli/wireguardtunnel.go index ab074b5782ae4..4f2cdd1a08a42 100644 --- a/cli/wireguardtunnel.go +++ b/cli/wireguardtunnel.go @@ -4,13 +4,13 @@ import ( "context" "fmt" "net" + "net/netip" "strconv" "sync" "github.com/pion/udp" "github.com/spf13/cobra" "golang.org/x/xerrors" - "inet.af/netaddr" coderagent "github.com/coder/coder/agent" "github.com/coder/coder/cli/cliui" @@ -93,7 +93,7 @@ func wireguardPortForward() *cobra.Command { // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) // wgn, err := peerwg.New( // slog.Make(sloghuman.Sink(os.Stderr)), - // []netaddr.IPPrefix{netaddr.IPPrefixFrom(ipv6, 128)}, + // []netip.Prefix{netip.PrefixFrom(ipv6, 128)}, // ) // if err != nil { // return xerrors.Errorf("create wireguard network: %w", err) @@ -180,7 +180,7 @@ func listenAndPortForwardWireguard(ctx context.Context, cmd *cobra.Command, wgn *peerwg.Network, wg *sync.WaitGroup, spec portForwardSpec, - agentIP netaddr.IP, + agentIP netip.Addr, ) (net.Listener, error) { _, _ = fmt.Fprintf(cmd.OutOrStderr(), "Forwarding '%v://%v' locally to '%v://%v' in the workspace\n", spec.listenNetwork, spec.listenAddress, spec.dialNetwork, spec.dialAddress) @@ -231,7 +231,8 @@ func listenAndPortForwardWireguard(ctx context.Context, cmd *cobra.Command, go func(netConn net.Conn) { defer netConn.Close() - ipPort := netaddr.MustParseIPPort(spec.dialAddress).WithIP(agentIP) + port := netip.MustParseAddrPort(spec.dialAddress) + ipPort := netip.AddrPortFrom(agentIP, port.Port()) var remoteConn net.Conn switch spec.dialNetwork { diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index 19cb3f4ff8cf9..0c811730543e4 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/http/httptest" + "net/netip" "net/url" "os" "strconv" @@ -22,7 +23,6 @@ import ( "golang.org/x/xerrors" "google.golang.org/api/idtoken" "google.golang.org/api/option" - "inet.af/netaddr" "tailscale.com/tailcfg" "cdr.dev/slog" @@ -610,14 +610,14 @@ func TestDERP(t *testing.T) { } w1IP := tailnet.IP() w1, err := tailnet.New(&tailnet.Options{ - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(w1IP, 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, Logger: logger.Named("w1"), DERPMap: derpMap, }) require.NoError(t, err) w2, err := tailnet.New(&tailnet.Options{ - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(tailnet.IP(), 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, Logger: logger.Named("w2"), DERPMap: derpMap, }) @@ -642,7 +642,7 @@ func TestDERP(t *testing.T) { }() <-conn - nc, err := w2.DialContextTCP(context.Background(), netaddr.IPPortFrom(w1IP, 35565)) + nc, err := w2.DialContextTCP(context.Background(), netip.AddrPortFrom(w1IP, 35565)) require.NoError(t, err) _ = nc.Close() <-conn diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 963dadd4ec8b2..fe88b127065a5 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -4,6 +4,18 @@ package database +import ( + "context" + "database/sql" + "encoding/json" + "time" + + "github.com/coder/coder/coderd/database/dbtypes" + "github.com/google/uuid" + "github.com/lib/pq" + "github.com/tabbed/pqtype" +) + const deleteAPIKeyByID = `-- name: DeleteAPIKeyByID :exec DELETE FROM diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 717646de925c0..2e3175cc677c2 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -9,14 +9,13 @@ import ( "io" "net" "net/http" + "net/netip" "strconv" "time" "github.com/google/uuid" "github.com/hashicorp/yamux" - "github.com/tabbed/pqtype" "golang.org/x/xerrors" - "inet.af/netaddr" "nhooyr.io/websocket" "nhooyr.io/websocket/wsjson" @@ -614,19 +613,6 @@ func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { return apps } -func inetToNetaddr(inet pqtype.Inet) netaddr.IPPrefix { - if !inet.Valid { - return netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 128) - } - - ipp, ok := netaddr.FromStdIPNet(&inet.IPNet) - if !ok { - return netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 128) - } - - return ipp -} - func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { var envs map[string]string if dbAgent.EnvironmentVariables.Valid { @@ -635,11 +621,11 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work return codersdk.WorkspaceAgent{}, xerrors.Errorf("unmarshal: %w", err) } } - ips := make([]netaddr.IP, 0) + ips := make([]netip.Addr, 0) for _, ip := range dbAgent.IPAddresses { var ipData [16]byte copy(ipData[:], []byte(ip.IPNet.IP)) - ips = append(ips, netaddr.IPFrom16(ipData)) + ips = append(ips, netip.AddrFrom16(ipData)) } workspaceAgent := codersdk.WorkspaceAgent{ ID: dbAgent.ID, diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 80522b6b66386..c6a67468642db 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "net/http/cookiejar" + "net/netip" "cloud.google.com/go/compute/metadata" "github.com/google/uuid" @@ -15,7 +16,6 @@ import ( "github.com/pion/webrtc/v3" "golang.org/x/net/proxy" "golang.org/x/xerrors" - "inet.af/netaddr" "nhooyr.io/websocket" "nhooyr.io/websocket/wsjson" "tailscale.com/tailcfg" @@ -317,10 +317,10 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUI if err != nil { return nil, xerrors.Errorf("decode derpmap: %w", err) } - ip := tailnet.IP() + ip := tailnet.IP() server, err := tailnet.New(&tailnet.Options{ - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(ip, 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, DERPMap: &derpMap, Logger: logger, }) @@ -342,9 +342,9 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUI if err != nil { return nil, xerrors.Errorf("get workspace agent: %w", err) } - ipRanges := make([]netaddr.IPPrefix, 0, len(workspaceAgent.IPAddresses)) + ipRanges := make([]netip.Prefix, 0, len(workspaceAgent.IPAddresses)) for _, address := range workspaceAgent.IPAddresses { - ipRanges = append(ipRanges, netaddr.IPPrefixFrom(address, 128)) + ipRanges = append(ipRanges, netip.PrefixFrom(address, 128)) } agentNode := &tailnet.Node{ Key: workspaceAgent.NodePublicKey, diff --git a/codersdk/workspaceresources.go b/codersdk/workspaceresources.go index 5ae52ec01b4d8..d5107e2f53bf4 100644 --- a/codersdk/workspaceresources.go +++ b/codersdk/workspaceresources.go @@ -5,10 +5,10 @@ import ( "encoding/json" "fmt" "net/http" + "net/netip" "time" "github.com/google/uuid" - "inet.af/netaddr" "tailscale.com/types/key" ) @@ -56,7 +56,7 @@ type WorkspaceAgent struct { Apps []WorkspaceApp `json:"apps"` // For internal routing only. - IPAddresses []netaddr.IP `json:"ip_addresses"` + IPAddresses []netip.Addr `json:"ip_addresses"` NodePublicKey key.NodePublic `json:"node_public_key"` DiscoPublicKey key.DiscoPublic `json:"disco_public_key"` // PreferredDERP represents the connected region. diff --git a/go.mod b/go.mod index 4b9edbd063cdf..1e27be9792ef1 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,6 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 gvisor.dev/gvisor v0.0.0-20220801230058-850e42eb4444 - inet.af/netaddr v0.0.0-20220617031823-097006376321 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 nhooyr.io/websocket v1.8.7 storj.io/drpc v0.0.33-0.20220622181519-9206537a4db7 @@ -281,10 +280,8 @@ require ( go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.8.0 // indirect go.opentelemetry.io/otel/metric v0.31.0 // indirect go.opentelemetry.io/proto/otlp v0.18.0 // indirect - go4.org/intern v0.0.0-20211027215823-ae77deb06f29 // indirect go4.org/mem v0.0.0-20210711025021-927187094b94 // indirect - go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf // indirect - go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect + go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect golang.zx2c4.com/wintun v0.0.0-20211104114900-415007cec224 // indirect golang.zx2c4.com/wireguard/windows v0.4.10 // indirect diff --git a/go.sum b/go.sum index 6d3f2da808e56..5e5f9c53985f8 100644 --- a/go.sum +++ b/go.sum @@ -556,7 +556,6 @@ github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-sysinfo v1.8.1 h1:4Yhj+HdV6WjbCRgGdZpPJ8lZQlXZLKDAeIkmQ/VRvi4= github.com/elastic/go-sysinfo v1.8.1/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM= @@ -1984,14 +1983,11 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go4.org/intern v0.0.0-20211027215823-ae77deb06f29 h1:UXLjNohABv4S58tHmeuIZDO6e3mHpW2Dx33gaNt03LE= -go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA= go4.org/mem v0.0.0-20210711025021-927187094b94 h1:OAAkygi2Js191AJP1Ds42MhJRgeofeKGjuoUqNp1QC4= go4.org/mem v0.0.0-20210711025021-927187094b94/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf h1:IdwJUzqoIo5lkr2EOyKoe5qipUaEjbOKKY5+fzPBZ3A= go4.org/netipx v0.0.0-20220725152314-7e7bdc8411bf/go.mod h1:+QXzaoURFd0rGDIjDNpyIkv+F9R7EmeKorvlKRnhqgA= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 h1:FyBZqvoA/jbNzuAWLQE2kG820zMAkcilx6BMjGbL/E4= -go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180501155221-613d6eafa307/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2837,8 +2833,6 @@ honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.4.0-0.dev.0.20220404092545-59d7a2877f83 h1:lZ9GIYaU+o5+X6ST702I/Ntyq9Y2oIMZ42rBQpem64A= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -inet.af/netaddr v0.0.0-20220617031823-097006376321 h1:B4dC8ySKTQXasnjDTMsoCMf1sQG4WsMej0WXaHxunmU= -inet.af/netaddr v0.0.0-20220617031823-097006376321/go.mod h1:OIezDfdzOgFhuw4HuWapWq2e9l0H9tK4F1j+ETRtF3k= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= diff --git a/peer/peerwg/derp.go b/peer/peerwg/derp.go index afbbcbc5e9704..77aecef6f5e43 100644 --- a/peer/peerwg/derp.go +++ b/peer/peerwg/derp.go @@ -4,7 +4,6 @@ import ( "net" "tailscale.com/tailcfg" - "tailscale.com/wgengine/magicsock" ) // This is currently set to use Tailscale's DERP server in DFW while we build in @@ -71,4 +70,4 @@ var DerpMap = &tailcfg.DERPMap{ // DefaultDerpHome is the ipv4 representation of a DERP server. The port is the // DERP id. We only support using DERP 9 for now. -var DefaultDerpHome = net.JoinHostPort(magicsock.DerpMagicIP, "1") +var DefaultDerpHome = net.JoinHostPort(tailcfg.DerpMagicIP, "1") diff --git a/peer/peerwg/handshake.go b/peer/peerwg/handshake.go index 08fdc234052ad..56340648ae9a4 100644 --- a/peer/peerwg/handshake.go +++ b/peer/peerwg/handshake.go @@ -2,11 +2,11 @@ package peerwg import ( "bytes" + "net/netip" "strconv" "github.com/google/uuid" "golang.org/x/xerrors" - "inet.af/netaddr" "tailscale.com/types/key" ) @@ -22,7 +22,7 @@ type Handshake struct { // NodePublicKey is the public key of the peer. NodePublicKey key.NodePublic `json:"public"` // IPv6 is the IPv6 address of the peer. - IPv6 netaddr.IP `json:"ipv6"` + IPv6 netip.Addr `json:"ipv6"` } // HandshakeRecipientHint parses the first part of a serialized @@ -58,7 +58,7 @@ func (h *Handshake) UnmarshalText(text []byte) error { return xerrors.Errorf("parse public: %w", err) } - h.IPv6, err = netaddr.ParseIP(string(sp[3])) + h.IPv6, err = netip.ParseAddr(string(sp[3])) if err != nil { return xerrors.Errorf("parse ipv6: %w", err) } diff --git a/peer/peerwg/ssh.go b/peer/peerwg/ssh.go index 9ffe8cc92c816..b94545c1e831e 100644 --- a/peer/peerwg/ssh.go +++ b/peer/peerwg/ssh.go @@ -3,14 +3,14 @@ package peerwg import ( "context" "net" + "net/netip" "golang.org/x/crypto/ssh" "golang.org/x/xerrors" - "inet.af/netaddr" ) -func (n *Network) SSH(ctx context.Context, ip netaddr.IP) (net.Conn, error) { - netConn, err := n.Netstack.DialContextTCP(ctx, netaddr.IPPortFrom(ip, 12212)) +func (n *Network) SSH(ctx context.Context, ip netip.Addr) (net.Conn, error) { + netConn, err := n.Netstack.DialContextTCP(ctx, netip.AddrPortFrom(ip, 12212)) if err != nil { return nil, xerrors.Errorf("dial agent ssh: %w", err) } @@ -18,7 +18,7 @@ func (n *Network) SSH(ctx context.Context, ip netaddr.IP) (net.Conn, error) { return netConn, nil } -func (n *Network) SSHClient(ctx context.Context, ip netaddr.IP) (*ssh.Client, error) { +func (n *Network) SSHClient(ctx context.Context, ip netip.Addr) (*ssh.Client, error) { netConn, err := n.SSH(ctx, ip) if err != nil { return nil, xerrors.Errorf("ssh: %w", err) diff --git a/peer/peerwg/wireguard.go b/peer/peerwg/wireguard.go index b29db41bd1cd3..6eeaa06a150b8 100644 --- a/peer/peerwg/wireguard.go +++ b/peer/peerwg/wireguard.go @@ -7,14 +7,15 @@ import ( "io" "log" "net" + "net/netip" "strconv" "sync" "time" "github.com/google/uuid" "github.com/tabbed/pqtype" + "go4.org/netipx" "golang.org/x/xerrors" - "inet.af/netaddr" "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" "tailscale.com/net/netns" @@ -55,8 +56,8 @@ func UUIDToInet(uid uuid.UUID) pqtype.Inet { } } -func UUIDToNetaddr(uid uuid.UUID) netaddr.IP { - return netaddr.IPFrom16(uuid.New()) +func UUIDToNetaddr(uid uuid.UUID) netip.Addr { + return netip.AddrFrom16(uuid.New()) } // privateUUID sets the uid to have the tailscale private ipv6 prefix. @@ -91,7 +92,7 @@ type Network struct { // New constructs a Wireguard network that filters traffic // to destinations matching the addresses provided. -func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { +func New(logger slog.Logger, addresses []netip.Prefix) (*Network, error) { nodePrivateKey := key.NewNode() nodePublicKey := nodePrivateKey.Public() // id, stableID := nodeIDs(nodePublicKey) @@ -104,21 +105,21 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { // Allow any protocol! IPProto: []ipproto.Proto{ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, ipproto.SCTP}, // Allow traffic sourced from anywhere. - Srcs: []netaddr.IPPrefix{ - netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), - netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + Srcs: []netip.Prefix{ + netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), + netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), }, // Allow traffic to route anywhere. Dsts: []filter.NetPortRange{ { - Net: netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), + Net: netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), Ports: filter.PortRange{ First: 0, Last: 65535, }, }, { - Net: netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + Net: netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), Ports: filter.PortRange{ First: 0, Last: 65535, @@ -149,7 +150,7 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { if err != nil { return nil, xerrors.Errorf("create wgengine: %w", err) } - dialer.UseNetstackForIP = func(ip netaddr.IP) bool { + dialer.UseNetstackForIP = func(ip netip.Addr) bool { _, ok := engine.PeerForIP(ip) return ok } @@ -175,7 +176,7 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { if err != nil { return nil, xerrors.Errorf("create netstack: %w", err) } - dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) { + dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { return netStack.DialContextTCP(ctx, dst) } netStack.ProcessLocalIPs = true @@ -204,14 +205,13 @@ func New(logger slog.Logger, addresses []netaddr.IPPrefix) (*Network, error) { netMapCopy.SelfNode = &tailcfg.Node{} engine.SetNetworkMap(&netMapCopy) - ipb := netaddr.IPSetBuilder{} + localIPSet := netipx.IPSetBuilder{} for _, addr := range netMap.Addresses { - ipb.AddPrefix(addr) + localIPSet.AddPrefix(addr) } - ips, _ := ipb.IPSet() - - iplb := netaddr.IPSetBuilder{} - ipl, _ := iplb.IPSet() + ips, _ := localIPSet.IPSet() + logIPSet := netipx.IPSetBuilder{} + ipl, _ := logIPSet.IPSet() engine.SetFilter(filter.New(netMap.PacketFilter, ips, ipl, nil, Logf)) wn := &Network{ @@ -307,8 +307,8 @@ func (n *Network) AddPeer(handshake Handshake) error { // Name: handshake.NodePublicKey.String() + ".com", Key: handshake.NodePublicKey, DiscoKey: handshake.DiscoPublicKey, - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(handshake.IPv6, 128)}, - AllowedIPs: []netaddr.IPPrefix{netaddr.IPPrefixFrom(handshake.IPv6, 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(handshake.IPv6, 128)}, + AllowedIPs: []netip.Prefix{netip.PrefixFrom(handshake.IPv6, 128)}, DERP: DefaultDerpHome, // Endpoints: []string{DefaultDerpHome}, }) @@ -332,7 +332,7 @@ func (n *Network) AddPeer(handshake Handshake) error { // Ping sends a discovery ping to the provided peer. // The peer address must be connected before a successful ping will work. -func (n *Network) Ping(ip netaddr.IP) *ipnstate.PingResult { +func (n *Network) Ping(ip netip.Addr) *ipnstate.PingResult { ch := make(chan *ipnstate.PingResult) n.wgEngine.Ping(ip, tailcfg.PingDisco, func(pr *ipnstate.PingResult) { ch <- pr diff --git a/peer/peerwg/wireguard_test.go b/peer/peerwg/wireguard_test.go index cc801bbde33cb..a08fcf17dd396 100644 --- a/peer/peerwg/wireguard_test.go +++ b/peer/peerwg/wireguard_test.go @@ -7,12 +7,12 @@ import ( "net" "net/http" "net/http/httptest" + "net/netip" "testing" "time" "github.com/google/uuid" "github.com/stretchr/testify/require" - "inet.af/netaddr" "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/net/stun/stuntest" @@ -30,14 +30,14 @@ func TestConnect(t *testing.T) { logger := slogtest.Make(t, nil) c1IPv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn1, err := peerwg.New(logger.Named("c1"), []netaddr.IPPrefix{ - netaddr.IPPrefixFrom(c1IPv6, 128), + wgn1, err := peerwg.New(logger.Named("c1"), []netip.Prefix{ + netip.PrefixFrom(c1IPv6, 128), }) require.NoError(t, err) c2IPv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn2, err := peerwg.New(logger.Named("c2"), []netaddr.IPPrefix{ - netaddr.IPPrefixFrom(c2IPv6, 128), + wgn2, err := peerwg.New(logger.Named("c2"), []netip.Prefix{ + netip.PrefixFrom(c2IPv6, 128), }) require.NoError(t, err) err = wgn1.AddPeer(peerwg.Handshake{ @@ -68,12 +68,12 @@ func TestConnect(t *testing.T) { <-conn time.Sleep(100 * time.Millisecond) fmt.Printf("\n\n\n\n\nDIALING TCP\n\n\n\n\n") - _, err = wgn2.Netstack.DialContextTCP(context.Background(), netaddr.IPPortFrom(c1IPv6, 35565)) + _, err = wgn2.Netstack.DialContextTCP(context.Background(), netip.AddrPortFrom(c1IPv6, 35565)) require.NoError(t, err) <-conn } -func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, stunIP netaddr.IP) (derpMap *tailcfg.DERPMap, cleanup func()) { +func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, stunIP netip.Addr) (derpMap *tailcfg.DERPMap, cleanup func()) { d := derp.NewServer(key.NewNode(), logf) httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d)) diff --git a/tailnet/tailnet.go b/tailnet/tailnet.go index 386080a5291ca..412fef841071d 100644 --- a/tailnet/tailnet.go +++ b/tailnet/tailnet.go @@ -4,13 +4,14 @@ import ( "context" "fmt" "net" + "net/netip" "sync" "time" "github.com/google/uuid" + "go4.org/netipx" "golang.org/x/xerrors" "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" - "inet.af/netaddr" "tailscale.com/hostinfo" "tailscale.com/ipn/ipnstate" "tailscale.com/net/dns" @@ -42,7 +43,7 @@ func init() { } type Options struct { - Addresses []netaddr.IPPrefix + Addresses []netip.Prefix DERPMap *tailcfg.DERPMap Logger slog.Logger @@ -70,21 +71,21 @@ func New(options *Options) (*Server, error) { // Allow any protocol! IPProto: []ipproto.Proto{ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, ipproto.SCTP}, // Allow traffic sourced from anywhere. - Srcs: []netaddr.IPPrefix{ - netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), - netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + Srcs: []netip.Prefix{ + netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), + netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), }, // Allow traffic to route anywhere. Dsts: []filter.NetPortRange{ { - Net: netaddr.IPPrefixFrom(netaddr.IPv4(0, 0, 0, 0), 0), + Net: netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), Ports: filter.PortRange{ First: 0, Last: 65535, }, }, { - Net: netaddr.IPPrefixFrom(netaddr.IPv6Unspecified(), 0), + Net: netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), Ports: filter.PortRange{ First: 0, Last: 65535, @@ -121,7 +122,7 @@ func New(options *Options) (*Server, error) { if err != nil { return nil, xerrors.Errorf("create wgengine: %w", err) } - dialer.UseNetstackForIP = func(ip netaddr.IP) bool { + dialer.UseNetstackForIP = func(ip netip.Addr) bool { _, ok := wireguardEngine.PeerForIP(ip) return ok } @@ -149,7 +150,7 @@ func New(options *Options) (*Server, error) { if err != nil { return nil, xerrors.Errorf("create netstack: %w", err) } - dialer.NetstackDialTCP = func(ctx context.Context, dst netaddr.IPPort) (net.Conn, error) { + dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { return netStack.DialContextTCP(ctx, dst) } netStack.ProcessLocalIPs = true @@ -177,12 +178,12 @@ func New(options *Options) (*Server, error) { netMapCopy := *netMap wireguardEngine.SetNetworkMap(&netMapCopy) - localIPSet := netaddr.IPSetBuilder{} + localIPSet := netipx.IPSetBuilder{} for _, addr := range netMap.Addresses { localIPSet.AddPrefix(addr) } localIPs, _ := localIPSet.IPSet() - logIPSet := netaddr.IPSetBuilder{} + logIPSet := netipx.IPSetBuilder{} logIPs, _ := logIPSet.IPSet() wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter")))) server := &Server{ @@ -202,7 +203,7 @@ func New(options *Options) (*Server, error) { } // IP generates a new IP with a static service prefix. -func IP() netaddr.IP { +func IP() netip.Addr { // This is Tailscale's ephemeral service prefix. // This can be changed easily later-on, because // all of our nodes are ephemeral. @@ -214,7 +215,7 @@ func IP() netaddr.IP { uid[3] = 0x5c uid[4] = 0xa1 uid[5] = 0xe0 - return netaddr.IPFrom16(uid) + return netip.AddrFrom16(uid) } // Server is an actively listening Wireguard connection. @@ -267,7 +268,7 @@ func (s *Server) UpdateNodes(nodes []*Node) error { DiscoKey: node.DiscoKey, Addresses: node.Addresses, AllowedIPs: node.AllowedIPs, - DERP: fmt.Sprintf("%s:%d", magicsock.DerpMagicIP, node.PreferredDERP), + DERP: fmt.Sprintf("%s:%d", tailcfg.DerpMagicIP, node.PreferredDERP), Hostinfo: hostinfo.New().View(), } } @@ -289,7 +290,7 @@ func (s *Server) UpdateNodes(nodes []*Node) error { } // Ping sends a ping to the Wireguard engine. -func (s *Server) Ping(ip netaddr.IP, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { +func (s *Server) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { s.wireguardEngine.Ping(ip, pingType, cb) } @@ -316,8 +317,8 @@ type Node struct { DiscoKey key.DiscoPublic `json:"disco"` PreferredDERP int `json:"preferred_derp"` DERPLatency map[string]float64 `json:"derp_latency"` - Addresses []netaddr.IPPrefix `json:"addresses"` - AllowedIPs []netaddr.IPPrefix `json:"allowed_ips"` + Addresses []netip.Prefix `json:"addresses"` + AllowedIPs []netip.Prefix `json:"allowed_ips"` } // This and below is taken _mostly_ verbatim from Tailscale: @@ -351,11 +352,11 @@ func (s *Server) Listen(network, addr string) (net.Listener, error) { return ln, nil } -func (s *Server) DialContextTCP(ctx context.Context, ipp netaddr.IPPort) (*gonet.TCPConn, error) { +func (s *Server) DialContextTCP(ctx context.Context, ipp netip.AddrPort) (*gonet.TCPConn, error) { return s.netStack.DialContextTCP(ctx, ipp) } -func (s *Server) DialContextUDP(ctx context.Context, ipp netaddr.IPPort) (*gonet.UDPConn, error) { +func (s *Server) DialContextUDP(ctx context.Context, ipp netip.AddrPort) (*gonet.UDPConn, error) { return s.netStack.DialContextUDP(ctx, ipp) } diff --git a/tailnet/tailnet_test.go b/tailnet/tailnet_test.go index 9536867bc7b0b..6ac27ba53096b 100644 --- a/tailnet/tailnet_test.go +++ b/tailnet/tailnet_test.go @@ -6,22 +6,21 @@ import ( "net" "net/http" "net/http/httptest" + "net/netip" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" - "inet.af/netaddr" "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/tailcfg" "tailscale.com/types/key" tslogger "tailscale.com/types/logger" - "github.com/coder/coder/tailnet" - "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/tailnet" ) func TestMain(m *testing.M) { @@ -35,14 +34,14 @@ func TestTailnet(t *testing.T) { w1IP := tailnet.IP() w1, err := tailnet.New(&tailnet.Options{ - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(w1IP, 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, Logger: logger.Named("w1"), DERPMap: derpMap, }) require.NoError(t, err) w2, err := tailnet.New(&tailnet.Options{ - Addresses: []netaddr.IPPrefix{netaddr.IPPrefixFrom(tailnet.IP(), 128)}, + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, Logger: logger.Named("w2"), DERPMap: derpMap, }) @@ -69,7 +68,7 @@ func TestTailnet(t *testing.T) { conn <- struct{}{} }() - nc, err := w2.DialContextTCP(context.Background(), netaddr.IPPortFrom(w1IP, 35565)) + nc, err := w2.DialContextTCP(context.Background(), netip.AddrPortFrom(w1IP, 35565)) require.NoError(t, err) _ = nc.Close() <-conn From 4f34245c217c365a5f90c161e154411cf3b43ed6 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 8 Aug 2022 16:04:26 -0500 Subject: [PATCH 10/54] linting --- cli/ssh.go | 4 +--- site/src/api/typesGenerated.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/ssh.go b/cli/ssh.go index 89cf3cbfcbf76..bf4e0c19b52f2 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -19,7 +19,6 @@ import ( gosshagent "golang.org/x/crypto/ssh/agent" "golang.org/x/term" "golang.org/x/xerrors" - tslogger "tailscale.com/types/logger" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" @@ -27,7 +26,6 @@ import ( "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" - "github.com/coder/coder/peer/peerwg" ) var workspacePollInterval = time.Minute @@ -114,7 +112,7 @@ func ssh() *cobra.Command { newSSHClient = conn.SSHClient } else { // TODO: more granual control of Tailscale logging. - peerwg.Logf = tslogger.Discard + // peerwg.Logf = tslogger.Discard // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) // wgn, err := peerwg.New( diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index fbb1521f1be3c..1c50f8a3b6d22 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -427,7 +427,7 @@ export interface WorkspaceAgent { readonly startup_script?: string readonly directory?: string readonly apps: WorkspaceApp[] - // Named type "inet.af/netaddr.IP" unknown, using "any" + // Named type "net/netip.Addr" unknown, using "any" // eslint-disable-next-line @typescript-eslint/no-explicit-any readonly ip_addresses: any[] // Named type "tailscale.com/types/key.NodePublic" unknown, using "any" From d36f61d61c6dc0e944cba828aafc4f254dc8bb01 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Mon, 8 Aug 2022 16:23:15 -0500 Subject: [PATCH 11/54] more linting --- cli/ssh.go | 4 +++- cli/wireguardtunnel.go | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/ssh.go b/cli/ssh.go index bf4e0c19b52f2..29a421c69301b 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -19,6 +19,7 @@ import ( gosshagent "golang.org/x/crypto/ssh/agent" "golang.org/x/term" "golang.org/x/xerrors" + tslogger "tailscale.com/types/logger" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" @@ -26,6 +27,7 @@ import ( "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" + "github.com/coder/coder/peer/peerwg" ) var workspacePollInterval = time.Minute @@ -112,7 +114,7 @@ func ssh() *cobra.Command { newSSHClient = conn.SSHClient } else { // TODO: more granual control of Tailscale logging. - // peerwg.Logf = tslogger.Discard + peerwg.Logf = tslogger.Discard //nolint // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) // wgn, err := peerwg.New( diff --git a/cli/wireguardtunnel.go b/cli/wireguardtunnel.go index 4f2cdd1a08a42..31d2c46913d00 100644 --- a/cli/wireguardtunnel.go +++ b/cli/wireguardtunnel.go @@ -176,6 +176,7 @@ func wireguardPortForward() *cobra.Command { return cmd } +//nolint:unused,deadcode func listenAndPortForwardWireguard(ctx context.Context, cmd *cobra.Command, wgn *peerwg.Network, wg *sync.WaitGroup, From 01c52c7cfa4cfc3676dd2fcd11b151b99099d809 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 11 Aug 2022 23:26:47 -0500 Subject: [PATCH 12/54] Add coordinator --- .vscode/settings.json | 3 + agent/agent.go | 4 +- agent/conn.go | 2 +- coderd/coderd_test.go | 4 +- codersdk/workspaceagents.go | 2 +- tailnet/{tailnet.go => conn.go} | 121 +++++++-------- tailnet/{tailnet_test.go => conn_test.go} | 4 +- tailnet/coordinator.go | 175 ++++++++++++++++++++++ tailnet/coordinator_test.go | 96 ++++++++++++ 9 files changed, 338 insertions(+), 73 deletions(-) rename tailnet/{tailnet.go => conn.go} (78%) rename tailnet/{tailnet_test.go => conn_test.go} (96%) create mode 100644 tailnet/coordinator.go create mode 100644 tailnet/coordinator_test.go diff --git a/.vscode/settings.json b/.vscode/settings.json index eb597cfcbadee..aff3b6e60817f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -81,7 +81,9 @@ "Srcs", "stretchr", "stuntest", + "tailbroker", "tailcfg", + "tailexchange", "tailnet", "Tailscale", "TCGETS", @@ -118,6 +120,7 @@ "workspacebuilds", "workspacename", "wsconncache", + "wsjson", "xerrors", "xstate", "yamux" diff --git a/agent/agent.go b/agent/agent.go index 160c87ee48a4f..299c0ea07961a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -126,7 +126,7 @@ type agent struct { sshServer *ssh.Server enableTailnet bool - network *tailnet.Server + network *tailnet.Conn nodeDialer NodeDialer } @@ -180,7 +180,7 @@ func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap ipRanges = append(ipRanges, netip.PrefixFrom(address, 128)) } var err error - a.network, err = tailnet.New(&tailnet.Options{ + a.network, err = tailnet.NewConn(&tailnet.Options{ Addresses: ipRanges, DERPMap: derpMap, Logger: a.logger.Named("tailnet"), diff --git a/agent/conn.go b/agent/conn.go index 55f09a33c247b..7c2180342124a 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -136,7 +136,7 @@ func (c *WebRTCConn) Close() error { type TailnetConn struct { Target netip.Addr - *tailnet.Server + *tailnet.Conn } func (c *TailnetConn) Closed() <-chan struct{} { diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index 0c811730543e4..1966d8368fd77 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -609,14 +609,14 @@ func TestDERP(t *testing.T) { }, } w1IP := tailnet.IP() - w1, err := tailnet.New(&tailnet.Options{ + w1, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, Logger: logger.Named("w1"), DERPMap: derpMap, }) require.NoError(t, err) - w2, err := tailnet.New(&tailnet.Options{ + w2, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, Logger: logger.Named("w2"), DERPMap: derpMap, diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index c6a67468642db..d2858c715a078 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -319,7 +319,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUI } ip := tailnet.IP() - server, err := tailnet.New(&tailnet.Options{ + server, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, DERPMap: &derpMap, Logger: logger, diff --git a/tailnet/tailnet.go b/tailnet/conn.go similarity index 78% rename from tailnet/tailnet.go rename to tailnet/conn.go index 412fef841071d..072a17b30637b 100644 --- a/tailnet/tailnet.go +++ b/tailnet/conn.go @@ -49,8 +49,8 @@ type Options struct { Logger slog.Logger } -// New constructs a new Wireguard server that will accept connections from the addresses provided. -func New(options *Options) (*Server, error) { +// NewConn constructs a new Wireguard server that will accept connections from the addresses provided. +func NewConn(options *Options) (*Conn, error) { if options == nil { options = &Options{} } @@ -186,7 +186,7 @@ func New(options *Options) (*Server, error) { logIPSet := netipx.IPSetBuilder{} logIPs, _ := logIPSet.IPSet() wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter")))) - server := &Server{ + server := &Conn{ logger: options.Logger, magicConn: magicConn, dialer: dialer, @@ -218,8 +218,8 @@ func IP() netip.Addr { return netip.AddrFrom16(uid) } -// Server is an actively listening Wireguard connection. -type Server struct { +// Conn is an actively listening Wireguard connection. +type Conn struct { mutex sync.Mutex logger slog.Logger @@ -237,15 +237,15 @@ type Server struct { // SetNodeCallback is triggered when a network change occurs and peer // renegotiation may be required. Clients should constantly be emitting // node changes. -func (s *Server) SetNodeCallback(callback func(node *Node)) { - s.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { - s.logger.Info(context.Background(), "latency", slog.F("latency", ni.DERPLatency)) +func (c *Conn) SetNodeCallback(callback func(node *Node)) { + c.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { + c.logger.Info(context.Background(), "latency", slog.F("latency", ni.DERPLatency)) callback(&Node{ - ID: s.netMap.SelfNode.ID, - Key: s.netMap.SelfNode.Key, - Addresses: s.netMap.SelfNode.Addresses, - AllowedIPs: s.netMap.SelfNode.AllowedIPs, - DiscoKey: s.magicConn.DiscoPublicKey(), + ID: c.netMap.SelfNode.ID, + Key: c.netMap.SelfNode.Key, + Addresses: c.netMap.SelfNode.Addresses, + AllowedIPs: c.netMap.SelfNode.AllowedIPs, + DiscoKey: c.magicConn.DiscoPublicKey(), PreferredDERP: ni.PreferredDERP, DERPLatency: ni.DERPLatency, }) @@ -254,11 +254,11 @@ func (s *Server) SetNodeCallback(callback func(node *Node)) { // UpdateNodes connects with a set of peers. This can be constantly updated, // and peers will continually be reconnected as necessary. -func (s *Server) UpdateNodes(nodes []*Node) error { - s.mutex.Lock() - defer s.mutex.Unlock() +func (c *Conn) UpdateNodes(nodes []*Node) error { + c.mutex.Lock() + defer c.mutex.Unlock() peerMap := map[tailcfg.NodeID]*tailcfg.Node{} - for _, peer := range s.netMap.Peers { + for _, peer := range c.netMap.Peers { peerMap[peer.ID] = peer } for _, node := range nodes { @@ -272,41 +272,41 @@ func (s *Server) UpdateNodes(nodes []*Node) error { Hostinfo: hostinfo.New().View(), } } - s.netMap.Peers = make([]*tailcfg.Node, 0, len(peerMap)) + c.netMap.Peers = make([]*tailcfg.Node, 0, len(peerMap)) for _, peer := range peerMap { - s.netMap.Peers = append(s.netMap.Peers, peer) + c.netMap.Peers = append(c.netMap.Peers, peer) } - cfg, err := nmcfg.WGCfg(s.netMap, Logger(s.logger.Named("wgconfig")), netmap.AllowSingleHosts, "") + cfg, err := nmcfg.WGCfg(c.netMap, Logger(c.logger.Named("wgconfig")), netmap.AllowSingleHosts, "") if err != nil { return xerrors.Errorf("update wireguard config: %w", err) } - err = s.wireguardEngine.Reconfig(cfg, s.wireguardRouter, &dns.Config{}, &tailcfg.Debug{}) + err = c.wireguardEngine.Reconfig(cfg, c.wireguardRouter, &dns.Config{}, &tailcfg.Debug{}) if err != nil { return xerrors.Errorf("reconfig: %w", err) } - netMapCopy := *s.netMap - s.wireguardEngine.SetNetworkMap(&netMapCopy) + netMapCopy := *c.netMap + c.wireguardEngine.SetNetworkMap(&netMapCopy) return nil } // Ping sends a ping to the Wireguard engine. -func (s *Server) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { - s.wireguardEngine.Ping(ip, pingType, cb) +func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { + c.wireguardEngine.Ping(ip, pingType, cb) } // Close shuts down the Wireguard connection. -func (s *Server) Close() error { - s.mutex.Lock() - defer s.mutex.Unlock() - for _, l := range s.listeners { +func (c *Conn) Close() error { + c.mutex.Lock() + defer c.mutex.Unlock() + for _, l := range c.listeners { _ = l.Close() } - _ = s.dialer.Close() - _ = s.magicConn.Close() - _ = s.netStack.Close() - _ = s.wireguardMonitor.Close() - _ = s.tunDevice.Close() - s.wireguardEngine.Close() + _ = c.dialer.Close() + _ = c.magicConn.Close() + _ = c.netStack.Close() + _ = c.wireguardMonitor.Close() + _ = c.tunDevice.Close() + c.wireguardEngine.Close() return nil } @@ -326,54 +326,54 @@ type Node struct { // Listen announces only on the Tailscale network. // It will start the server if it has not been started yet. -func (s *Server) Listen(network, addr string) (net.Listener, error) { +func (c *Conn) Listen(network, addr string) (net.Listener, error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, xerrors.Errorf("wgnet: %w", err) } lk := listenKey{network, host, port} ln := &listener{ - s: s, + s: c, key: lk, addr: addr, conn: make(chan net.Conn), } - s.mutex.Lock() - if s.listeners == nil { - s.listeners = map[listenKey]*listener{} + c.mutex.Lock() + if c.listeners == nil { + c.listeners = map[listenKey]*listener{} } - if _, ok := s.listeners[lk]; ok { - s.mutex.Unlock() + if _, ok := c.listeners[lk]; ok { + c.mutex.Unlock() return nil, xerrors.Errorf("wgnet: listener already open for %s, %s", network, addr) } - s.listeners[lk] = ln - s.mutex.Unlock() + c.listeners[lk] = ln + c.mutex.Unlock() return ln, nil } -func (s *Server) DialContextTCP(ctx context.Context, ipp netip.AddrPort) (*gonet.TCPConn, error) { - return s.netStack.DialContextTCP(ctx, ipp) +func (c *Conn) DialContextTCP(ctx context.Context, ipp netip.AddrPort) (*gonet.TCPConn, error) { + return c.netStack.DialContextTCP(ctx, ipp) } -func (s *Server) DialContextUDP(ctx context.Context, ipp netip.AddrPort) (*gonet.UDPConn, error) { - return s.netStack.DialContextUDP(ctx, ipp) +func (c *Conn) DialContextUDP(ctx context.Context, ipp netip.AddrPort) (*gonet.UDPConn, error) { + return c.netStack.DialContextUDP(ctx, ipp) } -func (s *Server) forwardTCP(c net.Conn, port uint16) { - s.mutex.Lock() - ln, ok := s.listeners[listenKey{"tcp", "", fmt.Sprint(port)}] - s.mutex.Unlock() +func (c *Conn) forwardTCP(conn net.Conn, port uint16) { + c.mutex.Lock() + ln, ok := c.listeners[listenKey{"tcp", "", fmt.Sprint(port)}] + c.mutex.Unlock() if !ok { - _ = c.Close() + _ = conn.Close() return } t := time.NewTimer(time.Second) defer t.Stop() select { - case ln.conn <- c: + case ln.conn <- conn: case <-t.C: - _ = c.Close() + _ = conn.Close() } } @@ -384,7 +384,7 @@ type listenKey struct { } type listener struct { - s *Server + s *Conn key listenKey addr string conn chan net.Conn @@ -420,12 +420,3 @@ func Logger(logger slog.Logger) tslogger.Logf { logger.Debug(context.Background(), fmt.Sprintf(format, args...)) }) } - -// The exchanger is entirely in-memory and works based on connected nodes. -// It uses a PubSub system to dynamically add/remove nodes from the network -// and build a netmap based on connection ID. -// -// Each node is allocated it's own internal connection ID. -// -// The connecting node *just* requires information about the other node. -// The other node needs connection information of all the others. diff --git a/tailnet/tailnet_test.go b/tailnet/conn_test.go similarity index 96% rename from tailnet/tailnet_test.go rename to tailnet/conn_test.go index 6ac27ba53096b..084475921e8d3 100644 --- a/tailnet/tailnet_test.go +++ b/tailnet/conn_test.go @@ -33,14 +33,14 @@ func TestTailnet(t *testing.T) { derpMap := runDERPAndStun(t, tailnet.Logger(logger.Named("derp"))) w1IP := tailnet.IP() - w1, err := tailnet.New(&tailnet.Options{ + w1, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, Logger: logger.Named("w1"), DERPMap: derpMap, }) require.NoError(t, err) - w2, err := tailnet.New(&tailnet.Options{ + w2, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, Logger: logger.Named("w2"), DERPMap: derpMap, diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go new file mode 100644 index 0000000000000..923ded6c38c02 --- /dev/null +++ b/tailnet/coordinator.go @@ -0,0 +1,175 @@ +package tailnet + +import ( + "context" + "errors" + "sync" + + "github.com/google/uuid" + "golang.org/x/xerrors" + "nhooyr.io/websocket" + "nhooyr.io/websocket/wsjson" +) + +// Coordinate matches the RW structure of a coordinator to exchange node messages. +func Coordinate(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node)) (func(node *Node), <-chan error) { + errChan := make(chan error, 1) + go func() { + for { + var nodes []*Node + err := wsjson.Read(ctx, socket, &nodes) + if err != nil { + errChan <- xerrors.Errorf("read: %w", err) + return + } + updateNodes(nodes) + } + }() + + return func(node *Node) { + err := wsjson.Write(ctx, socket, node) + if err != nil { + errChan <- xerrors.Errorf("write: %w", err) + } + }, errChan +} + +// NewCoordinator constructs a new in-memory connection coordinator. +func NewCoordinator() *Coordinator { + return &Coordinator{ + agentNodes: map[uuid.UUID]*Node{}, + agentClientNodes: map[uuid.UUID]map[uuid.UUID]*Node{}, + agentSockets: map[uuid.UUID]*websocket.Conn{}, + agentClientSockets: map[uuid.UUID]map[uuid.UUID]*websocket.Conn{}, + } +} + +// Coordinator brokers connections over WebSockets. +type Coordinator struct { + mutex sync.Mutex + // Stores the most recent node an agent sent. + agentNodes map[uuid.UUID]*Node + // Stores the most recent node reported by each client. + agentClientNodes map[uuid.UUID]map[uuid.UUID]*Node + // Stores the active connection from an agent. + agentSockets map[uuid.UUID]*websocket.Conn + // Stores the active connection from a client to an agent. + agentClientSockets map[uuid.UUID]map[uuid.UUID]*websocket.Conn +} + +// Client represents a tailnet looking to peer with an agent. +func (c *Coordinator) Client(ctx context.Context, agentID uuid.UUID, socket *websocket.Conn) error { + id := uuid.New() + c.mutex.Lock() + clients, ok := c.agentClientSockets[agentID] + if !ok { + clients = map[uuid.UUID]*websocket.Conn{} + c.agentClientSockets[agentID] = clients + } + clients[id] = socket + agentNode, ok := c.agentNodes[agentID] + if ok { + err := wsjson.Write(ctx, socket, []*Node{agentNode}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write agent node: %w", err) + } + } + + c.mutex.Unlock() + defer func() { + c.mutex.Lock() + defer c.mutex.Unlock() + clients, ok := c.agentClientSockets[agentID] + if !ok { + return + } + delete(clients, id) + nodes, ok := c.agentClientNodes[agentID] + if !ok { + return + } + delete(nodes, id) + }() + + for { + var node Node + err := wsjson.Read(ctx, socket, &node) + if errors.Is(err, context.Canceled) { + return nil + } + if err != nil { + return xerrors.Errorf("read json: %w", err) + } + c.mutex.Lock() + nodes, ok := c.agentClientNodes[agentID] + if !ok { + nodes = map[uuid.UUID]*Node{} + c.agentClientNodes[agentID] = nodes + } + nodes[id] = &node + + agentSocket, ok := c.agentSockets[agentID] + if !ok { + // If the agent isn't connected yet, that's fine. It'll reconcile later. + c.mutex.Unlock() + continue + } + err = wsjson.Write(ctx, agentSocket, []*Node{&node}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write node to agent: %w", err) + } + c.mutex.Unlock() + } +} + +func (c *Coordinator) Agent(ctx context.Context, agentID uuid.UUID, socket *websocket.Conn) error { + c.mutex.Lock() + agentSocket, ok := c.agentSockets[agentID] + if ok { + agentSocket.Close(websocket.StatusGoingAway, "another agent started with the same id") + } + c.agentSockets[agentID] = socket + nodes, ok := c.agentClientNodes[agentID] + if ok { + err := wsjson.Write(ctx, socket, nodes) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write nodes: %w", err) + } + } + c.mutex.Unlock() + defer func() { + c.mutex.Lock() + defer c.mutex.Unlock() + delete(c.agentSockets, agentID) + }() + + for { + var node Node + err := wsjson.Read(ctx, socket, &node) + if errors.Is(err, context.Canceled) { + return nil + } + if err != nil { + return xerrors.Errorf("read node: %w", err) + } + c.mutex.Lock() + c.agentNodes[agentID] = &node + + clients, ok := c.agentClientSockets[agentID] + if !ok { + c.mutex.Unlock() + continue + } + for _, client := range clients { + err = wsjson.Write(ctx, client, []*Node{&node}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("write to client: %w", err) + } + } + c.mutex.Unlock() + } +} diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go new file mode 100644 index 0000000000000..985fd209e0a0a --- /dev/null +++ b/tailnet/coordinator_test.go @@ -0,0 +1,96 @@ +package tailnet_test + +import ( + "bufio" + "context" + "net" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "nhooyr.io/websocket" + + "github.com/coder/coder/tailnet" +) + +func TestCoordinator(t *testing.T) { + t.Parallel() + t.Run("Nodes", func(t *testing.T) { + t.Parallel() + agentID := uuid.New() + coordinator := tailnet.NewCoordinator() + ctx, cancelFunc := context.WithCancel(context.Background()) + client1, client2 := pipeWS(t) + clientNode := make(chan []*tailnet.Node) + sendClientNode, clientErrChan := tailnet.Coordinate(ctx, client1, func(node []*tailnet.Node) { + clientNode <- node + }) + go coordinator.Client(ctx, agentID, client2) + + agent1, agent2 := pipeWS(t) + agentNode := make(chan []*tailnet.Node) + sendAgentNode, agentErrChan := tailnet.Coordinate(ctx, agent1, func(node []*tailnet.Node) { + agentNode <- node + }) + go coordinator.Agent(ctx, agentID, agent2) + + sendAgentNode(&tailnet.Node{}) + sendClientNode(&tailnet.Node{}) + + <-clientNode + <-agentNode + + cancelFunc() + <-clientErrChan + <-agentErrChan + }) +} + +// pipeWS creates a new piped WebSocket pair. +func pipeWS(t *testing.T) (clientConn, serverConn *websocket.Conn) { + t.Helper() + clientConn, _, _ = websocket.Dial(context.Background(), "ws://example.com", &websocket.DialOptions{ + HTTPClient: &http.Client{ + Transport: fakeTransport{ + h: func(w http.ResponseWriter, r *http.Request) { + serverConn, _ = websocket.Accept(w, r, nil) + }, + }, + }, + }) + t.Cleanup(func() { + _ = clientConn.Close(websocket.StatusGoingAway, "") + _ = serverConn.Close(websocket.StatusGoingAway, "") + }) + return clientConn, serverConn +} + +type fakeTransport struct { + h http.HandlerFunc +} + +func (t fakeTransport) RoundTrip(r *http.Request) (*http.Response, error) { + clientConn, serverConn := net.Pipe() + hj := testHijacker{ + ResponseRecorder: httptest.NewRecorder(), + serverConn: serverConn, + } + t.h.ServeHTTP(hj, r) + resp := hj.ResponseRecorder.Result() + if resp.StatusCode == http.StatusSwitchingProtocols { + resp.Body = clientConn + } + return resp, nil +} + +type testHijacker struct { + *httptest.ResponseRecorder + serverConn net.Conn +} + +var _ http.Hijacker = testHijacker{} + +func (hj testHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { + return hj.serverConn, bufio.NewReadWriter(bufio.NewReader(hj.serverConn), bufio.NewWriter(hj.serverConn)), nil +} From 94e90a38b2e1cada001dbd9c8898efb0d745804d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Fri, 12 Aug 2022 08:07:20 -0500 Subject: [PATCH 13/54] Add coordinator tests --- .vscode/settings.json | 2 + tailnet/coordinator.go | 163 ++++++++++++++++++++++-------------- tailnet/coordinator_test.go | 139 +++++++++++++++++++++++++----- 3 files changed, 222 insertions(+), 82 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index aff3b6e60817f..4bec63f232a80 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "cSpell.words": [ "apps", "awsidentity", + "bodyclose", "buildinfo", "buildname", "circbuf", @@ -92,6 +93,7 @@ "templateversions", "testdata", "testid", + "testutil", "tfexec", "tfjson", "tfplan", diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 923ded6c38c02..60e03de922b68 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -11,8 +11,8 @@ import ( "nhooyr.io/websocket/wsjson" ) -// Coordinate matches the RW structure of a coordinator to exchange node messages. -func Coordinate(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node)) (func(node *Node), <-chan error) { +// ServeCoordinator matches the RW structure of a coordinator to exchange node messages. +func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node)) (func(node *Node), <-chan error) { errChan := make(chan error, 1) go func() { for { @@ -37,139 +37,178 @@ func Coordinate(ctx context.Context, socket *websocket.Conn, updateNodes func(no // NewCoordinator constructs a new in-memory connection coordinator. func NewCoordinator() *Coordinator { return &Coordinator{ - agentNodes: map[uuid.UUID]*Node{}, - agentClientNodes: map[uuid.UUID]map[uuid.UUID]*Node{}, - agentSockets: map[uuid.UUID]*websocket.Conn{}, - agentClientSockets: map[uuid.UUID]map[uuid.UUID]*websocket.Conn{}, + nodes: map[uuid.UUID]*Node{}, + agentSockets: map[uuid.UUID]*websocket.Conn{}, + agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]*websocket.Conn{}, } } -// Coordinator brokers connections over WebSockets. +// Coordinator exchanges nodes with agents to establish connections. +// ┌──────────────────┐ ┌────────────────────┐ ┌───────────────────┐ ┌──────────────────┐ +// │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│ +// └──────────────────┘ └────────────────────┘ └───────────────────┘ └──────────────────┘ +// This coordinator is incompatible with multiple Coder +// replicas as all node data is in-memory. type Coordinator struct { mutex sync.Mutex - // Stores the most recent node an agent sent. - agentNodes map[uuid.UUID]*Node - // Stores the most recent node reported by each client. - agentClientNodes map[uuid.UUID]map[uuid.UUID]*Node - // Stores the active connection from an agent. + + // Maps agent and connection IDs to a node. + nodes map[uuid.UUID]*Node + // Maps agent ID to an open socket. agentSockets map[uuid.UUID]*websocket.Conn - // Stores the active connection from a client to an agent. - agentClientSockets map[uuid.UUID]map[uuid.UUID]*websocket.Conn + // Maps agent ID to connection ID for sending + // new node data as it comes in! + agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]*websocket.Conn } -// Client represents a tailnet looking to peer with an agent. -func (c *Coordinator) Client(ctx context.Context, agentID uuid.UUID, socket *websocket.Conn) error { - id := uuid.New() +// Node returns an in-memory node by ID. +func (c *Coordinator) Node(id uuid.UUID) *Node { c.mutex.Lock() - clients, ok := c.agentClientSockets[agentID] - if !ok { - clients = map[uuid.UUID]*websocket.Conn{} - c.agentClientSockets[agentID] = clients - } - clients[id] = socket - agentNode, ok := c.agentNodes[agentID] + defer c.mutex.Unlock() + node := c.nodes[id] + return node +} + +// ServeClient accepts a WebSocket connection that wants to +// connect to an agent with the specified ID. +func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, id uuid.UUID, agent uuid.UUID) error { + c.mutex.Lock() + // When a new connection is requested, we update it with the latest + // node of the agent. This allows the connection to establish. + node, ok := c.nodes[agent] if ok { - err := wsjson.Write(ctx, socket, []*Node{agentNode}) + err := wsjson.Write(ctx, socket, []*Node{node}) if err != nil { c.mutex.Unlock() - return xerrors.Errorf("write agent node: %w", err) + return xerrors.Errorf("write nodes: %w", err) } } - + connectionSockets, ok := c.agentToConnectionSockets[agent] + if !ok { + connectionSockets = map[uuid.UUID]*websocket.Conn{} + c.agentToConnectionSockets[agent] = connectionSockets + } + // Insert this connection into a map so the agent + // can publish node updates. + connectionSockets[id] = socket c.mutex.Unlock() defer func() { c.mutex.Lock() defer c.mutex.Unlock() - clients, ok := c.agentClientSockets[agentID] + // Clean all traces of this connection from the map. + delete(c.nodes, id) + connectionSockets, ok := c.agentToConnectionSockets[agent] if !ok { return } - delete(clients, id) - nodes, ok := c.agentClientNodes[agentID] - if !ok { + delete(connectionSockets, id) + if len(connectionSockets) != 0 { return } - delete(nodes, id) + delete(c.agentToConnectionSockets, agent) }() for { var node Node err := wsjson.Read(ctx, socket, &node) - if errors.Is(err, context.Canceled) { + if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { return nil } if err != nil { return xerrors.Errorf("read json: %w", err) } c.mutex.Lock() - nodes, ok := c.agentClientNodes[agentID] + // Update the node of this client in our in-memory map. + // If an agent entirely shuts down and reconnects, it + // needs to be aware of all clients attempting to + // establish connections. + c.nodes[id] = &node + agentSocket, ok := c.agentSockets[agent] if !ok { - nodes = map[uuid.UUID]*Node{} - c.agentClientNodes[agentID] = nodes - } - nodes[id] = &node - - agentSocket, ok := c.agentSockets[agentID] - if !ok { - // If the agent isn't connected yet, that's fine. It'll reconcile later. c.mutex.Unlock() continue } + // Write the new node from this client to the actively + // connected agent. err = wsjson.Write(ctx, agentSocket, []*Node{&node}) + if errors.Is(err, context.Canceled) { + c.mutex.Unlock() + return nil + } if err != nil { c.mutex.Unlock() - return xerrors.Errorf("write node to agent: %w", err) + return xerrors.Errorf("write json: %w", err) } c.mutex.Unlock() } } -func (c *Coordinator) Agent(ctx context.Context, agentID uuid.UUID, socket *websocket.Conn) error { +// ServeAgent accepts a WebSocket connection to an agent that +// listens to incoming connections and publishes node updates. +func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id uuid.UUID) error { c.mutex.Lock() - agentSocket, ok := c.agentSockets[agentID] - if ok { - agentSocket.Close(websocket.StatusGoingAway, "another agent started with the same id") - } - c.agentSockets[agentID] = socket - nodes, ok := c.agentClientNodes[agentID] + sockets, ok := c.agentToConnectionSockets[id] if ok { + // Publish all nodes that want to connect to the + // desired agent ID. + nodes := make([]*Node, 0, len(sockets)) + for targetID := range sockets { + node, ok := c.nodes[targetID] + if !ok { + continue + } + nodes = append(nodes, node) + } err := wsjson.Write(ctx, socket, nodes) if err != nil { c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) } } + + // If an old agent socket is connected, we close it + // to avoid any leaks. This shouldn't ever occur because + // we expect one agent to be running. + oldAgentSocket, ok := c.agentSockets[id] + if ok { + _ = oldAgentSocket.Close(websocket.StatusNormalClosure, "another agent connected with the same id") + } + c.agentSockets[id] = socket c.mutex.Unlock() defer func() { c.mutex.Lock() defer c.mutex.Unlock() - delete(c.agentSockets, agentID) + delete(c.agentSockets, id) + delete(c.nodes, id) }() for { var node Node err := wsjson.Read(ctx, socket, &node) - if errors.Is(err, context.Canceled) { + if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { return nil } if err != nil { - return xerrors.Errorf("read node: %w", err) + return xerrors.Errorf("read json: %w", err) } c.mutex.Lock() - c.agentNodes[agentID] = &node - - clients, ok := c.agentClientSockets[agentID] + c.nodes[id] = &node + connectionSockets, ok := c.agentToConnectionSockets[id] if !ok { c.mutex.Unlock() continue } - for _, client := range clients { - err = wsjson.Write(ctx, client, []*Node{&node}) - if err != nil { - c.mutex.Unlock() - return xerrors.Errorf("write to client: %w", err) - } + // Publish the new node to every listening socket. + var wg sync.WaitGroup + wg.Add(len(connectionSockets)) + for _, connectionSocket := range connectionSockets { + connectionSocket := connectionSocket + go func() { + _ = wsjson.Write(ctx, connectionSocket, []*Node{&node}) + wg.Done() + }() } + wg.Wait() c.mutex.Unlock() } } diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go index 985fd209e0a0a..d390d460da316 100644 --- a/tailnet/coordinator_test.go +++ b/tailnet/coordinator_test.go @@ -9,47 +9,146 @@ import ( "testing" "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "nhooyr.io/websocket" "github.com/coder/coder/tailnet" + "github.com/coder/coder/testutil" ) func TestCoordinator(t *testing.T) { t.Parallel() - t.Run("Nodes", func(t *testing.T) { + t.Run("ClientWithoutAgent", func(t *testing.T) { + t.Parallel() + coordinator := tailnet.NewCoordinator() + client, server := pipeWS(t) + sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) {}) + id := uuid.New() + closeChan := make(chan struct{}) + go func() { + err := coordinator.ServeClient(context.Background(), server, id, uuid.New()) + assert.NoError(t, err) + close(closeChan) + }() + sendNode(&tailnet.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(id) != nil + }, testutil.WaitShort, testutil.IntervalFast) + err := client.Close(websocket.StatusNormalClosure, "") + require.NoError(t, err) + <-errChan + <-closeChan + }) + + t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() - agentID := uuid.New() coordinator := tailnet.NewCoordinator() - ctx, cancelFunc := context.WithCancel(context.Background()) - client1, client2 := pipeWS(t) - clientNode := make(chan []*tailnet.Node) - sendClientNode, clientErrChan := tailnet.Coordinate(ctx, client1, func(node []*tailnet.Node) { - clientNode <- node + client, server := pipeWS(t) + sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) {}) + id := uuid.New() + closeChan := make(chan struct{}) + go func() { + err := coordinator.ServeAgent(context.Background(), server, id) + assert.NoError(t, err) + close(closeChan) + }() + sendNode(&tailnet.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(id) != nil + }, testutil.WaitShort, testutil.IntervalFast) + err := client.Close(websocket.StatusNormalClosure, "") + require.NoError(t, err) + <-errChan + <-closeChan + }) + + t.Run("AgentWithClient", func(t *testing.T) { + t.Parallel() + coordinator := tailnet.NewCoordinator() + + agentWS, agentServerWS := pipeWS(t) + defer agentWS.Close(websocket.StatusNormalClosure, "") + agentNodeChan := make(chan []*tailnet.Node) + sendAgentNode, agentErrChan := tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) { + agentNodeChan <- nodes }) - go coordinator.Client(ctx, agentID, client2) + agentID := uuid.New() + closeAgentChan := make(chan struct{}) + go func() { + err := coordinator.ServeAgent(context.Background(), agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + sendAgentNode(&tailnet.Node{}) + require.Eventually(t, func() bool { + return coordinator.Node(agentID) != nil + }, testutil.WaitShort, testutil.IntervalFast) - agent1, agent2 := pipeWS(t) - agentNode := make(chan []*tailnet.Node) - sendAgentNode, agentErrChan := tailnet.Coordinate(ctx, agent1, func(node []*tailnet.Node) { - agentNode <- node + clientWS, clientServerWS := pipeWS(t) + defer clientWS.Close(websocket.StatusNormalClosure, "") + defer clientServerWS.Close(websocket.StatusNormalClosure, "") + clientNodeChan := make(chan []*tailnet.Node) + sendClientNode, clientErrChan := tailnet.ServeCoordinator(context.Background(), clientWS, func(nodes []*tailnet.Node) { + clientNodeChan <- nodes }) - go coordinator.Agent(ctx, agentID, agent2) + clientID := uuid.New() + closeClientChan := make(chan struct{}) + go func() { + err := coordinator.ServeClient(context.Background(), clientServerWS, clientID, agentID) + assert.NoError(t, err) + close(closeClientChan) + }() + agentNodes := <-clientNodeChan + require.Len(t, agentNodes, 1) + sendClientNode(&tailnet.Node{}) + clientNodes := <-agentNodeChan + require.Len(t, clientNodes, 1) + // Ensure an update to the agent node reaches the client! sendAgentNode(&tailnet.Node{}) - sendClientNode(&tailnet.Node{}) + agentNodes = <-clientNodeChan + require.Len(t, agentNodes, 1) + + // Close the agent WebSocket so a new one can connect. + err := agentWS.Close(websocket.StatusNormalClosure, "") + require.NoError(t, err) + <-agentErrChan + <-closeAgentChan - <-clientNode - <-agentNode + // Create a new agent connection. This is to simulate a reconnect! + agentWS, agentServerWS = pipeWS(t) + defer agentWS.Close(websocket.StatusNormalClosure, "") + agentNodeChan = make(chan []*tailnet.Node) + _, agentErrChan = tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) { + agentNodeChan <- nodes + }) + closeAgentChan = make(chan struct{}) + go func() { + err := coordinator.ServeAgent(context.Background(), agentServerWS, agentID) + assert.NoError(t, err) + close(closeAgentChan) + }() + // Ensure the existing listening client sends it's node immediately! + clientNodes = <-agentNodeChan + require.Len(t, clientNodes, 1) - cancelFunc() - <-clientErrChan + err = agentWS.Close(websocket.StatusNormalClosure, "") + require.NoError(t, err) <-agentErrChan + <-closeAgentChan + + err = clientWS.Close(websocket.StatusNormalClosure, "") + require.NoError(t, err) + <-clientErrChan + <-closeClientChan }) } // pipeWS creates a new piped WebSocket pair. func pipeWS(t *testing.T) (clientConn, serverConn *websocket.Conn) { t.Helper() + // nolint:bodyclose clientConn, _, _ = websocket.Dial(context.Background(), "ws://example.com", &websocket.DialOptions{ HTTPClient: &http.Client{ Transport: fakeTransport{ @@ -60,8 +159,8 @@ func pipeWS(t *testing.T) (clientConn, serverConn *websocket.Conn) { }, }) t.Cleanup(func() { - _ = clientConn.Close(websocket.StatusGoingAway, "") - _ = serverConn.Close(websocket.StatusGoingAway, "") + _ = serverConn.Close(websocket.StatusInternalError, "") + _ = clientConn.Close(websocket.StatusInternalError, "") }) return clientConn, serverConn } From cd88dca0079a9c3577d211c0b993f70ec95e9011 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 15 Aug 2022 17:54:45 -0500 Subject: [PATCH 14/54] Fix coordination --- agent/conn.go | 2 +- coderd/coderd.go | 10 ++-- coderd/workspaceagents.go | 4 +- codersdk/workspaceagents.go | 84 ++++++++++++++++++++++------------ codersdk/workspaceresources.go | 7 --- site/site.go | 2 +- tailnet/coordinator.go | 11 ++++- tailnet/coordinator_test.go | 17 +++++-- 8 files changed, 86 insertions(+), 51 deletions(-) diff --git a/agent/conn.go b/agent/conn.go index 7c2180342124a..42b3c41ea45d1 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -181,5 +181,5 @@ func (c *TailnetConn) SSHClient() (*ssh.Client, error) { func (c *TailnetConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { _, rawPort, _ := net.SplitHostPort(addr) port, _ := strconv.Atoi(rawPort) - return c.Server.DialContextTCP(ctx, netip.AddrPortFrom(c.Target, uint16(port))) + return c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(c.Target, uint16(port))) } diff --git a/coderd/coderd.go b/coderd/coderd.go index 8ee686fc551d9..9a92ac19ba353 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -72,7 +72,8 @@ type Options struct { TURNServer *turnconn.Server TracerProvider *sdktrace.TracerProvider - DERPMap *tailcfg.DERPMap + ConnCoordinator *tailnet.Coordinator + DERPMap *tailcfg.DERPMap } // New constructs a Coder API handler. @@ -99,6 +100,9 @@ func New(options *Options) *API { if options.PrometheusRegistry == nil { options.PrometheusRegistry = prometheus.NewRegistry() } + if options.ConnCoordinator == nil { + options.ConnCoordinator = tailnet.NewCoordinator() + } siteCacheDir := options.CacheDir if siteCacheDir != "" { @@ -338,7 +342,7 @@ func New(options *Options) *API { r.Get("/iceservers", api.workspaceAgentICEServers) // Everything below this is Tailnet. - r.Get("/node", api.workspaceAgentNode) + r.Get("/coordinate", api.workspaceAgentClientCoordinate) }) r.Route("/{workspaceagent}", func(r chi.Router) { r.Use( @@ -355,7 +359,7 @@ func New(options *Options) *API { r.Get("/derpmap", func(w http.ResponseWriter, r *http.Request) { httpapi.Write(w, http.StatusOK, options.DERPMap) }) - r.Post("/node", api.postWorkspaceAgentNode) + r.Get("/coordinate", api.workspaceAgentClientCoordinate) }) }) r.Route("/workspaceresources/{workspaceresource}", func(r chi.Router) { diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 332e479239248..8e9ecd04706c9 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -507,10 +507,10 @@ func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Co }, nil } -// workspaceAgentNode accepts a WebSocket that reads node network updates. +// workspaceAgentClientCoordinate accepts a WebSocket that reads node network updates. // After accept a PubSub starts listening for new connection node updates // which are written to the WebSocket. -func (api *API) workspaceAgentNode(rw http.ResponseWriter, r *http.Request) { +func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.Request) { api.websocketWaitMutex.Lock() api.websocketWaitGroup.Add(1) api.websocketWaitMutex.Unlock() diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index d2858c715a078..661350941fdb2 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -3,12 +3,14 @@ package codersdk import ( "context" "encoding/json" + "errors" "fmt" "io" "net" "net/http" "net/http/cookiejar" "net/netip" + "time" "cloud.google.com/go/compute/metadata" "github.com/google/uuid" @@ -29,6 +31,7 @@ import ( "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/tailnet" + "github.com/coder/retry" ) type GoogleInstanceIdentityToken struct { @@ -51,6 +54,11 @@ type WorkspaceAgentAuthenticateResponse struct { SessionToken string `json:"session_token"` } +type WorkspaceAgentConnectionRequest struct { + DERPMap tailcfg.DERPMap `json:"derp_map"` + IPAddress netip.Addr `json:"ip_address"` +} + // AuthWorkspaceGoogleInstanceIdentity uses the Google Compute Engine Metadata API to // fetch a signed JWT, and exchange it for a session token for a workspace agent. // @@ -303,7 +311,7 @@ func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker return &workspaceAgentNodeBroker{conn}, nil } -func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUID, logger slog.Logger) (agent.Conn, error) { +func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logger, agentID uuid.UUID) (agent.Conn, error) { res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/derpmap", agentID), nil) if err != nil { return nil, err @@ -319,7 +327,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUI } ip := tailnet.IP() - server, err := tailnet.NewConn(&tailnet.Options{ + conn, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, DERPMap: &derpMap, Logger: logger, @@ -327,40 +335,56 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, agentID uuid.UUI if err != nil { return nil, xerrors.Errorf("create tailnet: %w", err) } - server.SetNodeCallback(func(node *tailnet.Node) { - res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/node", agentID), node) - if err != nil { - logger.Error(ctx, "update node", slog.Error(err), slog.F("node", node)) - return - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - logger.Error(ctx, "update node", slog.F("status_code", res.StatusCode), slog.F("node", node)) - } - }) - workspaceAgent, err := c.WorkspaceAgent(ctx, agentID) + + coordinateURL, err := c.URL.Parse("/api/v2/workspaceagents/me/coordinate") if err != nil { - return nil, xerrors.Errorf("get workspace agent: %w", err) - } - ipRanges := make([]netip.Prefix, 0, len(workspaceAgent.IPAddresses)) - for _, address := range workspaceAgent.IPAddresses { - ipRanges = append(ipRanges, netip.PrefixFrom(address, 128)) - } - agentNode := &tailnet.Node{ - Key: workspaceAgent.NodePublicKey, - DiscoKey: workspaceAgent.DiscoPublicKey, - PreferredDERP: workspaceAgent.PreferredDERP, - Addresses: ipRanges, - AllowedIPs: ipRanges, + return nil, xerrors.Errorf("parse url: %w", err) } - logger.Debug(ctx, "adding agent node", slog.F("node", agentNode)) - err = server.UpdateNodes([]*tailnet.Node{agentNode}) + jar, err := cookiejar.New(nil) if err != nil { - return nil, xerrors.Errorf("update nodes: %w", err) + return nil, xerrors.Errorf("create cookie jar: %w", err) + } + jar.SetCookies(coordinateURL, []*http.Cookie{{ + Name: SessionTokenKey, + Value: c.SessionToken, + }}) + httpClient := &http.Client{ + Jar: jar, } + go func() { + for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { + logger.Debug(ctx, "connecting") + ws, res, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ + HTTPClient: httpClient, + // Need to disable compression to avoid a data-race. + CompressionMode: websocket.CompressionDisabled, + }) + if errors.Is(err, context.Canceled) { + return + } + if err != nil { + logger.Debug(ctx, "failed to dial", slog.Error(err)) + continue + } + _ = res.Body.Close() + sendNode, errChan := tailnet.ServeCoordinator(ctx, ws, func(node []*tailnet.Node) error { + return conn.UpdateNodes(node) + }) + conn.SetNodeCallback(sendNode) + logger.Debug(ctx, "serving coordinator") + err = <-errChan + if errors.Is(err, context.Canceled) { + return + } + if err != nil { + logger.Debug(ctx, "error serving coordinator", slog.Error(err)) + continue + } + } + }() return &agent.TailnetConn{ Target: workspaceAgent.IPAddresses[0], - Server: server, + Conn: conn, }, nil } diff --git a/codersdk/workspaceresources.go b/codersdk/workspaceresources.go index d5107e2f53bf4..cdc98a69568bd 100644 --- a/codersdk/workspaceresources.go +++ b/codersdk/workspaceresources.go @@ -5,11 +5,9 @@ import ( "encoding/json" "fmt" "net/http" - "net/netip" "time" "github.com/google/uuid" - "tailscale.com/types/key" ) type WorkspaceAgentStatus string @@ -54,11 +52,6 @@ type WorkspaceAgent struct { StartupScript string `json:"startup_script,omitempty"` Directory string `json:"directory,omitempty"` Apps []WorkspaceApp `json:"apps"` - - // For internal routing only. - IPAddresses []netip.Addr `json:"ip_addresses"` - NodePublicKey key.NodePublic `json:"node_public_key"` - DiscoPublicKey key.DiscoPublic `json:"disco_public_key"` // PreferredDERP represents the connected region. PreferredDERP int `json:"preferred_derp"` // Maps DERP region to MS latency. diff --git a/site/site.go b/site/site.go index a436532d7fb69..6afebf731697b 100644 --- a/site/site.go +++ b/site/site.go @@ -346,7 +346,7 @@ func secureHeaders(next http.Handler) http.Handler { return secure.New(secure.Options{ PermissionsPolicy: permissions, - // Prevent the browser from sending Referer header with requests + // Prevent the browser from sending Referrer header with requests ReferrerPolicy: "no-referrer", }).Handler(cspHeaders(next)) } diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 60e03de922b68..1c51125918475 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -12,7 +12,7 @@ import ( ) // ServeCoordinator matches the RW structure of a coordinator to exchange node messages. -func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node)) (func(node *Node), <-chan error) { +func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) { errChan := make(chan error, 1) go func() { for { @@ -22,12 +22,19 @@ func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes f errChan <- xerrors.Errorf("read: %w", err) return } - updateNodes(nodes) + err = updateNodes(nodes) + if err != nil { + errChan <- xerrors.Errorf("update nodes: %w", err) + } } }() return func(node *Node) { err := wsjson.Write(ctx, socket, node) + if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { + errChan <- nil + return + } if err != nil { errChan <- xerrors.Errorf("write: %w", err) } diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go index d390d460da316..68e6016c9d412 100644 --- a/tailnet/coordinator_test.go +++ b/tailnet/coordinator_test.go @@ -23,7 +23,9 @@ func TestCoordinator(t *testing.T) { t.Parallel() coordinator := tailnet.NewCoordinator() client, server := pipeWS(t) - sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) {}) + sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) error { + return nil + }) id := uuid.New() closeChan := make(chan struct{}) go func() { @@ -45,7 +47,9 @@ func TestCoordinator(t *testing.T) { t.Parallel() coordinator := tailnet.NewCoordinator() client, server := pipeWS(t) - sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) {}) + sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) error { + return nil + }) id := uuid.New() closeChan := make(chan struct{}) go func() { @@ -70,8 +74,9 @@ func TestCoordinator(t *testing.T) { agentWS, agentServerWS := pipeWS(t) defer agentWS.Close(websocket.StatusNormalClosure, "") agentNodeChan := make(chan []*tailnet.Node) - sendAgentNode, agentErrChan := tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) { + sendAgentNode, agentErrChan := tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) error { agentNodeChan <- nodes + return nil }) agentID := uuid.New() closeAgentChan := make(chan struct{}) @@ -89,8 +94,9 @@ func TestCoordinator(t *testing.T) { defer clientWS.Close(websocket.StatusNormalClosure, "") defer clientServerWS.Close(websocket.StatusNormalClosure, "") clientNodeChan := make(chan []*tailnet.Node) - sendClientNode, clientErrChan := tailnet.ServeCoordinator(context.Background(), clientWS, func(nodes []*tailnet.Node) { + sendClientNode, clientErrChan := tailnet.ServeCoordinator(context.Background(), clientWS, func(nodes []*tailnet.Node) error { clientNodeChan <- nodes + return nil }) clientID := uuid.New() closeClientChan := make(chan struct{}) @@ -120,8 +126,9 @@ func TestCoordinator(t *testing.T) { agentWS, agentServerWS = pipeWS(t) defer agentWS.Close(websocket.StatusNormalClosure, "") agentNodeChan = make(chan []*tailnet.Node) - _, agentErrChan = tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) { + _, agentErrChan = tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) error { agentNodeChan <- nodes + return nil }) closeAgentChan = make(chan struct{}) go func() { From f4e58873059d1192d2e7a0130603a8363197b234 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 16 Aug 2022 11:53:18 -0500 Subject: [PATCH 15/54] It compiles! --- agent/agent.go | 74 ++++++++------------ cli/agent.go | 4 +- cli/cliflag/cliflag.go | 3 +- cli/cliflag/cliflag_test.go | 1 + cli/configssh.go | 2 +- coderd/authorize.go | 1 + coderd/coderd.go | 7 +- coderd/devtunnel/tunnel_test.go | 4 +- coderd/workspaceagents.go | 116 ++++++++++++++------------------ coderd/workspaceagents_test.go | 29 ++++---- codersdk/workspaceagents.go | 44 +++++------- tailnet/conn.go | 4 +- 12 files changed, 124 insertions(+), 165 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 1ec80c8b5b80f..1d36ea1a7474e 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -28,6 +28,7 @@ import ( "go.uber.org/atomic" gossh "golang.org/x/crypto/ssh" "golang.org/x/xerrors" + "nhooyr.io/websocket" "tailscale.com/tailcfg" "cdr.dev/slog" @@ -51,10 +52,10 @@ const ( ) type Options struct { - EnableTailnet bool - NodeDialer NodeDialer - WebRTCDialer WebRTCDialer - FetchMetadata FetchMetadata + EnableTailnet bool + CoordinatorDialer CoordinatorDialer + WebRTCDialer WebRTCDialer + FetchMetadata FetchMetadata ReconnectingPTYTimeout time.Duration EnvironmentVariables map[string]string @@ -71,18 +72,9 @@ type Metadata struct { type WebRTCDialer func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) -// NodeBroker handles the exchange of node information. -type NodeBroker interface { - io.Closer - // Read will be a constant stream of incoming connection requests. - Read(ctx context.Context) (*tailnet.Node, error) - // Write should be called with the listening agent node information. - Write(ctx context.Context, node *tailnet.Node) error -} - -// NodeDialer is a function that constructs a new broker. +// CoordinatorDialer is a function that constructs a new broker. // A dialer must be passed in to allow for reconnects. -type NodeDialer func(ctx context.Context) (NodeBroker, error) +type CoordinatorDialer func(ctx context.Context) (*websocket.Conn, error) // FetchMetadata is a function to obtain metadata for the agent. type FetchMetadata func(ctx context.Context) (Metadata, error) @@ -100,7 +92,7 @@ func New(options Options) io.Closer { closed: make(chan struct{}), envVars: options.EnvironmentVariables, enableTailnet: options.EnableTailnet, - nodeDialer: options.NodeDialer, + coordinatorDialer: options.CoordinatorDialer, fetchMetadata: options.FetchMetadata, } server.init(ctx) @@ -125,9 +117,9 @@ type agent struct { fetchMetadata FetchMetadata sshServer *ssh.Server - enableTailnet bool - network *tailnet.Conn - nodeDialer NodeDialer + enableTailnet bool + network *tailnet.Conn + coordinatorDialer CoordinatorDialer } func (a *agent) run(ctx context.Context) { @@ -190,7 +182,7 @@ func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap a.logger.Critical(ctx, "create tailnet", slog.Error(err)) return } - go a.runNodeBroker(ctx) + go a.runCoordinator(ctx) sshListener, err := a.network.Listen("tcp", ":12212") if err != nil { @@ -208,14 +200,14 @@ func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap }() } -// runNodeBroker listens for nodes and updates the self-node as it changes. -func (a *agent) runNodeBroker(ctx context.Context) { - var nodeBroker NodeBroker +// runCoordinator listens for nodes and updates the self-node as it changes. +func (a *agent) runCoordinator(ctx context.Context) { + var coordinator *websocket.Conn var err error // An exponential back-off occurs when the connection is failing to dial. // This is to prevent server spam in case of a coderd outage. for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { - nodeBroker, err = a.nodeDialer(ctx) + coordinator, err = a.coordinatorDialer(ctx) if err != nil { if errors.Is(err, context.Canceled) { return @@ -226,36 +218,24 @@ func (a *agent) runNodeBroker(ctx context.Context) { a.logger.Warn(context.Background(), "failed to dial", slog.Error(err)) continue } - a.logger.Info(context.Background(), "connected to node broker") + a.logger.Info(context.Background(), "connected to coordination server") break } + sendNodes, errChan := tailnet.ServeCoordinator(ctx, coordinator, a.network.UpdateNodes) + a.network.SetNodeCallback(sendNodes) select { case <-ctx.Done(): return - default: - } - - a.network.SetNodeCallback(func(node *tailnet.Node) { - err := nodeBroker.Write(ctx, node) - if err != nil { - a.logger.Warn(context.Background(), "write node", slog.Error(err), slog.F("node", node)) - } - }) - - for { - node, err := nodeBroker.Read(ctx) - if err != nil { - if a.isClosed() { - return - } - a.logger.Debug(ctx, "node broker accept exited; restarting connection", slog.Error(err)) - a.runNodeBroker(ctx) + case err := <-errChan: + if a.isClosed() { return } - err = a.network.UpdateNodes([]*tailnet.Node{node}) - if err != nil { - a.logger.Error(ctx, "update tailnet nodes", slog.Error(err), slog.F("node", node)) + if errors.Is(err, context.Canceled) { + return } + a.logger.Debug(ctx, "node broker accept exited; restarting connection", slog.Error(err)) + a.runCoordinator(ctx) + return } } @@ -887,7 +867,9 @@ func (a *agent) Close() error { } close(a.closed) a.closeCancel() + fmt.Printf("CLOSING NETWORK!!!!\n") if a.network != nil { + fmt.Printf("ACTUALLY CLOSING NETWORK!!!!\n") _ = a.network.Close() } _ = a.sshServer.Close() diff --git a/cli/agent.go b/cli/agent.go index 227b8bc222fb9..ba9989ddde1b9 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -181,8 +181,8 @@ func workspaceAgent() *cobra.Command { // shells so "gitssh" works! "CODER_AGENT_TOKEN": client.SessionToken, }, - EnableTailnet: wireguard, - NodeDialer: client.WorkspaceAgentNodeBroker, + EnableTailnet: wireguard, + CoordinatorDialer: client.ListenWorkspaceAgentTailnet, }) <-cmd.Context().Done() return closer.Close() diff --git a/cli/cliflag/cliflag.go b/cli/cliflag/cliflag.go index 053ea948f04cf..843416c3ff3ea 100644 --- a/cli/cliflag/cliflag.go +++ b/cli/cliflag/cliflag.go @@ -6,8 +6,7 @@ // // Will produce the following usage docs: // -// -a, --address string The address to serve the API and dashboard (uses $CODER_ADDRESS). (default "127.0.0.1:3000") -// +// -a, --address string The address to serve the API and dashboard (uses $CODER_ADDRESS). (default "127.0.0.1:3000") package cliflag import ( diff --git a/cli/cliflag/cliflag_test.go b/cli/cliflag/cliflag_test.go index a5c9f532abe79..acdf7d6765fb5 100644 --- a/cli/cliflag/cliflag_test.go +++ b/cli/cliflag/cliflag_test.go @@ -14,6 +14,7 @@ import ( ) // Testcliflag cannot run in parallel because it uses t.Setenv. +// //nolint:paralleltest func TestCliflag(t *testing.T) { t.Run("StringDefault", func(t *testing.T) { diff --git a/cli/configssh.go b/cli/configssh.go index 08aeaf6c9d37a..a3f9a4517c8e0 100644 --- a/cli/configssh.go +++ b/cli/configssh.go @@ -558,7 +558,7 @@ func currentBinPath(w io.Writer) (string, error) { // diffBytes takes two byte slices and diffs them as if they were in a // file named name. -//nolint: revive // Color is an option, not a control coupling. +// nolint: revive // Color is an option, not a control coupling. func diffBytes(name string, b1, b2 []byte, color bool) ([]byte, error) { var buf bytes.Buffer var opts []write.Option diff --git a/coderd/authorize.go b/coderd/authorize.go index 855348aaf6184..981af5e868c36 100644 --- a/coderd/authorize.go +++ b/coderd/authorize.go @@ -31,6 +31,7 @@ func AuthorizeFilter[O rbac.Objecter](api *API, r *http.Request, action rbac.Act // This function will log appropriately, but the caller must return an // error to the api client. // Eg: +// // if !api.Authorize(...) { // httpapi.Forbidden(rw) // return diff --git a/coderd/coderd.go b/coderd/coderd.go index 9a92ac19ba353..bb404518c8523 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -341,8 +341,7 @@ func New(options *Options) *API { r.Get("/turn", api.workspaceAgentTurn) r.Get("/iceservers", api.workspaceAgentICEServers) - // Everything below this is Tailnet. - r.Get("/coordinate", api.workspaceAgentClientCoordinate) + r.Get("/coordinate", api.workspaceAgentCoordinate) }) r.Route("/{workspaceagent}", func(r chi.Router) { r.Use( @@ -356,9 +355,7 @@ func New(options *Options) *API { r.Get("/pty", api.workspaceAgentPTY) r.Get("/iceservers", api.workspaceAgentICEServers) - r.Get("/derpmap", func(w http.ResponseWriter, r *http.Request) { - httpapi.Write(w, http.StatusOK, options.DERPMap) - }) + r.Get("/connection", api.workspaceAgentConnection) r.Get("/coordinate", api.workspaceAgentClientCoordinate) }) }) diff --git a/coderd/devtunnel/tunnel_test.go b/coderd/devtunnel/tunnel_test.go index 23dd92d966d54..eeddf1d8cb4dc 100644 --- a/coderd/devtunnel/tunnel_test.go +++ b/coderd/devtunnel/tunnel_test.go @@ -100,8 +100,8 @@ func TestTunnel(t *testing.T) { // fakeTunnelServer is a fake version of the real dev tunnel server. It fakes 2 client interactions // that we want to test: -// 1. Responding to a POST /tun from the client -// 2. Sending an HTTP request down the wireguard connection +// 1. Responding to a POST /tun from the client +// 2. Sending an HTTP request down the wireguard connection // // Note that for 2, we don't implement a full proxy that accepts arbitrary requests, we just send // a test request over the Wireguard tunnel to make sure that we can listen. The proxy behavior is diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 8e9ecd04706c9..d645a857c2306 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -1,7 +1,6 @@ package coderd import ( - "bytes" "context" "database/sql" "encoding/json" @@ -17,7 +16,6 @@ import ( "github.com/hashicorp/yamux" "golang.org/x/xerrors" "nhooyr.io/websocket" - "nhooyr.io/websocket/wsjson" "cdr.dev/slog" "github.com/coder/coder/agent" @@ -120,6 +118,12 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) { workspaceAgent := httpmw.WorkspaceAgent(r) + ips := make([]netip.Addr, 0) + for _, ip := range workspaceAgent.IPAddresses { + var ipData [16]byte + copy(ipData[:], []byte(ip.IPNet.IP)) + ips = append(ips, netip.AddrFrom16(ipData)) + } apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ @@ -130,7 +134,7 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) } httpapi.Write(rw, http.StatusOK, agent.Metadata{ - IPAddresses: apiAgent.IPAddresses, + IPAddresses: ips, DERPMap: api.DERPMap, EnvironmentVariables: apiAgent.EnvironmentVariables, StartupScript: apiAgent.StartupScript, @@ -507,10 +511,26 @@ func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Co }, nil } -// workspaceAgentClientCoordinate accepts a WebSocket that reads node network updates. -// After accept a PubSub starts listening for new connection node updates -// which are written to the WebSocket. -func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.Request) { +func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request) { + workspaceAgent := httpmw.WorkspaceAgentParam(r) + workspace := httpmw.WorkspaceParam(r) + if !api.Authorize(r, rbac.ActionRead, workspace) { + httpapi.ResourceNotFound(rw) + return + } + ips := make([]netip.Addr, 0) + for _, ip := range workspaceAgent.IPAddresses { + var ipData [16]byte + copy(ipData[:], []byte(ip.IPNet.IP)) + ips = append(ips, netip.AddrFrom16(ipData)) + } + httpapi.Write(rw, http.StatusOK, codersdk.WorkspaceAgentConnectionInfo{ + DERPMap: api.DERPMap, + IPAddresses: ips, + }) +} + +func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request) { api.websocketWaitMutex.Lock() api.websocketWaitGroup.Add(1) api.websocketWaitMutex.Unlock() @@ -526,54 +546,36 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R return } defer conn.Close(websocket.StatusNormalClosure, "") - agentIDBytes, _ := workspaceAgent.ID.MarshalText() - subCancel, err := api.Pubsub.Subscribe("tailnet", func(ctx context.Context, message []byte) { - // Since we subscribe to all peer broadcasts, we do a light check to - // make sure we're the intended recipient without fully decoding the - // message. - if len(message) < len(agentIDBytes) { - api.Logger.Error(ctx, "wireguard peer message too short", slog.F("got", len(message))) - return - } - // We aren't the intended recipient. - if !bytes.Equal(message[:len(agentIDBytes)], agentIDBytes) { - return - } - _ = conn.Write(ctx, websocket.MessageText, message[len(agentIDBytes):]) - }) + err = api.ConnCoordinator.ServeAgent(r.Context(), conn, workspaceAgent.ID) if err != nil { - api.Logger.Error(context.Background(), "pubsub listen", slog.Error(err)) + _ = conn.Close(websocket.StatusInternalError, err.Error()) return } - defer subCancel() +} - for { - var node tailnet.Node - err = wsjson.Read(r.Context(), conn, &node) - if err != nil { - return - } - err := api.Database.UpdateWorkspaceAgentNetworkByID(r.Context(), database.UpdateWorkspaceAgentNetworkByIDParams{ - ID: workspaceAgent.ID, - NodePublicKey: sql.NullString{ - String: node.Key.String(), - Valid: true, - }, - DERPLatency: node.DERPLatency, - DiscoPublicKey: sql.NullString{ - String: node.DiscoKey.String(), - Valid: true, - }, - PreferredDERP: int32(node.PreferredDERP), - UpdatedAt: database.Now(), +// workspaceAgentClientCoordinate accepts a WebSocket that reads node network updates. +// After accept a PubSub starts listening for new connection node updates +// which are written to the WebSocket. +func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.Request) { + api.websocketWaitMutex.Lock() + api.websocketWaitGroup.Add(1) + api.websocketWaitMutex.Unlock() + defer api.websocketWaitGroup.Done() + workspaceAgent := httpmw.WorkspaceAgentParam(r) + + conn, err := websocket.Accept(rw, r, nil) + if err != nil { + httpapi.Write(rw, http.StatusBadRequest, codersdk.Response{ + Message: "Failed to accept websocket.", + Detail: err.Error(), }) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Internal error setting agent keys.", - Detail: err.Error(), - }) - return - } + return + } + defer conn.Close(websocket.StatusNormalClosure, "") + err = api.ConnCoordinator.ServeClient(r.Context(), conn, uuid.New(), workspaceAgent.ID) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, err.Error()) + return } } @@ -655,20 +657,6 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work Apps: apps, PreferredDERP: int(dbAgent.PreferredDERP), DERPLatency: dbAgent.DERPLatency, - IPAddresses: ips, - } - - if dbAgent.NodePublicKey.Valid { - err := workspaceAgent.NodePublicKey.UnmarshalText([]byte(dbAgent.NodePublicKey.String)) - if err != nil { - return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse node public key: %w", err) - } - } - if dbAgent.DiscoPublicKey.Valid { - err := workspaceAgent.DiscoPublicKey.UnmarshalText([]byte(dbAgent.DiscoPublicKey.String)) - if err != nil { - return codersdk.WorkspaceAgent{}, xerrors.Errorf("parse disco public key: %w", err) - } } if dbAgent.FirstConnectedAt.Valid { diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index b9a0e7daebf14..afa5775fbb7d6 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -304,31 +304,30 @@ func TestWorkspaceAgentTailnet(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - EnableTailnet: true, - NodeDialer: agentClient.WorkspaceAgentNodeBroker, - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), - }) - t.Cleanup(func() { - _ = agentCloser.Close() + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + EnableTailnet: true, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) + defer agentCloser.Close() resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.LatestBuild.ID) - time.Sleep(3 * time.Second) - - conn, err := client.DialWorkspaceAgentTailnet(context.Background(), resources[0].Agents[0].ID, slogtest.Make(t, nil).Named("tailnet").Leveled(slog.LevelDebug)) + ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + conn, err := client.DialWorkspaceAgentTailnet(ctx, slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), resources[0].Agents[0].ID) require.NoError(t, err) - t.Cleanup(func() { - _ = conn.Close() - }) + defer conn.Close() sshClient, err := conn.SSHClient() require.NoError(t, err) session, err := sshClient.NewSession() require.NoError(t, err) output, err := session.CombinedOutput("echo test") require.NoError(t, err) - fmt.Printf("Output: %s\n", output) + _ = session.Close() + _ = sshClient.Close() + _ = conn.Close() + fmt.Printf("\n\n\n\nOutput: %s\n\n\n\n", output) } func TestWorkspaceAgentPTY(t *testing.T) { diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 661350941fdb2..e00c2676e7af5 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -54,9 +54,11 @@ type WorkspaceAgentAuthenticateResponse struct { SessionToken string `json:"session_token"` } -type WorkspaceAgentConnectionRequest struct { - DERPMap tailcfg.DERPMap `json:"derp_map"` - IPAddress netip.Addr `json:"ip_address"` +// WorkspaceAgentConnectionInfo returns required information for establishing +// a connection with a workspace. +type WorkspaceAgentConnectionInfo struct { + DERPMap *tailcfg.DERPMap `json:"derp_map"` + IPAddresses []netip.Addr `json:"ip_address"` } // AuthWorkspaceGoogleInstanceIdentity uses the Google Compute Engine Metadata API to @@ -280,8 +282,8 @@ func (c *Client) UpdateWorkspaceAgentNode(ctx context.Context, agentID uuid.UUID return nil } -func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker, error) { - serverURL, err := c.URL.Parse("/api/v2/workspaceagents/me/node") +func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (*websocket.Conn, error) { + coordinateURL, err := c.URL.Parse("/api/v2/workspaceagents/me/coordinate") if err != nil { return nil, xerrors.Errorf("parse url: %w", err) } @@ -289,30 +291,21 @@ func (c *Client) WorkspaceAgentNodeBroker(ctx context.Context) (agent.NodeBroker if err != nil { return nil, xerrors.Errorf("create cookie jar: %w", err) } - jar.SetCookies(serverURL, []*http.Cookie{{ + jar.SetCookies(coordinateURL, []*http.Cookie{{ Name: SessionTokenKey, Value: c.SessionToken, }}) httpClient := &http.Client{ Jar: jar, } - - conn, res, err := websocket.Dial(ctx, serverURL.String(), &websocket.DialOptions{ + conn, _, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, - // Need to disable compression to avoid a data-race. - CompressionMode: websocket.CompressionDisabled, }) - if err != nil { - if res == nil { - return nil, xerrors.Errorf("websocket dial: %w", err) - } - return nil, readBodyAsError(res) - } - return &workspaceAgentNodeBroker{conn}, nil + return conn, err } func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logger, agentID uuid.UUID) (agent.Conn, error) { - res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/derpmap", agentID), nil) + res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaceagents/%s/connection", agentID), nil) if err != nil { return nil, err } @@ -320,23 +313,23 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg if res.StatusCode != http.StatusOK { return nil, readBodyAsError(res) } - var derpMap tailcfg.DERPMap - err = json.NewDecoder(res.Body).Decode(&derpMap) + var connInfo WorkspaceAgentConnectionInfo + err = json.NewDecoder(res.Body).Decode(&connInfo) if err != nil { - return nil, xerrors.Errorf("decode derpmap: %w", err) + return nil, xerrors.Errorf("decode conn info: %w", err) } ip := tailnet.IP() conn, err := tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(ip, 128)}, - DERPMap: &derpMap, + DERPMap: connInfo.DERPMap, Logger: logger, }) if err != nil { return nil, xerrors.Errorf("create tailnet: %w", err) } - coordinateURL, err := c.URL.Parse("/api/v2/workspaceagents/me/coordinate") + coordinateURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", agentID)) if err != nil { return nil, xerrors.Errorf("parse url: %w", err) } @@ -354,7 +347,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg go func() { for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { logger.Debug(ctx, "connecting") - ws, res, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ + ws, _, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, // Need to disable compression to avoid a data-race. CompressionMode: websocket.CompressionDisabled, @@ -366,7 +359,6 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg logger.Debug(ctx, "failed to dial", slog.Error(err)) continue } - _ = res.Body.Close() sendNode, errChan := tailnet.ServeCoordinator(ctx, ws, func(node []*tailnet.Node) error { return conn.UpdateNodes(node) }) @@ -383,7 +375,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg } }() return &agent.TailnetConn{ - Target: workspaceAgent.IPAddresses[0], + Target: connInfo.IPAddresses[0], Conn: conn, }, nil } diff --git a/tailnet/conn.go b/tailnet/conn.go index 072a17b30637b..03bbde111beab 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -296,11 +296,11 @@ func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate. // Close shuts down the Wireguard connection. func (c *Conn) Close() error { - c.mutex.Lock() - defer c.mutex.Unlock() for _, l := range c.listeners { _ = l.Close() } + c.mutex.Lock() + defer c.mutex.Unlock() _ = c.dialer.Close() _ = c.magicConn.Close() _ = c.netStack.Close() From 93f965aa298a80b725bc5ca1477715003349f1f5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 01:37:55 +0000 Subject: [PATCH 16/54] Move all connection negotiation in-memory --- .vscode/settings.json | 2 + agent/agent.go | 36 +- agent/conn.go | 7 +- cli/root.go | 1 - cli/server.go | 75 ++- cli/ssh.go | 101 ++-- cli/wireguardtunnel.go | 257 ---------- coderd/database/databasefake/databasefake.go | 21 - coderd/database/dbtypes/dbtypes.go | 6 - coderd/database/dump.sql | 7 +- .../database/migrations/000029_tailnet.up.sql | 8 - ...ilnet.down.sql => 000034_tailnet.down.sql} | 0 .../database/migrations/000034_tailnet.up.sql | 3 + coderd/database/models.go | 6 - coderd/database/postgres/postgres.go | 2 +- coderd/database/querier.go | 1 - coderd/database/queries.sql.go | 82 +--- coderd/database/queries/workspaceagents.sql | 17 +- coderd/database/sqlc.yaml | 7 +- coderd/provisionerdaemons.go | 11 - coderd/provisionerjobs.go | 2 +- coderd/workspaceagents.go | 43 +- coderd/workspaceresources.go | 2 +- codersdk/workspaceagents.go | 18 +- peer/peerwg/derp.go | 73 --- peer/peerwg/handshake.go | 94 ---- peer/peerwg/ssh.go | 38 -- peer/peerwg/wireguard.go | 437 ------------------ peer/peerwg/wireguard_test.go | 116 ----- site/src/api/typesGenerated.ts | 16 +- tailnet/conn.go | 51 +- 31 files changed, 224 insertions(+), 1316 deletions(-) delete mode 100644 cli/wireguardtunnel.go delete mode 100644 coderd/database/dbtypes/dbtypes.go delete mode 100644 coderd/database/migrations/000029_tailnet.up.sql rename coderd/database/migrations/{000029_tailnet.down.sql => 000034_tailnet.down.sql} (100%) create mode 100644 coderd/database/migrations/000034_tailnet.up.sql delete mode 100644 peer/peerwg/derp.go delete mode 100644 peer/peerwg/handshake.go delete mode 100644 peer/peerwg/ssh.go delete mode 100644 peer/peerwg/wireguard.go delete mode 100644 peer/peerwg/wireguard_test.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 2ea4100a2d00b..07f5b0ccf729b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -38,6 +38,7 @@ "idtoken", "Iflag", "incpatch", + "ipnstate", "isatty", "Jobf", "Keygen", @@ -52,6 +53,7 @@ "namesgenerator", "namespacing", "netaddr", + "netip", "netmap", "netns", "netstack", diff --git a/agent/agent.go b/agent/agent.go index 1d36ea1a7474e..3e909c8715ced 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -51,6 +51,15 @@ const ( MagicSessionErrorCode = 229 ) +var ( + // tailnetIP is a static IPv6 address with the Tailscale prefix that is used to route + // connections from clients to this node. A dynamic address is not required because a Tailnet + // client only dials a single agent at a time. + tailnetIP = netip.MustParseAddr("fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4") + tailnetSSHPort = 1 + tailnetReconnectingPTYPort = 2 +) + type Options struct { EnableTailnet bool CoordinatorDialer CoordinatorDialer @@ -63,7 +72,6 @@ type Options struct { } type Metadata struct { - IPAddresses []netip.Addr `json:"ip_addresses"` DERPMap *tailcfg.DERPMap `json:"derpmap"` EnvironmentVariables map[string]string `json:"environment_variables"` StartupScript string `json:"startup_script"` @@ -163,18 +171,14 @@ func (a *agent) run(ctx context.Context) { go a.runWebRTCNetworking(ctx) if a.enableTailnet { - go a.runTailnet(ctx, metadata.IPAddresses, metadata.DERPMap) + go a.runTailnet(ctx, metadata.DERPMap) } } -func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap *tailcfg.DERPMap) { - ipRanges := make([]netip.Prefix, 0, len(addresses)) - for _, address := range addresses { - ipRanges = append(ipRanges, netip.PrefixFrom(address, 128)) - } +func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { var err error a.network, err = tailnet.NewConn(&tailnet.Options{ - Addresses: ipRanges, + Addresses: []netip.Prefix{netip.PrefixFrom(tailnetIP, 128)}, DERPMap: derpMap, Logger: a.logger.Named("tailnet"), }) @@ -184,7 +188,7 @@ func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap } go a.runCoordinator(ctx) - sshListener, err := a.network.Listen("tcp", ":12212") + sshListener, err := a.network.Listen("tcp", ":"+strconv.Itoa(tailnetSSHPort)) if err != nil { a.logger.Critical(ctx, "listen for ssh", slog.Error(err)) return @@ -198,6 +202,20 @@ func (a *agent) runTailnet(ctx context.Context, addresses []netip.Addr, derpMap go a.sshServer.HandleConn(conn) } }() + reconnectingPTYListener, err := a.network.Listen("tcp", ":"+strconv.Itoa(tailnetReconnectingPTYPort)) + if err != nil { + a.logger.Critical(ctx, "listen for reconnecting pty", slog.Error(err)) + return + } + go func() { + for { + conn, err := reconnectingPTYListener.Accept() + if err != nil { + return + } + go a.handleReconnectingPTY(ctx, "tailnet", conn) + } + }() } // runCoordinator listens for nodes and updates the self-node as it changes. diff --git a/agent/conn.go b/agent/conn.go index 42b3c41ea45d1..306b37c0c7aaa 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -135,7 +135,6 @@ func (c *WebRTCConn) Close() error { } type TailnetConn struct { - Target netip.Addr *tailnet.Conn } @@ -152,11 +151,11 @@ func (c *TailnetConn) CloseWithError(err error) error { } func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { - return nil, xerrors.New("not implemented") + return c.DialContextTCP(context.Background(), netip.AddrPortFrom(tailnetIP, uint16(tailnetReconnectingPTYPort))) } func (c *TailnetConn) SSH() (net.Conn, error) { - return c.DialContextTCP(context.Background(), netip.AddrPortFrom(c.Target, 12212)) + return c.DialContextTCP(context.Background(), netip.AddrPortFrom(tailnetIP, uint16(tailnetSSHPort))) } // SSHClient calls SSH to create a client that uses a weak cipher @@ -181,5 +180,5 @@ func (c *TailnetConn) SSHClient() (*ssh.Client, error) { func (c *TailnetConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { _, rawPort, _ := net.SplitHostPort(addr) port, _ := strconv.Atoi(rawPort) - return c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(c.Target, uint16(port))) + return c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(tailnetIP, uint16(port))) } diff --git a/cli/root.go b/cli/root.go index e63a308712f02..51837d49bca8b 100644 --- a/cli/root.go +++ b/cli/root.go @@ -133,7 +133,6 @@ func Root() *cobra.Command { update(), users(), versionCmd(), - wireguardPortForward(), workspaceAgent(), ) diff --git a/cli/server.go b/cli/server.go index 86016d9c9bd7a..b3e01cb0412ea 100644 --- a/cli/server.go +++ b/cli/server.go @@ -41,6 +41,7 @@ import ( "golang.org/x/xerrors" "google.golang.org/api/idtoken" "google.golang.org/api/option" + "tailscale.com/tailcfg" "cdr.dev/slog" "cdr.dev/slog/sloggers/sloghuman" @@ -94,6 +95,7 @@ func server() *cobra.Command { oidcEmailDomain string oidcIssuerURL string oidcScopes []string + tailscaleEnable bool telemetryEnable bool telemetryURL string tlsCertFile string @@ -231,6 +233,17 @@ func server() *cobra.Command { if err != nil { return xerrors.Errorf("parse URL: %w", err) } + accessURLPortRaw := accessURLParsed.Port() + if accessURLPortRaw == "" { + accessURLPortRaw = "80" + if accessURLParsed.Scheme == "https" { + accessURLPortRaw = "443" + } + } + accessURLPort, err := strconv.Atoi(accessURLPortRaw) + if err != nil { + return xerrors.Errorf("parse access URL port: %w", err) + } // Warn the user if the access URL appears to be a loopback address. isLocal, err := isLocalURL(ctx, accessURLParsed) @@ -272,10 +285,61 @@ func server() *cobra.Command { }) } options := &coderd.Options{ - AccessURL: accessURLParsed, - ICEServers: iceServers, - Logger: logger.Named("coderd"), - Database: databasefake.New(), + AccessURL: accessURLParsed, + ICEServers: iceServers, + Logger: logger.Named("coderd"), + Database: databasefake.New(), + DERPMap: &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "coder", + RegionName: "Coder", + Nodes: []*tailcfg.DERPNode{{ + Name: "1a", + RegionID: 1, + STUNOnly: true, + HostName: "stun.l.google.com", + STUNPort: 19302, + }, { + Name: "1b", + RegionID: 1, + HostName: accessURLParsed.Hostname(), + DERPPort: accessURLPort, + STUNPort: -1, + HTTPForTests: accessURLParsed.Scheme == "http", + }}, + }, + 2: { + RegionID: 2, + RegionCode: "nyc", + RegionName: "New York City", + Nodes: []*tailcfg.DERPNode{ + { + Name: "2c", + RegionID: 2, + HostName: "derp1c.tailscale.com", + IPv4: "104.248.8.210", + IPv6: "2604:a880:800:10::7a0:e001", + }, + }, + }, + 3: { + RegionID: 3, + RegionCode: "sin", + RegionName: "Singapore", + Nodes: []*tailcfg.DERPNode{ + { + Name: "3a", + RegionID: 3, + HostName: "derp3.tailscale.com", + IPv4: "68.183.179.66", + IPv6: "2400:6180:0:d1::67d:8001", + }, + }, + }, + }, + }, Pubsub: database.NewPubsubInMemory(), CacheDir: cacheDir, GoogleTokenValidator: googleTokenValidator, @@ -710,6 +774,9 @@ func server() *cobra.Command { "Specifies an issuer URL to use for OIDC.") cliflag.StringArrayVarP(root.Flags(), &oidcScopes, "oidc-scopes", "", "CODER_OIDC_SCOPES", []string{oidc.ScopeOpenID, "profile", "email"}, "Specifies scopes to grant when authenticating with OIDC.") + cliflag.BoolVarP(root.Flags(), &tailscaleEnable, "tailscale", "", "CODER_TAILSCALE", false, + "Specifies whether Tailscale networking is used for web applications and terminals.") + _ = root.Flags().MarkHidden("tailscale") enableTelemetryByDefault := !isTest() cliflag.BoolVarP(root.Flags(), &telemetryEnable, "telemetry", "", "CODER_TELEMETRY", enableTelemetryByDefault, "Specifies whether telemetry is enabled or not. Coder collects anonymized usage data to help improve our product.") cliflag.StringVarP(root.Flags(), &telemetryURL, "telemetry-url", "", "CODER_TELEMETRY_URL", "https://telemetry.coder.com", "Specifies a URL to send telemetry to.") diff --git a/cli/ssh.go b/cli/ssh.go index 29a421c69301b..61602137e0208 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -19,15 +19,17 @@ import ( gosshagent "golang.org/x/crypto/ssh/agent" "golang.org/x/term" "golang.org/x/xerrors" - tslogger "tailscale.com/types/logger" + "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" + + "github.com/coder/coder/agent" "github.com/coder/coder/cli/cliflag" "github.com/coder/coder/cli/cliui" "github.com/coder/coder/coderd/autobuild/notify" "github.com/coder/coder/coderd/util/ptr" "github.com/coder/coder/codersdk" "github.com/coder/coder/cryptorand" - "github.com/coder/coder/peer/peerwg" ) var workspacePollInterval = time.Minute @@ -85,86 +87,35 @@ func ssh() *cobra.Command { return xerrors.Errorf("await agent: %w", err) } - var newSSHClient func() (*gossh.Client, error) - + var conn agent.Conn if !wireguard { - conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil) - if err != nil { - return err - } - defer conn.Close() - - stopPolling := tryPollWorkspaceAutostop(ctx, client, workspace) - defer stopPolling() + conn, err = client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil) + } else { + conn, err = client.DialWorkspaceAgentTailnet(ctx, slog.Make(sloghuman.Sink(cmd.ErrOrStderr())).Leveled(slog.LevelDebug), workspaceAgent.ID) + } + if err != nil { + return err + } + defer conn.Close() - if stdio { - rawSSH, err := conn.SSH() - if err != nil { - return err - } - defer rawSSH.Close() + stopPolling := tryPollWorkspaceAutostop(ctx, client, workspace) + defer stopPolling() - go func() { - _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) - }() - _, _ = io.Copy(rawSSH, cmd.InOrStdin()) - return nil + if stdio { + rawSSH, err := conn.SSH() + if err != nil { + return err } + defer rawSSH.Close() - newSSHClient = conn.SSHClient - } else { - // TODO: more granual control of Tailscale logging. - peerwg.Logf = tslogger.Discard //nolint - - // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) - // wgn, err := peerwg.New( - // slog.Make(sloghuman.Sink(os.Stderr)), - // []netip.Prefix{netip.PrefixFrom(ipv6, 128)}, - // ) - // if err != nil { - // return xerrors.Errorf("create wireguard network: %w", err) - // } - - // err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ - // Recipient: workspaceAgent.ID, - // NodePublicKey: wgn.NodePrivateKey.Public(), - // DiscoPublicKey: wgn.DiscoPublicKey, - // IPv6: ipv6, - // }) - // if err != nil { - // return xerrors.Errorf("post wireguard peer: %w", err) - // } - - // err = wgn.AddPeer(peerwg.Handshake{ - // Recipient: workspaceAgent.ID, - // DiscoPublicKey: workspaceAgent.DiscoPublicKey, - // NodePublicKey: workspaceAgent.NodePublicKey, - // IPv6: workspaceAgent.IPAddresses[0], // TODO: fix? - // }) - // if err != nil { - // return xerrors.Errorf("add workspace agent as peer: %w", err) - // } - - // if stdio { - // rawSSH, err := wgn.SSH(cmd.Context(), workspaceAgent.IPAddresses[0]) // TODO: fix? - // if err != nil { - // return err - // } - // defer rawSSH.Close() - - // go func() { - // _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) - // }() - // _, _ = io.Copy(rawSSH, cmd.InOrStdin()) - // return nil - // } - - // newSSHClient = func() (*gossh.Client, error) { - // return wgn.SSHClient(ctx, workspaceAgent.IPAddresses[0]) // TODO: fix? - // } + go func() { + _, _ = io.Copy(cmd.OutOrStdout(), rawSSH) + }() + _, _ = io.Copy(rawSSH, cmd.InOrStdin()) + return nil } - sshClient, err := newSSHClient() + sshClient, err := conn.SSHClient() if err != nil { return err } diff --git a/cli/wireguardtunnel.go b/cli/wireguardtunnel.go deleted file mode 100644 index 31d2c46913d00..0000000000000 --- a/cli/wireguardtunnel.go +++ /dev/null @@ -1,257 +0,0 @@ -package cli - -import ( - "context" - "fmt" - "net" - "net/netip" - "strconv" - "sync" - - "github.com/pion/udp" - "github.com/spf13/cobra" - "golang.org/x/xerrors" - - coderagent "github.com/coder/coder/agent" - "github.com/coder/coder/cli/cliui" - "github.com/coder/coder/codersdk" - "github.com/coder/coder/peer/peerwg" -) - -func wireguardPortForward() *cobra.Command { - var ( - tcpForwards []string // : - udpForwards []string // : - // TODO: unix support - // unixForwards []string // : OR : - ) - cmd := &cobra.Command{ - Use: "wireguard-port-forward ", - Aliases: []string{"wireguard-tunnel"}, - Args: cobra.ExactArgs(1), - // Hide all wireguard commands for now while we test! - Hidden: true, - Example: formatExamples( - example{ - Description: "Port forward a single TCP port from 1234 in the workspace to port 5678 on your local machine", - Command: "coder wireguard-port-forward --tcp 5678:1234", - }, - example{ - Description: "Port forward a single UDP port from port 9000 to port 9000 on your local machine", - Command: "coder wireguard-port-forward --udp 9000", - }, - example{ - Description: "Port forward multiple TCP ports and a UDP port", - Command: "coder wireguard-port-forward --tcp 8080:8080 --tcp 9000:3000 --udp 5353:53", - }, - ), - RunE: func(cmd *cobra.Command, args []string) error { - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - specs, err := parsePortForwards(tcpForwards, nil, nil) - if err != nil { - return xerrors.Errorf("parse port-forward specs: %w", err) - } - if len(specs) == 0 { - err = cmd.Help() - if err != nil { - return xerrors.Errorf("generate help output: %w", err) - } - return xerrors.New("no port-forwards requested") - } - - client, err := createClient(cmd) - if err != nil { - return err - } - - workspace, workspaceAgent, err := getWorkspaceAndAgent(ctx, cmd, client, codersdk.Me, args[0], false) - if err != nil { - return err - } - if workspace.LatestBuild.Transition != codersdk.WorkspaceTransitionStart { - return xerrors.New("workspace must be in start transition to port-forward") - } - if workspace.LatestBuild.Job.CompletedAt == nil { - err = cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID, workspace.CreatedAt) - if err != nil { - return err - } - } - - err = cliui.Agent(ctx, cmd.ErrOrStderr(), cliui.AgentOptions{ - WorkspaceName: workspace.Name, - Fetch: func(ctx context.Context) (codersdk.WorkspaceAgent, error) { - return client.WorkspaceAgent(ctx, workspaceAgent.ID) - }, - }) - if err != nil { - return xerrors.Errorf("await agent: %w", err) - } - - // ipv6 := peerwg.UUIDToNetaddr(uuid.New()) - // wgn, err := peerwg.New( - // slog.Make(sloghuman.Sink(os.Stderr)), - // []netip.Prefix{netip.PrefixFrom(ipv6, 128)}, - // ) - // if err != nil { - // return xerrors.Errorf("create wireguard network: %w", err) - // } - - // err = client.PostWireguardPeer(cmd.Context(), workspace.ID, peerwg.Handshake{ - // Recipient: workspaceAgent.ID, - // NodePublicKey: wgn.NodePrivateKey.Public(), - // DiscoPublicKey: wgn.DiscoPublicKey, - // IPv6: ipv6, - // }) - // if err != nil { - // return xerrors.Errorf("post wireguard peer: %w", err) - // } - - // err = wgn.AddPeer(peerwg.Handshake{ - // Recipient: workspaceAgent.ID, - // DiscoPublicKey: workspaceAgent.DiscoPublicKey, - // NodePublicKey: workspaceAgent.WireguardPublicKey, - // IPv6: workspaceAgent.IPv6.IP(), - // }) - // if err != nil { - // return xerrors.Errorf("add workspace agent as peer: %w", err) - // } - - // // Start all listeners. - // var ( - // ctx, cancel = context.WithCancel(cmd.Context()) - // wg = new(sync.WaitGroup) - // listeners = make([]net.Listener, len(specs)) - // closeAllListeners = func() { - // for _, l := range listeners { - // if l == nil { - // continue - // } - // _ = l.Close() - // } - // } - // ) - // defer cancel() - // for i, spec := range specs { - // l, err := listenAndPortForwardWireguard(ctx, cmd, wgn, wg, spec, workspaceAgent.IPv6.IP()) - // if err != nil { - // closeAllListeners() - // return err - // } - // listeners[i] = l - // } - - // // Wait for the context to be canceled or for a signal and close - // // all listeners. - // var closeErr error - // go func() { - // sigs := make(chan os.Signal, 1) - // signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - // select { - // case <-ctx.Done(): - // closeErr = ctx.Err() - // case <-sigs: - // _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Received signal, closing all listeners and active connections") - // closeErr = xerrors.New("signal received") - // } - - // cancel() - // closeAllListeners() - // }() - - // _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Ready!") - // wg.Wait() - // return closeErr - return nil - }, - } - - cmd.Flags().StringArrayVarP(&tcpForwards, "tcp", "p", []string{}, "Forward a TCP port from the workspace to the local machine") - cmd.Flags().StringArrayVar(&udpForwards, "udp", []string{}, "Forward a UDP port from the workspace to the local machine. The UDP connection has TCP-like semantics to support stateful UDP protocols") - // cmd.Flags().StringArrayVar(&unixForwards, "unix", []string{}, "Forward a Unix socket in the workspace to a local Unix socket or TCP port") - - return cmd -} - -//nolint:unused,deadcode -func listenAndPortForwardWireguard(ctx context.Context, cmd *cobra.Command, - wgn *peerwg.Network, - wg *sync.WaitGroup, - spec portForwardSpec, - agentIP netip.Addr, -) (net.Listener, error) { - _, _ = fmt.Fprintf(cmd.OutOrStderr(), "Forwarding '%v://%v' locally to '%v://%v' in the workspace\n", spec.listenNetwork, spec.listenAddress, spec.dialNetwork, spec.dialAddress) - - var ( - l net.Listener - err error - ) - switch spec.listenNetwork { - case "tcp": - l, err = net.Listen(spec.listenNetwork, spec.listenAddress) - case "udp": - var host, port string - host, port, err = net.SplitHostPort(spec.listenAddress) - if err != nil { - return nil, xerrors.Errorf("split %q: %w", spec.listenAddress, err) - } - - var portInt int - portInt, err = strconv.Atoi(port) - if err != nil { - return nil, xerrors.Errorf("parse port %v from %q as int: %w", port, spec.listenAddress, err) - } - - l, err = udp.Listen(spec.listenNetwork, &net.UDPAddr{ - IP: net.ParseIP(host), - Port: portInt, - }) - // case "unix": - // l, err = net.Listen(spec.listenNetwork, spec.listenAddress) - default: - return nil, xerrors.Errorf("unknown listen network %q", spec.listenNetwork) - } - if err != nil { - return nil, xerrors.Errorf("listen '%v://%v': %w", spec.listenNetwork, spec.listenAddress, err) - } - - wg.Add(1) - go func(spec portForwardSpec) { - defer wg.Done() - for { - netConn, err := l.Accept() - if err != nil { - _, _ = fmt.Fprintf(cmd.OutOrStderr(), "Error accepting connection from '%v://%v': %+v\n", spec.listenNetwork, spec.listenAddress, err) - _, _ = fmt.Fprintln(cmd.OutOrStderr(), "Killing listener") - return - } - - go func(netConn net.Conn) { - defer netConn.Close() - - port := netip.MustParseAddrPort(spec.dialAddress) - ipPort := netip.AddrPortFrom(agentIP, port.Port()) - - var remoteConn net.Conn - switch spec.dialNetwork { - case "tcp": - remoteConn, err = wgn.Netstack.DialContextTCP(ctx, ipPort) - case "udp": - remoteConn, err = wgn.Netstack.DialContextUDP(ctx, ipPort) - } - if err != nil { - _, _ = fmt.Fprintf(cmd.OutOrStderr(), "Failed to dial '%v://%v' in workspace: %s\n", spec.dialNetwork, spec.dialAddress, err) - return - } - defer remoteConn.Close() - - coderagent.Bicopy(ctx, netConn, remoteConn) - }(netConn) - } - }(spec) - - return l, nil -} diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index e8ead0734272a..4791c5f56a75b 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -1689,7 +1689,6 @@ func (q *fakeQuerier) InsertWorkspaceAgent(_ context.Context, arg database.Inser StartupScript: arg.StartupScript, InstanceMetadata: arg.InstanceMetadata, ResourceMetadata: arg.ResourceMetadata, - IPAddresses: arg.IPAddresses, } q.provisionerJobAgents = append(q.provisionerJobAgents, agent) @@ -2003,26 +2002,6 @@ func (q *fakeQuerier) UpdateWorkspaceAgentConnectionByID(_ context.Context, arg return sql.ErrNoRows } -func (q *fakeQuerier) UpdateWorkspaceAgentNetworkByID(_ context.Context, arg database.UpdateWorkspaceAgentNetworkByIDParams) error { - q.mutex.Lock() - defer q.mutex.Unlock() - - for index, agent := range q.provisionerJobAgents { - if agent.ID != arg.ID { - continue - } - - agent.DiscoPublicKey = arg.DiscoPublicKey - agent.NodePublicKey = arg.NodePublicKey - agent.PreferredDERP = arg.PreferredDERP - agent.DERPLatency = arg.DERPLatency - agent.UpdatedAt = database.Now() - q.provisionerJobAgents[index] = agent - return nil - } - return sql.ErrNoRows -} - func (q *fakeQuerier) UpdateProvisionerJobByID(_ context.Context, arg database.UpdateProvisionerJobByIDParams) error { q.mutex.Lock() defer q.mutex.Unlock() diff --git a/coderd/database/dbtypes/dbtypes.go b/coderd/database/dbtypes/dbtypes.go deleted file mode 100644 index a158587269476..0000000000000 --- a/coderd/database/dbtypes/dbtypes.go +++ /dev/null @@ -1,6 +0,0 @@ -package dbtypes - -// DERPLatency represents a KV mapping of latency to DERP servers. -// This type is only used for generation. sqlc doesn't support -// complex Go types. -type DERPLatency map[string]float64 diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index e5fd492a79942..d5f643a5c66d8 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -295,12 +295,7 @@ CREATE TABLE workspace_agents ( startup_script character varying(65534), instance_metadata jsonb, resource_metadata jsonb, - directory character varying(4096) DEFAULT ''::character varying NOT NULL, - ip_addresses inet[] DEFAULT ARRAY[]::inet[] NOT NULL, - node_public_key character varying(128), - disco_public_key character varying(128), - preferred_derp integer DEFAULT 0 NOT NULL, - derp_latency jsonb DEFAULT '{}'::jsonb NOT NULL + directory character varying(4096) DEFAULT ''::character varying NOT NULL ); CREATE TABLE workspace_apps ( diff --git a/coderd/database/migrations/000029_tailnet.up.sql b/coderd/database/migrations/000029_tailnet.up.sql deleted file mode 100644 index 2b03666d713bd..0000000000000 --- a/coderd/database/migrations/000029_tailnet.up.sql +++ /dev/null @@ -1,8 +0,0 @@ -ALTER TABLE workspace_agents DROP COLUMN wireguard_node_ipv6; -ALTER TABLE workspace_agents DROP COLUMN wireguard_node_public_key; -ALTER TABLE workspace_agents DROP COLUMN wireguard_disco_public_key; -ALTER TABLE workspace_agents ADD COLUMN ip_addresses inet[] NOT NULL DEFAULT array[]::inet[]; -ALTER TABLE workspace_agents ADD COLUMN node_public_key varchar(128); -ALTER TABLE workspace_agents ADD COLUMN disco_public_key varchar(128); -ALTER TABLE workspace_agents ADD COLUMN preferred_derp integer NOT NULL DEFAULT 0; -ALTER TABLE workspace_agents ADD COLUMN derp_latency jsonb NOT NULL DEFAULT '{}'; diff --git a/coderd/database/migrations/000029_tailnet.down.sql b/coderd/database/migrations/000034_tailnet.down.sql similarity index 100% rename from coderd/database/migrations/000029_tailnet.down.sql rename to coderd/database/migrations/000034_tailnet.down.sql diff --git a/coderd/database/migrations/000034_tailnet.up.sql b/coderd/database/migrations/000034_tailnet.up.sql new file mode 100644 index 0000000000000..bcda153ee80f1 --- /dev/null +++ b/coderd/database/migrations/000034_tailnet.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE workspace_agents DROP COLUMN wireguard_node_ipv6; +ALTER TABLE workspace_agents DROP COLUMN wireguard_node_public_key; +ALTER TABLE workspace_agents DROP COLUMN wireguard_disco_public_key; diff --git a/coderd/database/models.go b/coderd/database/models.go index c3e8c5de16845..e6af18ee80eea 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -10,7 +10,6 @@ import ( "fmt" "time" - "github.com/coder/coder/coderd/database/dbtypes" "github.com/google/uuid" "github.com/tabbed/pqtype" ) @@ -524,11 +523,6 @@ type WorkspaceAgent struct { InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` Directory string `db:"directory" json:"directory"` - IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` - NodePublicKey sql.NullString `db:"node_public_key" json:"node_public_key"` - DiscoPublicKey sql.NullString `db:"disco_public_key" json:"disco_public_key"` - PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` - DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` } type WorkspaceApp struct { diff --git a/coderd/database/postgres/postgres.go b/coderd/database/postgres/postgres.go index a3277ba13050e..a5cb6a39bf787 100644 --- a/coderd/database/postgres/postgres.go +++ b/coderd/database/postgres/postgres.go @@ -76,7 +76,7 @@ func Open() (string, func(), error) { resource, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "postgres", - Tag: "11", + Tag: "13", Env: []string{ "POSTGRES_PASSWORD=postgres", "POSTGRES_USER=postgres", diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 6d762d0f54739..99811074b89c1 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -131,7 +131,6 @@ type querier interface { UpdateUserRoles(ctx context.Context, arg UpdateUserRolesParams) (User, error) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusParams) (User, error) UpdateWorkspaceAgentConnectionByID(ctx context.Context, arg UpdateWorkspaceAgentConnectionByIDParams) error - UpdateWorkspaceAgentNetworkByID(ctx context.Context, arg UpdateWorkspaceAgentNetworkByIDParams) error UpdateWorkspaceAutostart(ctx context.Context, arg UpdateWorkspaceAutostartParams) error UpdateWorkspaceBuildByID(ctx context.Context, arg UpdateWorkspaceBuildByIDParams) error UpdateWorkspaceDeletedByID(ctx context.Context, arg UpdateWorkspaceDeletedByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 0e8ab28fe4c80..4fc96466b8cef 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -10,7 +10,6 @@ import ( "encoding/json" "time" - "github.com/coder/coder/coderd/database/dbtypes" "github.com/google/uuid" "github.com/lib/pq" "github.com/tabbed/pqtype" @@ -2876,7 +2875,7 @@ func (q *sqlQuerier) UpdateUserStatus(ctx context.Context, arg UpdateUserStatusP const getWorkspaceAgentByAuthToken = `-- name: GetWorkspaceAgentByAuthToken :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE @@ -2906,18 +2905,13 @@ func (q *sqlQuerier) GetWorkspaceAgentByAuthToken(ctx context.Context, authToken &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ) return i, err } const getWorkspaceAgentByID = `-- name: GetWorkspaceAgentByID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE @@ -2945,18 +2939,13 @@ func (q *sqlQuerier) GetWorkspaceAgentByID(ctx context.Context, id uuid.UUID) (W &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ) return i, err } const getWorkspaceAgentByInstanceID = `-- name: GetWorkspaceAgentByInstanceID :one SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE @@ -2986,18 +2975,13 @@ func (q *sqlQuerier) GetWorkspaceAgentByInstanceID(ctx context.Context, authInst &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ) return i, err } const getWorkspaceAgentsByResourceIDs = `-- name: GetWorkspaceAgentsByResourceIDs :many SELECT - id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency + id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE @@ -3031,11 +3015,6 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ); err != nil { return nil, err } @@ -3051,7 +3030,7 @@ func (q *sqlQuerier) GetWorkspaceAgentsByResourceIDs(ctx context.Context, ids [] } const getWorkspaceAgentsCreatedAfter = `-- name: GetWorkspaceAgentsCreatedAfter :many -SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency FROM workspace_agents WHERE created_at > $1 +SELECT id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory FROM workspace_agents WHERE created_at > $1 ` func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceAgent, error) { @@ -3081,11 +3060,6 @@ func (q *sqlQuerier) GetWorkspaceAgentsCreatedAfter(ctx context.Context, created &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ); err != nil { return nil, err } @@ -3116,11 +3090,10 @@ INSERT INTO startup_script, directory, instance_metadata, - resource_metadata, - ip_addresses + resource_metadata ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory, ip_addresses, node_public_key, disco_public_key, preferred_derp, derp_latency + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING id, created_at, updated_at, name, first_connected_at, last_connected_at, disconnected_at, resource_id, auth_token, auth_instance_id, architecture, environment_variables, operating_system, startup_script, instance_metadata, resource_metadata, directory ` type InsertWorkspaceAgentParams struct { @@ -3138,7 +3111,6 @@ type InsertWorkspaceAgentParams struct { Directory string `db:"directory" json:"directory"` InstanceMetadata pqtype.NullRawMessage `db:"instance_metadata" json:"instance_metadata"` ResourceMetadata pqtype.NullRawMessage `db:"resource_metadata" json:"resource_metadata"` - IPAddresses []pqtype.Inet `db:"ip_addresses" json:"ip_addresses"` } func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspaceAgentParams) (WorkspaceAgent, error) { @@ -3157,7 +3129,6 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa arg.Directory, arg.InstanceMetadata, arg.ResourceMetadata, - pq.Array(arg.IPAddresses), ) var i WorkspaceAgent err := row.Scan( @@ -3178,11 +3149,6 @@ func (q *sqlQuerier) InsertWorkspaceAgent(ctx context.Context, arg InsertWorkspa &i.InstanceMetadata, &i.ResourceMetadata, &i.Directory, - pq.Array(&i.IPAddresses), - &i.NodePublicKey, - &i.DiscoPublicKey, - &i.PreferredDERP, - &i.DERPLatency, ) return i, err } @@ -3218,40 +3184,6 @@ func (q *sqlQuerier) UpdateWorkspaceAgentConnectionByID(ctx context.Context, arg return err } -const updateWorkspaceAgentNetworkByID = `-- name: UpdateWorkspaceAgentNetworkByID :exec -UPDATE - workspace_agents -SET - updated_at = $2, - node_public_key = $3, - disco_public_key = $4, - preferred_derp = $5, - derp_latency = $6 -WHERE - id = $1 -` - -type UpdateWorkspaceAgentNetworkByIDParams struct { - ID uuid.UUID `db:"id" json:"id"` - UpdatedAt time.Time `db:"updated_at" json:"updated_at"` - NodePublicKey sql.NullString `db:"node_public_key" json:"node_public_key"` - DiscoPublicKey sql.NullString `db:"disco_public_key" json:"disco_public_key"` - PreferredDERP int32 `db:"preferred_derp" json:"preferred_derp"` - DERPLatency dbtypes.DERPLatency `db:"derp_latency" json:"derp_latency"` -} - -func (q *sqlQuerier) UpdateWorkspaceAgentNetworkByID(ctx context.Context, arg UpdateWorkspaceAgentNetworkByIDParams) error { - _, err := q.db.ExecContext(ctx, updateWorkspaceAgentNetworkByID, - arg.ID, - arg.UpdatedAt, - arg.NodePublicKey, - arg.DiscoPublicKey, - arg.PreferredDERP, - arg.DERPLatency, - ) - return err -} - const getWorkspaceAppByAgentIDAndName = `-- name: GetWorkspaceAppByAgentIDAndName :one SELECT id, created_at, agent_id, name, icon, command, url, relative_path FROM workspace_apps WHERE agent_id = $1 AND name = $2 ` diff --git a/coderd/database/queries/workspaceagents.sql b/coderd/database/queries/workspaceagents.sql index 0e91f1b169687..8cc2f33c1f1f9 100644 --- a/coderd/database/queries/workspaceagents.sql +++ b/coderd/database/queries/workspaceagents.sql @@ -53,11 +53,10 @@ INSERT INTO startup_script, directory, instance_metadata, - resource_metadata, - ip_addresses + resource_metadata ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) RETURNING *; -- name: UpdateWorkspaceAgentConnectionByID :exec UPDATE @@ -69,15 +68,3 @@ SET updated_at = $5 WHERE id = $1; - --- name: UpdateWorkspaceAgentNetworkByID :exec -UPDATE - workspace_agents -SET - updated_at = $2, - node_public_key = $3, - disco_public_key = $4, - preferred_derp = $5, - derp_latency = $6 -WHERE - id = $1; diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 39e5384a38858..526614ae104a9 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -16,10 +16,6 @@ packages: # deleted after generation. output_db_file_name: db_tmp.go -overrides: - - column: workspace_agents.derp_latency - go_type: github.com/coder/coder/coderd/database/dbtypes.DERPLatency - rename: api_key: APIKey login_type_oidc: LoginTypeOIDC @@ -33,5 +29,4 @@ rename: rbac_roles: RBACRoles ip_address: IPAddress ip_addresses: IPAddresses - preferred_derp: PreferredDERP - derp_latency: DERPLatency + diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index 8c52920bf392e..352f3154bc6b8 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "io" - "net" "net/http" "net/url" "reflect" @@ -32,7 +31,6 @@ import ( "github.com/coder/coder/provisionerd/proto" "github.com/coder/coder/provisionersdk" sdkproto "github.com/coder/coder/provisionersdk/proto" - "github.com/coder/coder/tailnet" ) func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { @@ -788,7 +786,6 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. } agentID := uuid.New() - ip := tailnet.IP().As16() dbAgent, err := db.InsertWorkspaceAgent(ctx, database.InsertWorkspaceAgentParams{ ID: agentID, CreatedAt: database.Now(), @@ -805,14 +802,6 @@ func insertWorkspaceResource(ctx context.Context, db database.Store, jobID uuid. String: prAgent.StartupScript, Valid: prAgent.StartupScript != "", }, - // Generate a new random IP! - IPAddresses: []pqtype.Inet{{ - Valid: true, - IPNet: net.IPNet{ - IP: ip[:], - Mask: net.CIDRMask(128, 128), - }, - }}, }) if err != nil { return xerrors.Errorf("insert agent: %w", err) diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 04cbbd4d821df..fad540c5dc2d9 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -264,7 +264,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request, } } - apiAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading job agent.", diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index d645a857c2306..c3a1e017a021b 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -8,7 +8,6 @@ import ( "io" "net" "net/http" - "net/netip" "strconv" "time" @@ -47,7 +46,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) { }) return } - apiAgent, err := convertWorkspaceAgent(workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -71,7 +70,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -118,13 +117,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) { workspaceAgent := httpmw.WorkspaceAgent(r) - ips := make([]netip.Addr, 0) - for _, ip := range workspaceAgent.IPAddresses { - var ipData [16]byte - copy(ipData[:], []byte(ip.IPNet.IP)) - ips = append(ips, netip.AddrFrom16(ipData)) - } - apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -134,7 +127,6 @@ func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) } httpapi.Write(rw, http.StatusOK, agent.Metadata{ - IPAddresses: ips, DERPMap: api.DERPMap, EnvironmentVariables: apiAgent.EnvironmentVariables, StartupScript: apiAgent.StartupScript, @@ -376,7 +368,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -512,21 +504,13 @@ func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Co } func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request) { - workspaceAgent := httpmw.WorkspaceAgentParam(r) workspace := httpmw.WorkspaceParam(r) if !api.Authorize(r, rbac.ActionRead, workspace) { httpapi.ResourceNotFound(rw) return } - ips := make([]netip.Addr, 0) - for _, ip := range workspaceAgent.IPAddresses { - var ipData [16]byte - copy(ipData[:], []byte(ip.IPNet.IP)) - ips = append(ips, netip.AddrFrom16(ipData)) - } httpapi.Write(rw, http.StatusOK, codersdk.WorkspaceAgentConnectionInfo{ - DERPMap: api.DERPMap, - IPAddresses: ips, + DERPMap: api.DERPMap, }) } @@ -628,20 +612,14 @@ func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { return apps } -func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { +func convertWorkspaceAgent(coordinator *tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { var envs map[string]string if dbAgent.EnvironmentVariables.Valid { err := json.Unmarshal(dbAgent.EnvironmentVariables.RawMessage, &envs) if err != nil { - return codersdk.WorkspaceAgent{}, xerrors.Errorf("unmarshal: %w", err) + return codersdk.WorkspaceAgent{}, xerrors.Errorf("unmarshal env vars: %w", err) } } - ips := make([]netip.Addr, 0) - for _, ip := range dbAgent.IPAddresses { - var ipData [16]byte - copy(ipData[:], []byte(ip.IPNet.IP)) - ips = append(ips, netip.AddrFrom16(ipData)) - } workspaceAgent := codersdk.WorkspaceAgent{ ID: dbAgent.ID, CreatedAt: dbAgent.CreatedAt, @@ -655,8 +633,11 @@ func convertWorkspaceAgent(dbAgent database.WorkspaceAgent, apps []codersdk.Work EnvironmentVariables: envs, Directory: dbAgent.Directory, Apps: apps, - PreferredDERP: int(dbAgent.PreferredDERP), - DERPLatency: dbAgent.DERPLatency, + } + node := coordinator.Node(dbAgent.ID) + if node != nil { + workspaceAgent.PreferredDERP = node.PreferredDERP + workspaceAgent.DERPLatency = node.DERPLatency } if dbAgent.FirstConnectedAt.Valid { diff --git a/coderd/workspaceresources.go b/coderd/workspaceresources.go index 61f5ff2f4101e..84a8828d1eaf7 100644 --- a/coderd/workspaceresources.go +++ b/coderd/workspaceresources.go @@ -70,7 +70,7 @@ func (api *API) workspaceResource(rw http.ResponseWriter, r *http.Request) { } } - convertedAgent, err := convertWorkspaceAgent(agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + convertedAgent, err := convertWorkspaceAgent(api.ConnCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index e00c2676e7af5..646fdcb15d260 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -57,8 +57,7 @@ type WorkspaceAgentAuthenticateResponse struct { // WorkspaceAgentConnectionInfo returns required information for establishing // a connection with a workspace. type WorkspaceAgentConnectionInfo struct { - DERPMap *tailcfg.DERPMap `json:"derp_map"` - IPAddresses []netip.Addr `json:"ip_address"` + DERPMap *tailcfg.DERPMap `json:"derp_map"` } // AuthWorkspaceGoogleInstanceIdentity uses the Google Compute Engine Metadata API to @@ -328,6 +327,18 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg if err != nil { return nil, xerrors.Errorf("create tailnet: %w", err) } + go func() { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + logger.Info(ctx, "ipn", slog.F("status", conn.Status())) + } + }() coordinateURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", agentID)) if err != nil { @@ -375,8 +386,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg } }() return &agent.TailnetConn{ - Target: connInfo.IPAddresses[0], - Conn: conn, + Conn: conn, }, nil } diff --git a/peer/peerwg/derp.go b/peer/peerwg/derp.go deleted file mode 100644 index 77aecef6f5e43..0000000000000 --- a/peer/peerwg/derp.go +++ /dev/null @@ -1,73 +0,0 @@ -package peerwg - -import ( - "net" - - "tailscale.com/tailcfg" -) - -// This is currently set to use Tailscale's DERP server in DFW while we build in -// our own support for DERP servers. -var DerpMap = &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, - RegionCode: "goog", - RegionName: "Google", - Avoid: false, - Nodes: []*tailcfg.DERPNode{ - { - Name: "9a", - RegionID: 1, - HostName: "derp9.tailscale.com", - CertName: "", - IPv4: "207.148.3.137", - IPv6: "2001:19f0:6401:1d9c:5400:2ff:feef:bb82", - STUNPort: 0, - STUNOnly: false, - DERPPort: 0, - InsecureForTests: false, - STUNTestIP: "", - }, - // { - // Name: "9c", - // RegionID: 9, - // HostName: "derp9c.tailscale.com", - // CertName: "", - // IPv4: "155.138.243.219", - // IPv6: "2001:19f0:6401:fe7:5400:3ff:fe8d:6d9c", - // STUNPort: 0, - // STUNOnly: false, - // DERPPort: 0, - // InsecureForTests: false, - // STUNTestIP: "", - // }, - // { - // Name: "9b", - // RegionID: 9, - // HostName: "derp9b.tailscale.com", - // CertName: "", - // IPv4: "144.202.67.195", - // IPv6: "2001:19f0:6401:eb5:5400:3ff:fe8d:6d9b", - // STUNPort: 0, - // STUNOnly: false, - // DERPPort: 0, - // InsecureForTests: false, - // STUNTestIP: "", - // }, - { - Name: "goog", - RegionID: 2, - HostName: "stun.l.google.com", - STUNPort: 19302, - STUNOnly: true, - }, - }, - }, - }, - // OmitDefaultRegions: true, -} - -// DefaultDerpHome is the ipv4 representation of a DERP server. The port is the -// DERP id. We only support using DERP 9 for now. -var DefaultDerpHome = net.JoinHostPort(tailcfg.DerpMagicIP, "1") diff --git a/peer/peerwg/handshake.go b/peer/peerwg/handshake.go deleted file mode 100644 index 56340648ae9a4..0000000000000 --- a/peer/peerwg/handshake.go +++ /dev/null @@ -1,94 +0,0 @@ -package peerwg - -import ( - "bytes" - "net/netip" - "strconv" - - "github.com/google/uuid" - "golang.org/x/xerrors" - "tailscale.com/types/key" -) - -const handshakeSeparator byte = '|' - -// Handshake is a message received from a wireguard peer, indicating -// it would like to connect. -type Handshake struct { - // Recipient is the uuid of the agent that the message was intended for. - Recipient uuid.UUID `json:"recipient"` - // DiscoPublicKey is the disco public key of the peer. - DiscoPublicKey key.DiscoPublic `json:"disco"` - // NodePublicKey is the public key of the peer. - NodePublicKey key.NodePublic `json:"public"` - // IPv6 is the IPv6 address of the peer. - IPv6 netip.Addr `json:"ipv6"` -} - -// HandshakeRecipientHint parses the first part of a serialized -// Handshake to quickly determine if the message is meant for the -// provided recipient. -func HandshakeRecipientHint(agentID []byte, msg []byte) (bool, error) { - idx := bytes.Index(msg, []byte{handshakeSeparator}) - if idx == -1 { - return false, xerrors.Errorf("invalid peer message, no separator") - } - - return bytes.Equal(agentID, msg[:idx]), nil -} - -func (h *Handshake) UnmarshalText(text []byte) error { - sp := bytes.Split(text, []byte{handshakeSeparator}) - if len(sp) != 4 { - return xerrors.Errorf("expected 4 parts, got %d", len(sp)) - } - - err := h.Recipient.UnmarshalText(sp[0]) - if err != nil { - return xerrors.Errorf("parse recipient: %w", err) - } - - err = h.DiscoPublicKey.UnmarshalText(sp[1]) - if err != nil { - return xerrors.Errorf("parse disco: %w", err) - } - - err = h.NodePublicKey.UnmarshalText(sp[2]) - if err != nil { - return xerrors.Errorf("parse public: %w", err) - } - - h.IPv6, err = netip.ParseAddr(string(sp[3])) - if err != nil { - return xerrors.Errorf("parse ipv6: %w", err) - } - - return nil -} - -func (h Handshake) MarshalText() ([]byte, error) { - const expectedLen = 223 - var buf bytes.Buffer - buf.Grow(expectedLen) - - recp, _ := h.Recipient.MarshalText() - _, _ = buf.Write(recp) - _ = buf.WriteByte(handshakeSeparator) - - disco, _ := h.DiscoPublicKey.MarshalText() - _, _ = buf.Write(disco) - _ = buf.WriteByte(handshakeSeparator) - - pub, _ := h.NodePublicKey.MarshalText() - _, _ = buf.Write(pub) - _ = buf.WriteByte(handshakeSeparator) - - ipv6 := h.IPv6.StringExpanded() - _, _ = buf.WriteString(ipv6) - - // Ensure we're always allocating exactly enough. - if buf.Len() != expectedLen { - panic("buffer length mismatch: want 223, got " + strconv.Itoa(buf.Len())) - } - return buf.Bytes(), nil -} diff --git a/peer/peerwg/ssh.go b/peer/peerwg/ssh.go deleted file mode 100644 index b94545c1e831e..0000000000000 --- a/peer/peerwg/ssh.go +++ /dev/null @@ -1,38 +0,0 @@ -package peerwg - -import ( - "context" - "net" - "net/netip" - - "golang.org/x/crypto/ssh" - "golang.org/x/xerrors" -) - -func (n *Network) SSH(ctx context.Context, ip netip.Addr) (net.Conn, error) { - netConn, err := n.Netstack.DialContextTCP(ctx, netip.AddrPortFrom(ip, 12212)) - if err != nil { - return nil, xerrors.Errorf("dial agent ssh: %w", err) - } - - return netConn, nil -} - -func (n *Network) SSHClient(ctx context.Context, ip netip.Addr) (*ssh.Client, error) { - netConn, err := n.SSH(ctx, ip) - if err != nil { - return nil, xerrors.Errorf("ssh: %w", err) - } - - sshConn, channels, requests, err := ssh.NewClientConn(netConn, "localhost:22", &ssh.ClientConfig{ - // SSH host validation isn't helpful, because obtaining a peer - // connection already signifies user-intent to dial a workspace. - // #nosec - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - }) - if err != nil { - return nil, xerrors.Errorf("new ssh client conn: %w", err) - } - - return ssh.NewClient(sshConn, channels, requests), nil -} diff --git a/peer/peerwg/wireguard.go b/peer/peerwg/wireguard.go deleted file mode 100644 index 6eeaa06a150b8..0000000000000 --- a/peer/peerwg/wireguard.go +++ /dev/null @@ -1,437 +0,0 @@ -package peerwg - -import ( - "context" - "fmt" - "hash/fnv" - "io" - "log" - "net" - "net/netip" - "strconv" - "sync" - "time" - - "github.com/google/uuid" - "github.com/tabbed/pqtype" - "go4.org/netipx" - "golang.org/x/xerrors" - "tailscale.com/ipn/ipnstate" - "tailscale.com/net/dns" - "tailscale.com/net/netns" - "tailscale.com/net/tsdial" - "tailscale.com/tailcfg" - "tailscale.com/types/ipproto" - "tailscale.com/types/key" - tslogger "tailscale.com/types/logger" - "tailscale.com/types/netmap" - "tailscale.com/wgengine" - "tailscale.com/wgengine/filter" - "tailscale.com/wgengine/magicsock" - "tailscale.com/wgengine/monitor" - "tailscale.com/wgengine/netstack" - "tailscale.com/wgengine/router" - "tailscale.com/wgengine/wgcfg/nmcfg" - - "cdr.dev/slog" -) - -var Logf tslogger.Logf = log.Printf - -func init() { - // Globally disable network namespacing. - // All networking happens in userspace. - netns.SetEnabled(false) -} - -func UUIDToInet(uid uuid.UUID) pqtype.Inet { - uid = privateUUID(uid) - - return pqtype.Inet{ - Valid: true, - IPNet: net.IPNet{ - IP: uid[:], - Mask: net.CIDRMask(128, 128), - }, - } -} - -func UUIDToNetaddr(uid uuid.UUID) netip.Addr { - return netip.AddrFrom16(uuid.New()) -} - -// privateUUID sets the uid to have the tailscale private ipv6 prefix. -func privateUUID(uid uuid.UUID) uuid.UUID { - // fd7a:115c:a1e0 - uid[0] = 0xfd - uid[1] = 0x7a - uid[2] = 0x11 - uid[3] = 0x5c - uid[4] = 0xa1 - uid[5] = 0xe0 - return uid -} - -type Network struct { - mu sync.Mutex - logger slog.Logger - - Netstack *netstack.Impl - magicSock *magicsock.Conn - netMap *netmap.NetworkMap - router *router.Config - wgEngine wgengine.Engine - - // listeners is a map of listening sockets that will be forwarded traffic - // from the wireguard interface. - listeners map[listenKey]*listener - - DiscoPublicKey key.DiscoPublic - NodePrivateKey key.NodePrivate -} - -// New constructs a Wireguard network that filters traffic -// to destinations matching the addresses provided. -func New(logger slog.Logger, addresses []netip.Prefix) (*Network, error) { - nodePrivateKey := key.NewNode() - nodePublicKey := nodePrivateKey.Public() - // id, stableID := nodeIDs(nodePublicKey) - - netMap := &netmap.NetworkMap{ - NodeKey: nodePublicKey, - PrivateKey: nodePrivateKey, - Addresses: addresses, - PacketFilter: []filter.Match{{ - // Allow any protocol! - IPProto: []ipproto.Proto{ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, ipproto.SCTP}, - // Allow traffic sourced from anywhere. - Srcs: []netip.Prefix{ - netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), - netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), - }, - // Allow traffic to route anywhere. - Dsts: []filter.NetPortRange{ - { - Net: netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0), - Ports: filter.PortRange{ - First: 0, - Last: 65535, - }, - }, - { - Net: netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0), - Ports: filter.PortRange{ - First: 0, - Last: 65535, - }, - }, - }, - Caps: []filter.CapMatch{}, - }}, - } - netMap.SelfNode = &tailcfg.Node{ - Key: nodePublicKey, - Addresses: addresses, - } - - wgMonitor, err := monitor.New(Logf) - if err != nil { - return nil, xerrors.Errorf("create link monitor: %w", err) - } - - dialer := &tsdial.Dialer{ - Logf: Logf, - } - // Create a wireguard engine in userspace. - engine, err := wgengine.NewUserspaceEngine(Logf, wgengine.Config{ - LinkMonitor: wgMonitor, - Dialer: dialer, - }) - if err != nil { - return nil, xerrors.Errorf("create wgengine: %w", err) - } - dialer.UseNetstackForIP = func(ip netip.Addr) bool { - _, ok := engine.PeerForIP(ip) - return ok - } - - // This is taken from Tailscale: - // https://github.com/tailscale/tailscale/blob/0f05b2c13ff0c305aa7a1655fa9c17ed969d65be/tsnet/tsnet.go#L247-L255 - // nolint - tunDev, magicConn, dnsManager, ok := engine.(wgengine.InternalsGetter).GetInternals() - if !ok { - return nil, xerrors.New("could not get wgengine internals") - } - - // Update the keys for the magic connection! - err = magicConn.SetPrivateKey(nodePrivateKey) - if err != nil { - return nil, xerrors.Errorf("set node private key: %w", err) - } - netMap.SelfNode.DiscoKey = magicConn.DiscoPublicKey() - - // Create the networking stack. - // This is called to route connections. - netStack, err := netstack.Create(Logf, tunDev, engine, magicConn, dialer, dnsManager) - if err != nil { - return nil, xerrors.Errorf("create netstack: %w", err) - } - dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) { - return netStack.DialContextTCP(ctx, dst) - } - netStack.ProcessLocalIPs = true - err = netStack.Start() - if err != nil { - return nil, xerrors.Errorf("start netstack: %w", err) - } - engine = wgengine.NewWatchdog(engine) - - // Update the wireguard configuration to allow traffic to flow. - cfg, err := nmcfg.WGCfg(netMap, Logf, netmap.AllowSingleHosts, "") - if err != nil { - return nil, xerrors.Errorf("create wgcfg: %w", err) - } - - rtr := &router.Config{ - LocalAddrs: cfg.Addresses, - } - err = engine.Reconfig(cfg, rtr, &dns.Config{}, &tailcfg.Debug{}) - if err != nil { - return nil, xerrors.Errorf("reconfig: %w", err) - } - - engine.SetDERPMap(DerpMap) - netMapCopy := *netMap - netMapCopy.SelfNode = &tailcfg.Node{} - engine.SetNetworkMap(&netMapCopy) - - localIPSet := netipx.IPSetBuilder{} - for _, addr := range netMap.Addresses { - localIPSet.AddPrefix(addr) - } - ips, _ := localIPSet.IPSet() - logIPSet := netipx.IPSetBuilder{} - ipl, _ := logIPSet.IPSet() - engine.SetFilter(filter.New(netMap.PacketFilter, ips, ipl, nil, Logf)) - - wn := &Network{ - logger: logger, - NodePrivateKey: nodePrivateKey, - DiscoPublicKey: magicConn.DiscoPublicKey(), - wgEngine: engine, - Netstack: netStack, - magicSock: magicConn, - netMap: netMap, - router: rtr, - listeners: map[listenKey]*listener{}, - } - netStack.ForwardTCPIn = wn.forwardTCP - - return wn, nil -} - -// forwardTCP handles incoming connections from Wireguard in userspace. -func (n *Network) forwardTCP(conn net.Conn, port uint16) { - n.mu.Lock() - listener, ok := n.listeners[listenKey{"tcp", "", fmt.Sprint(port)}] - n.mu.Unlock() - if !ok { - // No in-memory listener exists, forward to host. - n.forwardTCPToLocalHandler(conn, port) - return - } - - timer := time.NewTimer(time.Second) - defer timer.Stop() - select { - case listener.conn <- conn: - case <-timer.C: - _ = conn.Close() - } -} - -// forwardTCPToLocalHandler forwards the provided net.Conn to the -// matching port bound to localhost. -func (n *Network) forwardTCPToLocalHandler(c net.Conn, port uint16) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - defer c.Close() - - dialAddrStr := net.JoinHostPort("127.0.0.1", strconv.Itoa(int(port))) - var stdDialer net.Dialer - server, err := stdDialer.DialContext(ctx, "tcp", dialAddrStr) - if err != nil { - n.logger.Debug(ctx, "dial local port", slog.F("port", port), slog.Error(err)) - return - } - defer server.Close() - - connClosed := make(chan error, 2) - go func() { - _, err := io.Copy(server, c) - connClosed <- err - }() - go func() { - _, err := io.Copy(c, server) - connClosed <- err - }() - err = <-connClosed - if err != nil { - n.logger.Debug(ctx, "proxy connection closed with error", slog.Error(err)) - } - n.logger.Debug(ctx, "forwarded connection closed", slog.F("local_addr", dialAddrStr)) -} - -// AddPeer allows connections from another Wireguard instance with the -// handshake credentials. -func (n *Network) AddPeer(handshake Handshake) error { - n.mu.Lock() - defer n.mu.Unlock() - - // If the peer already exists in the network map, do nothing. - for _, p := range n.netMap.Peers { - if p.Key == handshake.NodePublicKey { - n.logger.Debug(context.Background(), "peer already in netmap", slog.F("peer", handshake.NodePublicKey.ShortString())) - return nil - } - } - - // The Tailscale engine owns this slice, so we need to copy to make - // modifications. - peers := append(([]*tailcfg.Node)(nil), n.netMap.Peers...) - - // id, stableID := nodeIDs(handshake.NodePublicKey) - peers = append(peers, &tailcfg.Node{ - // ID: id, - // StableID: stableID, - // Name: handshake.NodePublicKey.String() + ".com", - Key: handshake.NodePublicKey, - DiscoKey: handshake.DiscoPublicKey, - Addresses: []netip.Prefix{netip.PrefixFrom(handshake.IPv6, 128)}, - AllowedIPs: []netip.Prefix{netip.PrefixFrom(handshake.IPv6, 128)}, - DERP: DefaultDerpHome, - // Endpoints: []string{DefaultDerpHome}, - }) - - n.netMap.Peers = peers - - cfg, err := nmcfg.WGCfg(n.netMap, Logf, netmap.AllowSingleHosts|netmap.AllowSubnetRoutes, tailcfg.StableNodeID("nBBoJZ5CNTRL")) - if err != nil { - return xerrors.Errorf("create wgcfg: %w", err) - } - - err = n.wgEngine.Reconfig(cfg, n.router, &dns.Config{}, &tailcfg.Debug{}) - if err != nil { - return xerrors.Errorf("reconfig: %w", err) - } - - // Always give the Tailscale engine a copy of our network map. - n.wgEngine.SetNetworkMap(copyNetMap(n.netMap)) - return nil -} - -// Ping sends a discovery ping to the provided peer. -// The peer address must be connected before a successful ping will work. -func (n *Network) Ping(ip netip.Addr) *ipnstate.PingResult { - ch := make(chan *ipnstate.PingResult) - n.wgEngine.Ping(ip, tailcfg.PingDisco, func(pr *ipnstate.PingResult) { - ch <- pr - }) - return <-ch -} - -// Listener returns a net.Listener in userspace that can be used to accept -// connections from the Wireguard network to the specified address. If a -// listener exists for a given address, all connections will be forwarded to the -// listener instead of being routed to the host. -func (n *Network) Listen(network, addr string) (net.Listener, error) { - host, port, err := net.SplitHostPort(addr) - if err != nil { - return nil, xerrors.Errorf("split addr host port: %w", err) - } - - lkey := listenKey{network, host, port} - ln := &listener{ - wn: n, - key: lkey, - addr: addr, - - conn: make(chan net.Conn, 1), - } - - n.mu.Lock() - defer n.mu.Unlock() - - if _, ok := n.listeners[lkey]; ok { - return nil, xerrors.Errorf("listener already open for %s, %s", network, addr) - } - n.listeners[lkey] = ln - - return ln, nil -} - -func (n *Network) Close() error { - // Close all listeners. - for _, l := range n.listeners { - _ = l.Close() - } - - // Close the Wireguard netstack and engine. - _ = n.Netstack.Close() - n.wgEngine.Close() - - return nil -} - -type listenKey struct { - network string - host string - port string -} - -type listener struct { - wn *Network - key listenKey - addr string - conn chan net.Conn -} - -func (ln *listener) Accept() (net.Conn, error) { - c, ok := <-ln.conn - if !ok { - return nil, xerrors.Errorf("tsnet: %w", net.ErrClosed) - } - return c, nil -} - -func (ln *listener) Addr() net.Addr { return addr{ln} } -func (ln *listener) Close() error { - ln.wn.mu.Lock() - defer ln.wn.mu.Unlock() - - if v, ok := ln.wn.listeners[ln.key]; ok && v == ln { - delete(ln.wn.listeners, ln.key) - close(ln.conn) - } - - return nil -} - -type addr struct{ ln *listener } - -func (a addr) Network() string { return a.ln.key.network } -func (a addr) String() string { return a.ln.addr } - -// nodeIDs generates Tailscale node IDs for the provided public key. -func nodeIDs(public key.NodePublic) (tailcfg.NodeID, tailcfg.StableNodeID) { - idhash := fnv.New64() - pub, _ := public.MarshalText() - _, _ = idhash.Write(pub) - - return tailcfg.NodeID(idhash.Sum64()), tailcfg.StableNodeID(pub) -} - -func copyNetMap(nm *netmap.NetworkMap) *netmap.NetworkMap { - nmCopy := *nm - return &nmCopy -} diff --git a/peer/peerwg/wireguard_test.go b/peer/peerwg/wireguard_test.go deleted file mode 100644 index a08fcf17dd396..0000000000000 --- a/peer/peerwg/wireguard_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package peerwg_test - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "net/http/httptest" - "net/netip" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/require" - "tailscale.com/derp" - "tailscale.com/derp/derphttp" - "tailscale.com/net/stun/stuntest" - "tailscale.com/tailcfg" - "tailscale.com/types/key" - "tailscale.com/types/logger" - "tailscale.com/types/nettype" - - "cdr.dev/slog/sloggers/slogtest" - "github.com/coder/coder/peer/peerwg" -) - -func TestConnect(t *testing.T) { - t.Parallel() - - logger := slogtest.Make(t, nil) - c1IPv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn1, err := peerwg.New(logger.Named("c1"), []netip.Prefix{ - netip.PrefixFrom(c1IPv6, 128), - }) - require.NoError(t, err) - - c2IPv6 := peerwg.UUIDToNetaddr(uuid.New()) - wgn2, err := peerwg.New(logger.Named("c2"), []netip.Prefix{ - netip.PrefixFrom(c2IPv6, 128), - }) - require.NoError(t, err) - err = wgn1.AddPeer(peerwg.Handshake{ - DiscoPublicKey: wgn2.DiscoPublicKey, - NodePublicKey: wgn2.NodePrivateKey.Public(), - IPv6: c2IPv6, - }) - require.NoError(t, err) - - conn := make(chan struct{}) - go func() { - listener, err := wgn1.Listen("tcp", ":35565") - require.NoError(t, err) - conn <- struct{}{} - fmt.Printf("Started listening...\n") - _, err = listener.Accept() - fmt.Printf("Got connection!\n") - require.NoError(t, err) - conn <- struct{}{} - }() - - err = wgn2.AddPeer(peerwg.Handshake{ - DiscoPublicKey: wgn1.DiscoPublicKey, - NodePublicKey: wgn1.NodePrivateKey.Public(), - IPv6: c1IPv6, - }) - require.NoError(t, err) - <-conn - time.Sleep(100 * time.Millisecond) - fmt.Printf("\n\n\n\n\nDIALING TCP\n\n\n\n\n") - _, err = wgn2.Netstack.DialContextTCP(context.Background(), netip.AddrPortFrom(c1IPv6, 35565)) - require.NoError(t, err) - <-conn -} - -func runDERPAndStun(t *testing.T, logf logger.Logf, l nettype.PacketListener, stunIP netip.Addr) (derpMap *tailcfg.DERPMap, cleanup func()) { - d := derp.NewServer(key.NewNode(), logf) - - httpsrv := httptest.NewUnstartedServer(derphttp.Handler(d)) - httpsrv.Config.ErrorLog = logger.StdLogger(logf) - httpsrv.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) - httpsrv.StartTLS() - - stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, l) - - m := &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, - RegionCode: "test", - Nodes: []*tailcfg.DERPNode{ - { - Name: "t1", - RegionID: 1, - HostName: "test-node.unused", - IPv4: "127.0.0.1", - IPv6: "none", - STUNPort: stunAddr.Port, - DERPPort: httpsrv.Listener.Addr().(*net.TCPAddr).Port, - InsecureForTests: true, - STUNTestIP: stunIP.String(), - }, - }, - }, - }, - } - - cleanup = func() { - httpsrv.CloseClientConnections() - httpsrv.Close() - d.Close() - stunCleanup() - } - - return m, cleanup -} diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 1c50f8a3b6d22..f696f7108161a 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -427,15 +427,6 @@ export interface WorkspaceAgent { readonly startup_script?: string readonly directory?: string readonly apps: WorkspaceApp[] - // Named type "net/netip.Addr" unknown, using "any" - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly ip_addresses: any[] - // Named type "tailscale.com/types/key.NodePublic" unknown, using "any" - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly node_public_key: any - // Named type "tailscale.com/types/key.DiscoPublic" unknown, using "any" - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly disco_public_key: any readonly preferred_derp: number readonly latency: Record } @@ -445,6 +436,13 @@ export interface WorkspaceAgentAuthenticateResponse { readonly session_token: string } +// From codersdk/workspaceagents.go +export interface WorkspaceAgentConnectionInfo { + // Named type "tailscale.com/tailcfg.DERPMap" unknown, using "any" + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readonly derp_map?: any +} + // From codersdk/workspaceresources.go export interface WorkspaceAgentInstanceMetadata { readonly jail_orchestrator: string diff --git a/tailnet/conn.go b/tailnet/conn.go index 03bbde111beab..11a2fff15ad58 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -232,23 +232,46 @@ type Conn struct { wireguardRouter *router.Config wireguardEngine wgengine.Engine listeners map[listenKey]*listener + + // It's only possible to store these values via status functions, + // so the values must be stored for retrieval later on. + lastEndpoints []string + lastPreferredDERP int + lastDERPLatency map[string]float64 } // SetNodeCallback is triggered when a network change occurs and peer // renegotiation may be required. Clients should constantly be emitting // node changes. func (c *Conn) SetNodeCallback(callback func(node *Node)) { - c.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { - c.logger.Info(context.Background(), "latency", slog.F("latency", ni.DERPLatency)) - callback(&Node{ + makeNode := func() *Node { + return &Node{ ID: c.netMap.SelfNode.ID, Key: c.netMap.SelfNode.Key, Addresses: c.netMap.SelfNode.Addresses, AllowedIPs: c.netMap.SelfNode.AllowedIPs, DiscoKey: c.magicConn.DiscoPublicKey(), - PreferredDERP: ni.PreferredDERP, - DERPLatency: ni.DERPLatency, - }) + Endpoints: c.lastEndpoints, + PreferredDERP: c.lastPreferredDERP, + DERPLatency: c.lastDERPLatency, + } + } + c.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.lastPreferredDERP = ni.PreferredDERP + c.lastDERPLatency = ni.DERPLatency + callback(makeNode()) + }) + c.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { + endpoints := make([]string, 0, len(s.LocalAddrs)) + for _, addr := range s.LocalAddrs { + endpoints = append(endpoints, addr.Addr.String()) + } + c.mutex.Lock() + defer c.mutex.Unlock() + c.lastEndpoints = endpoints + callback(makeNode()) }) } @@ -258,7 +281,14 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { c.mutex.Lock() defer c.mutex.Unlock() peerMap := map[tailcfg.NodeID]*tailcfg.Node{} + status := c.Status() for _, peer := range c.netMap.Peers { + if peerStatus, ok := status.Peer[peer.Key]; ok { + // Clear out inactive connections! + if !peerStatus.Active { + continue + } + } peerMap[peer.ID] = peer } for _, node := range nodes { @@ -268,6 +298,7 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { DiscoKey: node.DiscoKey, Addresses: node.Addresses, AllowedIPs: node.AllowedIPs, + Endpoints: node.Endpoints, DERP: fmt.Sprintf("%s:%d", tailcfg.DerpMagicIP, node.PreferredDERP), Hostinfo: hostinfo.New().View(), } @@ -289,6 +320,13 @@ func (c *Conn) UpdateNodes(nodes []*Node) error { return nil } +// Status returns the current ipnstate of a connection. +func (c *Conn) Status() *ipnstate.Status { + sb := &ipnstate.StatusBuilder{} + c.magicConn.UpdateStatus(sb) + return sb.Status() +} + // Ping sends a ping to the Wireguard engine. func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) { c.wireguardEngine.Ping(ip, pingType, cb) @@ -319,6 +357,7 @@ type Node struct { DERPLatency map[string]float64 `json:"derp_latency"` Addresses []netip.Prefix `json:"addresses"` AllowedIPs []netip.Prefix `json:"allowed_ips"` + Endpoints []string `json:"endpoints"` } // This and below is taken _mostly_ verbatim from Tailscale: From d03849ee679d3f77cc3ef6536819f7d112359ccd Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 02:48:41 +0000 Subject: [PATCH 17/54] Migrate coordinator to use net.conn --- agent/agent.go | 9 ++-- agent/agent_test.go | 96 ++++++++++++++++++++++++++++++++- cli/ssh.go | 3 +- coderd/workspaceagents.go | 4 +- codersdk/workspaceagents.go | 9 ++-- tailnet/coordinator.go | 79 +++++++++++++++++++--------- tailnet/coordinator_test.go | 102 +++++++++--------------------------- 7 files changed, 184 insertions(+), 118 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 3e909c8715ced..255030a1f16bb 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -28,7 +28,6 @@ import ( "go.uber.org/atomic" gossh "golang.org/x/crypto/ssh" "golang.org/x/xerrors" - "nhooyr.io/websocket" "tailscale.com/tailcfg" "cdr.dev/slog" @@ -82,7 +81,7 @@ type WebRTCDialer func(ctx context.Context, logger slog.Logger) (*peerbroker.Lis // CoordinatorDialer is a function that constructs a new broker. // A dialer must be passed in to allow for reconnects. -type CoordinatorDialer func(ctx context.Context) (*websocket.Conn, error) +type CoordinatorDialer func(ctx context.Context) (net.Conn, error) // FetchMetadata is a function to obtain metadata for the agent. type FetchMetadata func(ctx context.Context) (Metadata, error) @@ -220,7 +219,7 @@ func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { // runCoordinator listens for nodes and updates the self-node as it changes. func (a *agent) runCoordinator(ctx context.Context) { - var coordinator *websocket.Conn + var coordinator net.Conn var err error // An exponential back-off occurs when the connection is failing to dial. // This is to prevent server spam in case of a coderd outage. @@ -239,7 +238,7 @@ func (a *agent) runCoordinator(ctx context.Context) { a.logger.Info(context.Background(), "connected to coordination server") break } - sendNodes, errChan := tailnet.ServeCoordinator(ctx, coordinator, a.network.UpdateNodes) + sendNodes, errChan := tailnet.ServeCoordinator(coordinator, a.network.UpdateNodes) a.network.SetNodeCallback(sendNodes) select { case <-ctx.Done(): @@ -885,9 +884,7 @@ func (a *agent) Close() error { } close(a.closed) a.closeCancel() - fmt.Printf("CLOSING NETWORK!!!!\n") if a.network != nil { - fmt.Printf("ACTUALLY CLOSING NETWORK!!!!\n") _ = a.network.Close() } _ = a.sshServer.Close() diff --git a/agent/agent_test.go b/agent/agent_test.go index a5eb613650d27..6f9f6185d1cb9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3,10 +3,14 @@ package agent_test import ( "bufio" "context" + "crypto/tls" "encoding/json" "fmt" "io" "net" + "net/http" + "net/http/httptest" + "net/netip" "os" "os/exec" "path/filepath" @@ -17,6 +21,11 @@ import ( "time" "golang.org/x/xerrors" + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + tslogger "tailscale.com/types/logger" scp "github.com/bramvdbogaerde/go-scp" "github.com/google/uuid" @@ -38,6 +47,7 @@ import ( "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/pty/ptytest" + "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" ) @@ -423,7 +433,14 @@ func TestAgent(t *testing.T) { t.Run("Tailscale", func(t *testing.T) { t.Parallel() - + derpMap := runDERPAndStun(t, tailnet.Logger(slogtest.Make(t, nil))) + conn := setupSSHSession(t, agent.Metadata{ + DERPMap: derpMap, + }) + defer conn.Close() + output, err := conn.CombinedOutput("echo test") + require.NoError(t, err) + t.Log(string(output)) }) } @@ -469,6 +486,9 @@ func setupSSHSession(t *testing.T, options agent.Metadata) *ssh.Session { func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) agent.Conn { client, server := provisionersdk.TransportPipe() + tailscale := metadata.DERPMap != nil + coordinator := tailnet.NewCoordinator() + agentID := uuid.New() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { return metadata, nil @@ -477,6 +497,12 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) listener, err := peerbroker.Listen(server, nil) return listener, err }, + CoordinatorDialer: func(ctx context.Context) (net.Conn, error) { + clientConn, serverConn := net.Pipe() + go coordinator.ServeAgent(serverConn, agentID) + return clientConn, nil + }, + EnableTailnet: tailscale, Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), ReconnectingPTYTimeout: ptyTimeout, }) @@ -488,6 +514,24 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) api := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) stream, err := api.NegotiateConnection(context.Background()) assert.NoError(t, err) + if tailscale { + conn, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, + DERPMap: metadata.DERPMap, + Logger: slogtest.Make(t, nil).Named("tailnet"), + }) + require.NoError(t, err) + + clientConn, serverConn := net.Pipe() + go coordinator.ServeClient(serverConn, uuid.New(), agentID) + sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error { + return conn.UpdateNodes(node) + }) + conn.SetNodeCallback(sendNode) + return &agent.TailnetConn{ + Conn: conn, + } + } conn, err := peerbroker.Dial(stream, []webrtc.ICEServer{}, &peer.ConnOptions{ Logger: slogtest.Make(t, nil), }) @@ -532,3 +576,53 @@ func assertWritePayload(t *testing.T, w io.Writer, payload []byte) { assert.NoError(t, err, "write payload") assert.Equal(t, len(payload), n, "payload length does not match") } + +func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) { + d := derp.NewServer(key.NewNode(), logf) + server := httptest.NewUnstartedServer(derphttp.Handler(d)) + server.Config.ErrorLog = tslogger.StdLogger(logf) + server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + server.StartTLS() + + // stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + t.Cleanup(func() { + server.CloseClientConnections() + server.Close() + d.Close() + // stunCleanup() + }) + + tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) + if !ok { + t.FailNow() + } + + return &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "test", + RegionName: "Testlandia", + Nodes: []*tailcfg.DERPNode{ + { + Name: "t1", + RegionID: 1, + HostName: "stun.l.google.com", + DERPPort: -1, + STUNPort: 19302, + STUNOnly: true, + }, + { + Name: "t2", + RegionID: 1, + IPv4: "127.0.0.1", + IPv6: "none", + STUNPort: -1, + DERPPort: tcpAddr.Port, + InsecureForTests: true, + }, + }, + }, + }, + } +} diff --git a/cli/ssh.go b/cli/ssh.go index 61602137e0208..3b06ea562c140 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -21,7 +21,6 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" - "cdr.dev/slog/sloggers/sloghuman" "github.com/coder/coder/agent" "github.com/coder/coder/cli/cliflag" @@ -91,7 +90,7 @@ func ssh() *cobra.Command { if !wireguard { conn, err = client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil) } else { - conn, err = client.DialWorkspaceAgentTailnet(ctx, slog.Make(sloghuman.Sink(cmd.ErrOrStderr())).Leveled(slog.LevelDebug), workspaceAgent.ID) + conn, err = client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, workspaceAgent.ID) } if err != nil { return err diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index c3a1e017a021b..520ea9da38274 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -530,7 +530,7 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request return } defer conn.Close(websocket.StatusNormalClosure, "") - err = api.ConnCoordinator.ServeAgent(r.Context(), conn, workspaceAgent.ID) + err = api.ConnCoordinator.ServeAgent(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return @@ -556,7 +556,7 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R return } defer conn.Close(websocket.StatusNormalClosure, "") - err = api.ConnCoordinator.ServeClient(r.Context(), conn, uuid.New(), workspaceAgent.ID) + err = api.ConnCoordinator.ServeClient(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 646fdcb15d260..c349a25b4257e 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -281,7 +281,7 @@ func (c *Client) UpdateWorkspaceAgentNode(ctx context.Context, agentID uuid.UUID return nil } -func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (*websocket.Conn, error) { +func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, error) { coordinateURL, err := c.URL.Parse("/api/v2/workspaceagents/me/coordinate") if err != nil { return nil, xerrors.Errorf("parse url: %w", err) @@ -300,7 +300,10 @@ func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (*websocket.Co conn, _, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, }) - return conn, err + if err != nil { + return nil, err + } + return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil } func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logger, agentID uuid.UUID) (agent.Conn, error) { @@ -370,7 +373,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg logger.Debug(ctx, "failed to dial", slog.Error(err)) continue } - sendNode, errChan := tailnet.ServeCoordinator(ctx, ws, func(node []*tailnet.Node) error { + sendNode, errChan := tailnet.ServeCoordinator(websocket.NetConn(ctx, ws, websocket.MessageBinary), func(node []*tailnet.Node) error { return conn.UpdateNodes(node) }) conn.SetNodeCallback(sendNode) diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 1c51125918475..ef55712128a2f 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -1,23 +1,24 @@ package tailnet import ( - "context" + "encoding/json" "errors" + "io" + "net" "sync" "github.com/google/uuid" "golang.org/x/xerrors" - "nhooyr.io/websocket" - "nhooyr.io/websocket/wsjson" ) // ServeCoordinator matches the RW structure of a coordinator to exchange node messages. -func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) { +func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) { errChan := make(chan error, 1) go func() { + decoder := json.NewDecoder(conn) for { var nodes []*Node - err := wsjson.Read(ctx, socket, &nodes) + err := decoder.Decode(&nodes) if err != nil { errChan <- xerrors.Errorf("read: %w", err) return @@ -30,8 +31,13 @@ func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes f }() return func(node *Node) { - err := wsjson.Write(ctx, socket, node) - if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { + data, err := json.Marshal(node) + if err != nil { + errChan <- xerrors.Errorf("marshal node: %w", err) + return + } + _, err = conn.Write(data) + if errors.Is(err, io.EOF) { errChan <- nil return } @@ -45,8 +51,8 @@ func ServeCoordinator(ctx context.Context, socket *websocket.Conn, updateNodes f func NewCoordinator() *Coordinator { return &Coordinator{ nodes: map[uuid.UUID]*Node{}, - agentSockets: map[uuid.UUID]*websocket.Conn{}, - agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]*websocket.Conn{}, + agentSockets: map[uuid.UUID]net.Conn{}, + agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{}, } } @@ -62,10 +68,10 @@ type Coordinator struct { // Maps agent and connection IDs to a node. nodes map[uuid.UUID]*Node // Maps agent ID to an open socket. - agentSockets map[uuid.UUID]*websocket.Conn + agentSockets map[uuid.UUID]net.Conn // Maps agent ID to connection ID for sending // new node data as it comes in! - agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]*websocket.Conn + agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]net.Conn } // Node returns an in-memory node by ID. @@ -78,13 +84,18 @@ func (c *Coordinator) Node(id uuid.UUID) *Node { // ServeClient accepts a WebSocket connection that wants to // connect to an agent with the specified ID. -func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, id uuid.UUID, agent uuid.UUID) error { +func (c *Coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error { c.mutex.Lock() // When a new connection is requested, we update it with the latest // node of the agent. This allows the connection to establish. node, ok := c.nodes[agent] if ok { - err := wsjson.Write(ctx, socket, []*Node{node}) + data, err := json.Marshal([]*Node{node}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("marshal node: %w", err) + } + _, err = conn.Write(data) if err != nil { c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) @@ -92,12 +103,12 @@ func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, i } connectionSockets, ok := c.agentToConnectionSockets[agent] if !ok { - connectionSockets = map[uuid.UUID]*websocket.Conn{} + connectionSockets = map[uuid.UUID]net.Conn{} c.agentToConnectionSockets[agent] = connectionSockets } // Insert this connection into a map so the agent // can publish node updates. - connectionSockets[id] = socket + connectionSockets[id] = conn c.mutex.Unlock() defer func() { c.mutex.Lock() @@ -115,10 +126,11 @@ func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, i delete(c.agentToConnectionSockets, agent) }() + decoder := json.NewDecoder(conn) for { var node Node - err := wsjson.Read(ctx, socket, &node) - if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { + err := decoder.Decode(&node) + if errors.Is(err, io.EOF) { return nil } if err != nil { @@ -137,8 +149,13 @@ func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, i } // Write the new node from this client to the actively // connected agent. - err = wsjson.Write(ctx, agentSocket, []*Node{&node}) - if errors.Is(err, context.Canceled) { + data, err := json.Marshal([]*Node{&node}) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("marshal nodes: %w", err) + } + _, err = agentSocket.Write(data) + if errors.Is(err, io.EOF) { c.mutex.Unlock() return nil } @@ -152,7 +169,7 @@ func (c *Coordinator) ServeClient(ctx context.Context, socket *websocket.Conn, i // ServeAgent accepts a WebSocket connection to an agent that // listens to incoming connections and publishes node updates. -func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id uuid.UUID) error { +func (c *Coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error { c.mutex.Lock() sockets, ok := c.agentToConnectionSockets[id] if ok { @@ -166,7 +183,12 @@ func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id } nodes = append(nodes, node) } - err := wsjson.Write(ctx, socket, nodes) + data, err := json.Marshal(nodes) + if err != nil { + c.mutex.Unlock() + return xerrors.Errorf("marshal json: %w", err) + } + _, err = conn.Write(data) if err != nil { c.mutex.Unlock() return xerrors.Errorf("write nodes: %w", err) @@ -178,9 +200,9 @@ func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id // we expect one agent to be running. oldAgentSocket, ok := c.agentSockets[id] if ok { - _ = oldAgentSocket.Close(websocket.StatusNormalClosure, "another agent connected with the same id") + _ = oldAgentSocket.Close() } - c.agentSockets[id] = socket + c.agentSockets[id] = conn c.mutex.Unlock() defer func() { c.mutex.Lock() @@ -189,10 +211,11 @@ func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id delete(c.nodes, id) }() + decoder := json.NewDecoder(conn) for { var node Node - err := wsjson.Read(ctx, socket, &node) - if errors.Is(err, context.Canceled) || errors.As(err, &websocket.CloseError{}) { + err := decoder.Decode(&node) + if errors.Is(err, io.EOF) { return nil } if err != nil { @@ -205,13 +228,17 @@ func (c *Coordinator) ServeAgent(ctx context.Context, socket *websocket.Conn, id c.mutex.Unlock() continue } + data, err := json.Marshal([]*Node{&node}) + if err != nil { + return xerrors.Errorf("marshal nodes: %w", err) + } // Publish the new node to every listening socket. var wg sync.WaitGroup wg.Add(len(connectionSockets)) for _, connectionSocket := range connectionSockets { connectionSocket := connectionSocket go func() { - _ = wsjson.Write(ctx, connectionSocket, []*Node{&node}) + _, _ = connectionSocket.Write(data) wg.Done() }() } diff --git a/tailnet/coordinator_test.go b/tailnet/coordinator_test.go index 68e6016c9d412..f3fdab88d5ef8 100644 --- a/tailnet/coordinator_test.go +++ b/tailnet/coordinator_test.go @@ -1,17 +1,12 @@ package tailnet_test import ( - "bufio" - "context" "net" - "net/http" - "net/http/httptest" "testing" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "nhooyr.io/websocket" "github.com/coder/coder/tailnet" "github.com/coder/coder/testutil" @@ -22,14 +17,14 @@ func TestCoordinator(t *testing.T) { t.Run("ClientWithoutAgent", func(t *testing.T) { t.Parallel() coordinator := tailnet.NewCoordinator() - client, server := pipeWS(t) - sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) error { + client, server := net.Pipe() + sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil }) id := uuid.New() closeChan := make(chan struct{}) go func() { - err := coordinator.ServeClient(context.Background(), server, id, uuid.New()) + err := coordinator.ServeClient(server, id, uuid.New()) assert.NoError(t, err) close(closeChan) }() @@ -37,7 +32,7 @@ func TestCoordinator(t *testing.T) { require.Eventually(t, func() bool { return coordinator.Node(id) != nil }, testutil.WaitShort, testutil.IntervalFast) - err := client.Close(websocket.StatusNormalClosure, "") + err := client.Close() require.NoError(t, err) <-errChan <-closeChan @@ -46,14 +41,14 @@ func TestCoordinator(t *testing.T) { t.Run("AgentWithoutClients", func(t *testing.T) { t.Parallel() coordinator := tailnet.NewCoordinator() - client, server := pipeWS(t) - sendNode, errChan := tailnet.ServeCoordinator(context.Background(), client, func(node []*tailnet.Node) error { + client, server := net.Pipe() + sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error { return nil }) id := uuid.New() closeChan := make(chan struct{}) go func() { - err := coordinator.ServeAgent(context.Background(), server, id) + err := coordinator.ServeAgent(server, id) assert.NoError(t, err) close(closeChan) }() @@ -61,7 +56,7 @@ func TestCoordinator(t *testing.T) { require.Eventually(t, func() bool { return coordinator.Node(id) != nil }, testutil.WaitShort, testutil.IntervalFast) - err := client.Close(websocket.StatusNormalClosure, "") + err := client.Close() require.NoError(t, err) <-errChan <-closeChan @@ -71,17 +66,17 @@ func TestCoordinator(t *testing.T) { t.Parallel() coordinator := tailnet.NewCoordinator() - agentWS, agentServerWS := pipeWS(t) - defer agentWS.Close(websocket.StatusNormalClosure, "") + agentWS, agentServerWS := net.Pipe() + defer agentWS.Close() agentNodeChan := make(chan []*tailnet.Node) - sendAgentNode, agentErrChan := tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) error { + sendAgentNode, agentErrChan := tailnet.ServeCoordinator(agentWS, func(nodes []*tailnet.Node) error { agentNodeChan <- nodes return nil }) agentID := uuid.New() closeAgentChan := make(chan struct{}) go func() { - err := coordinator.ServeAgent(context.Background(), agentServerWS, agentID) + err := coordinator.ServeAgent(agentServerWS, agentID) assert.NoError(t, err) close(closeAgentChan) }() @@ -90,18 +85,18 @@ func TestCoordinator(t *testing.T) { return coordinator.Node(agentID) != nil }, testutil.WaitShort, testutil.IntervalFast) - clientWS, clientServerWS := pipeWS(t) - defer clientWS.Close(websocket.StatusNormalClosure, "") - defer clientServerWS.Close(websocket.StatusNormalClosure, "") + clientWS, clientServerWS := net.Pipe() + defer clientWS.Close() + defer clientServerWS.Close() clientNodeChan := make(chan []*tailnet.Node) - sendClientNode, clientErrChan := tailnet.ServeCoordinator(context.Background(), clientWS, func(nodes []*tailnet.Node) error { + sendClientNode, clientErrChan := tailnet.ServeCoordinator(clientWS, func(nodes []*tailnet.Node) error { clientNodeChan <- nodes return nil }) clientID := uuid.New() closeClientChan := make(chan struct{}) go func() { - err := coordinator.ServeClient(context.Background(), clientServerWS, clientID, agentID) + err := coordinator.ServeClient(clientServerWS, clientID, agentID) assert.NoError(t, err) close(closeClientChan) }() @@ -117,22 +112,22 @@ func TestCoordinator(t *testing.T) { require.Len(t, agentNodes, 1) // Close the agent WebSocket so a new one can connect. - err := agentWS.Close(websocket.StatusNormalClosure, "") + err := agentWS.Close() require.NoError(t, err) <-agentErrChan <-closeAgentChan // Create a new agent connection. This is to simulate a reconnect! - agentWS, agentServerWS = pipeWS(t) - defer agentWS.Close(websocket.StatusNormalClosure, "") + agentWS, agentServerWS = net.Pipe() + defer agentWS.Close() agentNodeChan = make(chan []*tailnet.Node) - _, agentErrChan = tailnet.ServeCoordinator(context.Background(), agentWS, func(nodes []*tailnet.Node) error { + _, agentErrChan = tailnet.ServeCoordinator(agentWS, func(nodes []*tailnet.Node) error { agentNodeChan <- nodes return nil }) closeAgentChan = make(chan struct{}) go func() { - err := coordinator.ServeAgent(context.Background(), agentServerWS, agentID) + err := coordinator.ServeAgent(agentServerWS, agentID) assert.NoError(t, err) close(closeAgentChan) }() @@ -140,63 +135,14 @@ func TestCoordinator(t *testing.T) { clientNodes = <-agentNodeChan require.Len(t, clientNodes, 1) - err = agentWS.Close(websocket.StatusNormalClosure, "") + err = agentWS.Close() require.NoError(t, err) <-agentErrChan <-closeAgentChan - err = clientWS.Close(websocket.StatusNormalClosure, "") + err = clientWS.Close() require.NoError(t, err) <-clientErrChan <-closeClientChan }) } - -// pipeWS creates a new piped WebSocket pair. -func pipeWS(t *testing.T) (clientConn, serverConn *websocket.Conn) { - t.Helper() - // nolint:bodyclose - clientConn, _, _ = websocket.Dial(context.Background(), "ws://example.com", &websocket.DialOptions{ - HTTPClient: &http.Client{ - Transport: fakeTransport{ - h: func(w http.ResponseWriter, r *http.Request) { - serverConn, _ = websocket.Accept(w, r, nil) - }, - }, - }, - }) - t.Cleanup(func() { - _ = serverConn.Close(websocket.StatusInternalError, "") - _ = clientConn.Close(websocket.StatusInternalError, "") - }) - return clientConn, serverConn -} - -type fakeTransport struct { - h http.HandlerFunc -} - -func (t fakeTransport) RoundTrip(r *http.Request) (*http.Response, error) { - clientConn, serverConn := net.Pipe() - hj := testHijacker{ - ResponseRecorder: httptest.NewRecorder(), - serverConn: serverConn, - } - t.h.ServeHTTP(hj, r) - resp := hj.ResponseRecorder.Result() - if resp.StatusCode == http.StatusSwitchingProtocols { - resp.Body = clientConn - } - return resp, nil -} - -type testHijacker struct { - *httptest.ResponseRecorder - serverConn net.Conn -} - -var _ http.Hijacker = testHijacker{} - -func (hj testHijacker) Hijack() (net.Conn, *bufio.ReadWriter, error) { - return hj.serverConn, bufio.NewReadWriter(bufio.NewReader(hj.serverConn), bufio.NewWriter(hj.serverConn)), nil -} From 5e7197c2dfc02135146653c3da7b981c8dee98e2 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 05:42:49 +0000 Subject: [PATCH 18/54] Add closed func --- agent/conn.go | 4 ---- tailnet/conn.go | 13 +++++++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/agent/conn.go b/agent/conn.go index 306b37c0c7aaa..dc32220d81de6 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -138,10 +138,6 @@ type TailnetConn struct { *tailnet.Conn } -func (c *TailnetConn) Closed() <-chan struct{} { - return nil -} - func (c *TailnetConn) Ping() (time.Duration, error) { return 0, nil } diff --git a/tailnet/conn.go b/tailnet/conn.go index 11a2fff15ad58..08d86b9b2fc1f 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -221,6 +221,7 @@ func IP() netip.Addr { // Conn is an actively listening Wireguard connection. type Conn struct { mutex sync.Mutex + closed chan struct{} logger slog.Logger dialer *tsdial.Dialer @@ -332,6 +333,12 @@ func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate. c.wireguardEngine.Ping(ip, pingType, cb) } +// Closed is a channel that ends when the connection has +// been closed. +func (c *Conn) Closed() chan<- struct{} { + return c.closed +} + // Close shuts down the Wireguard connection. func (c *Conn) Close() error { for _, l := range c.listeners { @@ -339,6 +346,12 @@ func (c *Conn) Close() error { } c.mutex.Lock() defer c.mutex.Unlock() + select { + case <-c.closed: + return nil + default: + } + close(c.closed) _ = c.dialer.Close() _ = c.magicConn.Close() _ = c.netStack.Close() From a117d757ec3a796f5b5718d4ba817bd03b45e274 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 06:04:15 +0000 Subject: [PATCH 19/54] Fix close listener func --- tailnet/conn.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tailnet/conn.go b/tailnet/conn.go index 08d86b9b2fc1f..583588d31fd16 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -187,6 +187,7 @@ func NewConn(options *Options) (*Conn, error) { logIPs, _ := logIPSet.IPSet() wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter")))) server := &Conn{ + closed: make(chan struct{}), logger: options.Logger, magicConn: magicConn, dialer: dialer, @@ -335,15 +336,12 @@ func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate. // Closed is a channel that ends when the connection has // been closed. -func (c *Conn) Closed() chan<- struct{} { +func (c *Conn) Closed() <-chan struct{} { return c.closed } // Close shuts down the Wireguard connection. func (c *Conn) Close() error { - for _, l := range c.listeners { - _ = l.Close() - } c.mutex.Lock() defer c.mutex.Unlock() select { @@ -351,6 +349,9 @@ func (c *Conn) Close() error { return nil default: } + for _, l := range c.listeners { + _ = l.closeNoLock() + } close(c.closed) _ = c.dialer.Close() _ = c.magicConn.Close() @@ -454,6 +455,10 @@ func (ln *listener) Addr() net.Addr { return addr{ln} } func (ln *listener) Close() error { ln.s.mutex.Lock() defer ln.s.mutex.Unlock() + return ln.closeNoLock() +} + +func (ln *listener) closeNoLock() error { if v, ok := ln.s.listeners[ln.key]; ok && v == ln { delete(ln.s.listeners, ln.key) close(ln.conn) From cb89378f5eef3293fe174e6f9598fae7a3c648d5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 06:42:04 +0000 Subject: [PATCH 20/54] Make reconnecting PTY work --- agent/agent.go | 94 ++++++++++++++++++++++----------------- agent/conn.go | 7 +++ coderd/coderd.go | 2 +- coderd/workspaceagents.go | 33 ++++++++++++++ tailnet/conn.go | 34 +++++++++++++- 5 files changed, 126 insertions(+), 44 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 255030a1f16bb..55cfd31e9506c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -212,7 +212,12 @@ func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { if err != nil { return } - go a.handleReconnectingPTY(ctx, "tailnet", conn) + var msg reconnectingPTYInit + err = json.NewDecoder(conn).Decode(&msg) + if err != nil { + continue + } + go a.handleReconnectingPTY(ctx, msg, conn) } }() } @@ -354,7 +359,38 @@ func (a *agent) handlePeerConn(ctx context.Context, conn *peer.Conn) { case ProtocolSSH: go a.sshServer.HandleConn(channel.NetConn()) case ProtocolReconnectingPTY: - go a.handleReconnectingPTY(ctx, channel.Label(), channel.NetConn()) + rawID := channel.Label() + // The ID format is referenced in conn.go. + // :: + idParts := strings.SplitN(rawID, ":", 4) + if len(idParts) != 4 { + a.logger.Warn(ctx, "client sent invalid id format", slog.F("raw-id", rawID)) + continue + } + id := idParts[0] + // Enforce a consistent format for IDs. + _, err := uuid.Parse(id) + if err != nil { + a.logger.Warn(ctx, "client sent reconnection token that isn't a uuid", slog.F("id", id), slog.Error(err)) + continue + } + // Parse the initial terminal dimensions. + height, err := strconv.Atoi(idParts[1]) + if err != nil { + a.logger.Warn(ctx, "client sent invalid height", slog.F("id", id), slog.F("height", idParts[1])) + continue + } + width, err := strconv.Atoi(idParts[2]) + if err != nil { + a.logger.Warn(ctx, "client sent invalid width", slog.F("id", id), slog.F("width", idParts[2])) + continue + } + go a.handleReconnectingPTY(ctx, reconnectingPTYInit{ + ID: id, + Height: uint16(height), + Width: uint16(width), + Command: idParts[3], + }, channel.NetConn()) case ProtocolDial: go a.handleDial(ctx, channel.Label(), channel.NetConn()) default: @@ -610,45 +646,19 @@ func (a *agent) handleSSHSession(session ssh.Session) (retErr error) { return cmd.Wait() } -func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn net.Conn) { +func (a *agent) handleReconnectingPTY(ctx context.Context, msg reconnectingPTYInit, conn net.Conn) { defer conn.Close() - // The ID format is referenced in conn.go. - // :: - idParts := strings.SplitN(rawID, ":", 4) - if len(idParts) != 4 { - a.logger.Warn(ctx, "client sent invalid id format", slog.F("raw-id", rawID)) - return - } - id := idParts[0] - // Enforce a consistent format for IDs. - _, err := uuid.Parse(id) - if err != nil { - a.logger.Warn(ctx, "client sent reconnection token that isn't a uuid", slog.F("id", id), slog.Error(err)) - return - } - // Parse the initial terminal dimensions. - height, err := strconv.Atoi(idParts[1]) - if err != nil { - a.logger.Warn(ctx, "client sent invalid height", slog.F("id", id), slog.F("height", idParts[1])) - return - } - width, err := strconv.Atoi(idParts[2]) - if err != nil { - a.logger.Warn(ctx, "client sent invalid width", slog.F("id", id), slog.F("width", idParts[2])) - return - } - var rpty *reconnectingPTY - rawRPTY, ok := a.reconnectingPTYs.Load(id) + rawRPTY, ok := a.reconnectingPTYs.Load(msg.ID) if ok { rpty, ok = rawRPTY.(*reconnectingPTY) if !ok { - a.logger.Warn(ctx, "found invalid type in reconnecting pty map", slog.F("id", id)) + a.logger.Warn(ctx, "found invalid type in reconnecting pty map", slog.F("id", msg.ID)) } } else { // Empty command will default to the users shell! - cmd, err := a.createCommand(ctx, idParts[3], nil) + cmd, err := a.createCommand(ctx, msg.Command, nil) if err != nil { a.logger.Warn(ctx, "create reconnecting pty command", slog.Error(err)) return @@ -657,7 +667,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne ptty, process, err := pty.Start(cmd) if err != nil { - a.logger.Warn(ctx, "start reconnecting pty command", slog.F("id", id)) + a.logger.Warn(ctx, "start reconnecting pty command", slog.F("id", msg.ID)) } // Default to buffer 64KiB. @@ -678,7 +688,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne timeout: time.AfterFunc(a.reconnectingPTYTimeout, cancelFunc), circularBuffer: circularBuffer, } - a.reconnectingPTYs.Store(id, rpty) + a.reconnectingPTYs.Store(msg.ID, rpty) go func() { // CommandContext isn't respected for Windows PTYs right now, // so we need to manually track the lifecycle. @@ -707,7 +717,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne _, err = rpty.circularBuffer.Write(part) rpty.circularBufferMutex.Unlock() if err != nil { - a.logger.Error(ctx, "reconnecting pty write buffer", slog.Error(err), slog.F("id", id)) + a.logger.Error(ctx, "reconnecting pty write buffer", slog.Error(err), slog.F("id", msg.ID)) break } rpty.activeConnsMutex.Lock() @@ -721,22 +731,22 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne // ID from memory. _ = process.Kill() rpty.Close() - a.reconnectingPTYs.Delete(id) + a.reconnectingPTYs.Delete(msg.ID) a.connCloseWait.Done() }() } // Resize the PTY to initial height + width. - err = rpty.ptty.Resize(uint16(height), uint16(width)) + err := rpty.ptty.Resize(uint16(msg.Height), uint16(msg.Width)) if err != nil { // We can continue after this, it's not fatal! - a.logger.Error(ctx, "resize reconnecting pty", slog.F("id", id), slog.Error(err)) + a.logger.Error(ctx, "resize reconnecting pty", slog.F("id", msg.ID), slog.Error(err)) } // Write any previously stored data for the TTY. rpty.circularBufferMutex.RLock() _, err = conn.Write(rpty.circularBuffer.Bytes()) rpty.circularBufferMutex.RUnlock() if err != nil { - a.logger.Warn(ctx, "write reconnecting pty buffer", slog.F("id", id), slog.Error(err)) + a.logger.Warn(ctx, "write reconnecting pty buffer", slog.F("id", msg.ID), slog.Error(err)) return } connectionID := uuid.NewString() @@ -782,12 +792,12 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne return } if err != nil { - a.logger.Warn(ctx, "reconnecting pty buffer read error", slog.F("id", id), slog.Error(err)) + a.logger.Warn(ctx, "reconnecting pty buffer read error", slog.F("id", msg.ID), slog.Error(err)) return } _, err = rpty.ptty.Input().Write([]byte(req.Data)) if err != nil { - a.logger.Warn(ctx, "write to reconnecting pty", slog.F("id", id), slog.Error(err)) + a.logger.Warn(ctx, "write to reconnecting pty", slog.F("id", msg.ID), slog.Error(err)) return } // Check if a resize needs to happen! @@ -797,7 +807,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, rawID string, conn ne err = rpty.ptty.Resize(req.Height, req.Width) if err != nil { // We can continue after this, it's not fatal! - a.logger.Error(ctx, "resize reconnecting pty", slog.F("id", id), slog.Error(err)) + a.logger.Error(ctx, "resize reconnecting pty", slog.F("id", msg.ID), slog.Error(err)) } } } diff --git a/agent/conn.go b/agent/conn.go index dc32220d81de6..518e65e62b0af 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -146,6 +146,13 @@ func (c *TailnetConn) CloseWithError(err error) error { return c.Close() } +type reconnectingPTYInit struct { + ID string + Height uint16 + Width uint16 + Command string +} + func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { return c.DialContextTCP(context.Background(), netip.AddrPortFrom(tailnetIP, uint16(tailnetReconnectingPTYPort))) } diff --git a/coderd/coderd.go b/coderd/coderd.go index bb404518c8523..cbeb47974e9f2 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -119,7 +119,7 @@ func New(options *Options) *API { Handler: r, siteHandler: site.Handler(site.FS(), binFS), } - api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgent, 0) + api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0) api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) oauthConfigs := &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 520ea9da38274..26bda735fc844 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -8,6 +8,7 @@ import ( "io" "net" "net/http" + "net/netip" "strconv" "time" @@ -435,6 +436,38 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { _, _ = io.Copy(ptNetConn, wsNetConn) } +func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (agent.Conn, error) { + clientConn, serverConn := net.Pipe() + go func() { + <-r.Context().Done() + _ = clientConn.Close() + _ = serverConn.Close() + }() + + conn, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, + DERPMap: api.DERPMap, + Logger: api.Logger.Named("tailnet").Leveled(slog.LevelDebug), + }) + if err != nil { + return nil, xerrors.Errorf("create tailnet conn: %w", err) + } + + sendNodes, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error { + return conn.UpdateNodes(node) + }) + conn.SetNodeCallback(sendNodes) + go func() { + err := api.ConnCoordinator.ServeClient(serverConn, uuid.New(), agentID) + if err != nil { + _ = conn.Close() + } + }() + return &agent.TailnetConn{ + Conn: conn, + }, nil +} + // dialWorkspaceAgent connects to a workspace agent by ID. Only rely on // r.Context() for cancellation if it's use is safe or r.Hijack() has // not been performed. diff --git a/tailnet/conn.go b/tailnet/conn.go index 583588d31fd16..9984c41f1ad00 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -3,8 +3,10 @@ package tailnet import ( "context" "fmt" + "io" "net" "net/netip" + "strconv" "sync" "time" @@ -418,7 +420,7 @@ func (c *Conn) forwardTCP(conn net.Conn, port uint16) { ln, ok := c.listeners[listenKey{"tcp", "", fmt.Sprint(port)}] c.mutex.Unlock() if !ok { - _ = conn.Close() + c.forwardTCPToLocal(conn, port) return } t := time.NewTimer(time.Second) @@ -430,6 +432,36 @@ func (c *Conn) forwardTCP(conn net.Conn, port uint16) { } } +func (c *Conn) forwardTCPToLocal(conn net.Conn, port uint16) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer conn.Close() + + dialAddrStr := net.JoinHostPort("127.0.0.1", strconv.Itoa(int(port))) + var stdDialer net.Dialer + server, err := stdDialer.DialContext(ctx, "tcp", dialAddrStr) + if err != nil { + c.logger.Debug(ctx, "dial local port", slog.F("port", port), slog.Error(err)) + return + } + defer server.Close() + + connClosed := make(chan error, 2) + go func() { + _, err := io.Copy(server, conn) + connClosed <- err + }() + go func() { + _, err := io.Copy(conn, server) + connClosed <- err + }() + err = <-connClosed + if err != nil { + c.logger.Debug(ctx, "proxy connection closed with error", slog.Error(err)) + } + c.logger.Debug(ctx, "forwarded connection closed", slog.F("local_addr", dialAddrStr)) +} + type listenKey struct { network string host string From 85be897fe2ccb70295e17bd6d9619fa88a219803 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 06:51:10 +0000 Subject: [PATCH 21/54] Fix reconnecting PTY --- agent/conn.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/agent/conn.go b/agent/conn.go index 518e65e62b0af..06ff3f7998769 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -154,7 +154,26 @@ type reconnectingPTYInit struct { } func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command string) (net.Conn, error) { - return c.DialContextTCP(context.Background(), netip.AddrPortFrom(tailnetIP, uint16(tailnetReconnectingPTYPort))) + conn, err := c.DialContextTCP(context.Background(), netip.AddrPortFrom(tailnetIP, uint16(tailnetReconnectingPTYPort))) + if err != nil { + return nil, err + } + data, err := json.Marshal(reconnectingPTYInit{ + ID: id, + Height: height, + Width: width, + Command: command, + }) + if err != nil { + _ = conn.Close() + return nil, err + } + _, err = conn.Write(data) + if err != nil { + _ = conn.Close() + return nil, err + } + return conn, nil } func (c *TailnetConn) SSH() (net.Conn, error) { From 4356b007851a61f4d806cd3ce2cb3397aa758337 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 14:03:55 +0000 Subject: [PATCH 22/54] Update CI to Go 1.19 --- .github/workflows/coder.yaml | 14 +++++------ agent/agent_test.go | 16 ++++-------- coderd/database/postgres/postgres.go | 1 + codersdk/workspaceagents.go | 37 ---------------------------- go.mod | 1 - site/src/api/typesGenerated.ts | 7 ------ tailnet/conn_test.go | 16 ++++-------- 7 files changed, 18 insertions(+), 74 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 733dd4d02c576..be126942fe903 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -91,7 +91,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: golangci-lint uses: golangci/golangci-lint-action@v3.2.0 with: @@ -165,7 +165,7 @@ jobs: version: "3.20.0" - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -241,7 +241,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -328,7 +328,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -411,7 +411,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -516,7 +516,7 @@ jobs: # Go is required for uploading the test results to datadog - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - uses: actions/setup-node@v3 with: @@ -574,7 +574,7 @@ jobs: # Go is required for uploading the test results to datadog - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - uses: hashicorp/setup-terraform@v2 with: diff --git a/agent/agent_test.go b/agent/agent_test.go index 6f9f6185d1cb9..20ace9a7dedf5 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -23,9 +23,11 @@ import ( "golang.org/x/xerrors" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" "tailscale.com/types/key" tslogger "tailscale.com/types/logger" + "tailscale.com/types/nettype" scp "github.com/bramvdbogaerde/go-scp" "github.com/google/uuid" @@ -584,12 +586,12 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) server.StartTLS() - // stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) t.Cleanup(func() { server.CloseClientConnections() server.Close() d.Close() - // stunCleanup() + stunCleanup() }) tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) @@ -604,20 +606,12 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) RegionCode: "test", RegionName: "Testlandia", Nodes: []*tailcfg.DERPNode{ - { - Name: "t1", - RegionID: 1, - HostName: "stun.l.google.com", - DERPPort: -1, - STUNPort: 19302, - STUNOnly: true, - }, { Name: "t2", RegionID: 1, IPv4: "127.0.0.1", IPv6: "none", - STUNPort: -1, + STUNPort: stunAddr.Port, DERPPort: tcpAddr.Port, InsecureForTests: true, }, diff --git a/coderd/database/postgres/postgres.go b/coderd/database/postgres/postgres.go index a5cb6a39bf787..a1637cfb0261c 100644 --- a/coderd/database/postgres/postgres.go +++ b/coderd/database/postgres/postgres.go @@ -155,6 +155,7 @@ func Open() (string, func(), error) { if err != nil { return "", nil, retryErr } + return dbURL, func() { _ = pool.Purge(resource) _ = os.RemoveAll(tempDir) diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index c349a25b4257e..1c1b11f2238ff 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -19,7 +19,6 @@ import ( "golang.org/x/net/proxy" "golang.org/x/xerrors" "nhooyr.io/websocket" - "nhooyr.io/websocket/wsjson" "tailscale.com/tailcfg" "cdr.dev/slog" @@ -265,22 +264,6 @@ func (c *Client) ListenWorkspaceAgent(ctx context.Context, logger slog.Logger) ( }) } -// UpdateWorkspaceAgentNode publishes a node update for the provided agent. -// This should be used to negotiate a connection. -func (c *Client) UpdateWorkspaceAgentNode(ctx context.Context, agentID uuid.UUID, node *tailnet.Node) error { - res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/workspaceagents/%s/node", - agentID, - ), node) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return readBodyAsError(res) - } - return nil -} - func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, error) { coordinateURL, err := c.URL.Parse("/api/v2/workspaceagents/me/coordinate") if err != nil { @@ -530,23 +513,3 @@ func (c *Client) turnProxyDialer(ctx context.Context, httpClient *http.Client, p return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil }) } - -// workspaceAgentNodeBroker is used to listen for node updates -// and write them. -type workspaceAgentNodeBroker struct { - conn *websocket.Conn -} - -func (w *workspaceAgentNodeBroker) Read(ctx context.Context) (*tailnet.Node, error) { - var node tailnet.Node - err := wsjson.Read(ctx, w.conn, &node) - return &node, err -} - -func (w *workspaceAgentNodeBroker) Write(ctx context.Context, node *tailnet.Node) error { - return wsjson.Write(ctx, w.conn, node) -} - -func (w *workspaceAgentNodeBroker) Close() error { - return w.conn.Close(websocket.StatusGoingAway, "") -} diff --git a/go.mod b/go.mod index 5caf10e25f04a..8688e948e97cf 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ replace github.com/chzyer/readline => github.com/kylecarbs/readline v0.0.0-20220 // Required until https://github.com/briandowns/spinner/pull/136 is merged. replace github.com/briandowns/spinner => github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e -// Required until is merged. replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 // Required until https://github.com/fergusstrange/embedded-postgres/pull/75 is merged. diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index f696f7108161a..601f8220f1173 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -527,13 +527,6 @@ export interface WorkspaceResourceMetadata { readonly sensitive: boolean } -// From codersdk/workspaceagents.go -export interface workspaceAgentNodeBroker { - // Named type "nhooyr.io/websocket.Conn" unknown, using "any" - // eslint-disable-next-line @typescript-eslint/no-explicit-any - readonly conn?: any -} - // From codersdk/workspacebuilds.go export type BuildReason = "autostart" | "autostop" | "initiator" diff --git a/tailnet/conn_test.go b/tailnet/conn_test.go index 084475921e8d3..da7b798119bd5 100644 --- a/tailnet/conn_test.go +++ b/tailnet/conn_test.go @@ -14,9 +14,11 @@ import ( "go.uber.org/goleak" "tailscale.com/derp" "tailscale.com/derp/derphttp" + "tailscale.com/net/stun/stuntest" "tailscale.com/tailcfg" "tailscale.com/types/key" tslogger "tailscale.com/types/logger" + "tailscale.com/types/nettype" "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" @@ -84,12 +86,12 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) server.StartTLS() - // stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) t.Cleanup(func() { server.CloseClientConnections() server.Close() d.Close() - // stunCleanup() + stunCleanup() }) tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) @@ -104,20 +106,12 @@ func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) RegionCode: "test", RegionName: "Testlandia", Nodes: []*tailcfg.DERPNode{ - { - Name: "t1", - RegionID: 1, - HostName: "stun.l.google.com", - DERPPort: -1, - STUNPort: 19302, - STUNOnly: true, - }, { Name: "t2", RegionID: 1, IPv4: "127.0.0.1", IPv6: "none", - STUNPort: -1, + STUNPort: stunAddr.Port, DERPPort: tcpAddr.Port, InsecureForTests: true, }, From de09c87c45c1cea677187e62a857353af2b4760b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 19:02:54 +0000 Subject: [PATCH 23/54] Add CLI flags for DERP mapping --- cli/cliflag/cliflag.go | 17 ++++++++++++++ cli/cliflag/cliflag_test.go | 45 ++++++++++++++++++++++++++++++++++--- cli/server.go | 18 +++++++++++++-- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/cli/cliflag/cliflag.go b/cli/cliflag/cliflag.go index 843416c3ff3ea..42722ecc1cfb3 100644 --- a/cli/cliflag/cliflag.go +++ b/cli/cliflag/cliflag.go @@ -90,6 +90,23 @@ func Uint8VarP(flagset *pflag.FlagSet, ptr *uint8, name string, shorthand string flagset.Uint8VarP(ptr, name, shorthand, uint8(vi64), fmtUsage(usage, env)) } +// IntVarP sets a uint8 flag on the given flag set. +func IntVarP(flagset *pflag.FlagSet, ptr *int, name string, shorthand string, env string, def int, usage string) { + val, ok := os.LookupEnv(env) + if !ok || val == "" { + flagset.IntVarP(ptr, name, shorthand, def, fmtUsage(usage, env)) + return + } + + vi64, err := strconv.ParseUint(val, 10, 8) + if err != nil { + flagset.IntVarP(ptr, name, shorthand, def, fmtUsage(usage, env)) + return + } + + flagset.IntVarP(ptr, name, shorthand, int(vi64), fmtUsage(usage, env)) +} + func Bool(flagset *pflag.FlagSet, name, shorthand, env string, def bool, usage string) { val, ok := os.LookupEnv(env) if !ok || val == "" { diff --git a/cli/cliflag/cliflag_test.go b/cli/cliflag/cliflag_test.go index acdf7d6765fb5..5d826166307a5 100644 --- a/cli/cliflag/cliflag_test.go +++ b/cli/cliflag/cliflag_test.go @@ -108,7 +108,7 @@ func TestCliflag(t *testing.T) { require.Equal(t, []string{}, got) }) - t.Run("IntDefault", func(t *testing.T) { + t.Run("UInt8Default", func(t *testing.T) { var ptr uint8 flagset, name, shorthand, env, usage := randomFlag() def, _ := cryptorand.Int63n(10) @@ -121,7 +121,7 @@ func TestCliflag(t *testing.T) { require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env)) }) - t.Run("IntEnvVar", func(t *testing.T) { + t.Run("UInt8EnvVar", func(t *testing.T) { var ptr uint8 flagset, name, shorthand, env, usage := randomFlag() envValue, _ := cryptorand.Int63n(10) @@ -134,7 +134,7 @@ func TestCliflag(t *testing.T) { require.Equal(t, uint8(envValue), got) }) - t.Run("IntFailParse", func(t *testing.T) { + t.Run("UInt8FailParse", func(t *testing.T) { var ptr uint8 flagset, name, shorthand, env, usage := randomFlag() envValue, _ := cryptorand.String(10) @@ -147,6 +147,45 @@ func TestCliflag(t *testing.T) { require.Equal(t, uint8(def), got) }) + t.Run("IntDefault", func(t *testing.T) { + var ptr int + flagset, name, shorthand, env, usage := randomFlag() + def, _ := cryptorand.Int63n(10) + + cliflag.IntVarP(flagset, &ptr, name, shorthand, env, int(def), usage) + got, err := flagset.GetInt(name) + require.NoError(t, err) + require.Equal(t, int(def), got) + require.Contains(t, flagset.FlagUsages(), usage) + require.Contains(t, flagset.FlagUsages(), fmt.Sprintf("Consumes $%s", env)) + }) + + t.Run("IntEnvVar", func(t *testing.T) { + var ptr int + flagset, name, shorthand, env, usage := randomFlag() + envValue, _ := cryptorand.Int63n(10) + t.Setenv(env, strconv.FormatUint(uint64(envValue), 10)) + def, _ := cryptorand.Int() + + cliflag.IntVarP(flagset, &ptr, name, shorthand, env, def, usage) + got, err := flagset.GetInt(name) + require.NoError(t, err) + require.Equal(t, int(envValue), got) + }) + + t.Run("IntFailParse", func(t *testing.T) { + var ptr int + flagset, name, shorthand, env, usage := randomFlag() + envValue, _ := cryptorand.String(10) + t.Setenv(env, envValue) + def, _ := cryptorand.Int63n(10) + + cliflag.IntVarP(flagset, &ptr, name, shorthand, env, int(def), usage) + got, err := flagset.GetInt(name) + require.NoError(t, err) + require.Equal(t, int(def), got) + }) + t.Run("BoolDefault", func(t *testing.T) { var ptr bool flagset, name, shorthand, env, usage := randomFlag() diff --git a/cli/server.go b/cli/server.go index b3e01cb0412ea..03589c1a0a01d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -74,6 +74,12 @@ func server() *cobra.Command { accessURL string address string autobuildPollInterval time.Duration + derpServerEnabled bool + derpServerRegionID int + derpServerRegionCode string + derpServerRegionName string + derpServerSTUNURLs []string + derpConfigURL string promEnabled bool promAddress string pprofEnabled bool @@ -291,8 +297,8 @@ func server() *cobra.Command { Database: databasefake.New(), DERPMap: &tailcfg.DERPMap{ Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, + derpServerRegionID: { + RegionID: derpServerRegionID, RegionCode: "coder", RegionName: "Coder", Nodes: []*tailcfg.DERPNode{{ @@ -735,6 +741,14 @@ func server() *cobra.Command { cliflag.DurationVarP(root.Flags(), &autobuildPollInterval, "autobuild-poll-interval", "", "CODER_AUTOBUILD_POLL_INTERVAL", time.Minute, "Specifies the interval at which to poll for and execute automated workspace build operations.") cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.") cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.") + cliflag.StringVarP(root.Flags(), &derpConfigURL, "derp-config-url", "", "CODER_DERP_CONFIG_URL", "", "Specifies a URL to periodically fetch a DERP map. See: https://tailscale.com/kb/1118/custom-derp-servers/") + cliflag.BoolVarP(root.Flags(), &derpServerEnabled, "derp-server-enable", "", "CODER_DERP_SERVER_ENABLE", true, "Specifies whether to enable or disable the embedded DERP server.") + cliflag.IntVarP(root.Flags(), &derpServerRegionID, "derp-server-region-id", "", "CODER_DERP_SERVER_REGION_ID", 999, "Specifies the region ID to use for the embedded DERP server.") + cliflag.StringVarP(root.Flags(), &derpServerRegionCode, "derp-server-region-code", "", "CODER_DERP_SERVER_REGION_CODE", "coder", "Specifies the region code that is displayed in the Coder UI for the embedded DERP server.") + cliflag.StringVarP(root.Flags(), &derpServerRegionName, "derp-server-region-name", "", "CODER_DERP_SERVER_REGION_NAME", "Coder Embedded DERP", "Specifies the region name that is displayed in the Coder UI for the embedded DERP server.") + cliflag.StringArrayVarP(root.Flags(), &derpServerSTUNURLs, "derp-server-stun-urls", "", "CODER_DERP_SERVER_STUN_URLS", []string{ + "stun.l.google.com:19302", + }, "Specify URLs for STUN servers to establish P2P connections. Set empty to disable P2P connections entirely.") cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") cliflag.BoolVarP(root.Flags(), &pprofEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.") From 92f0c93952d8bad7e62530878e48b7b7ae48926e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 19:32:30 +0000 Subject: [PATCH 24/54] Fix Tailnet test --- agent/agent.go | 16 +++++++++++----- agent/agent_test.go | 16 +++++++++++++--- tailnet/coordinator.go | 4 ---- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 55cfd31e9506c..30e446903b549 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -60,7 +60,6 @@ var ( ) type Options struct { - EnableTailnet bool CoordinatorDialer CoordinatorDialer WebRTCDialer WebRTCDialer FetchMetadata FetchMetadata @@ -98,7 +97,6 @@ func New(options Options) io.Closer { closeCancel: cancelFunc, closed: make(chan struct{}), envVars: options.EnvironmentVariables, - enableTailnet: options.EnableTailnet, coordinatorDialer: options.CoordinatorDialer, fetchMetadata: options.FetchMetadata, } @@ -124,7 +122,6 @@ type agent struct { fetchMetadata FetchMetadata sshServer *ssh.Server - enableTailnet bool network *tailnet.Conn coordinatorDialer CoordinatorDialer } @@ -169,12 +166,15 @@ func (a *agent) run(ctx context.Context) { }() go a.runWebRTCNetworking(ctx) - if a.enableTailnet { + if metadata.DERPMap != nil { go a.runTailnet(ctx, metadata.DERPMap) } } func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { + if a.network != nil { + return + } var err error a.network, err = tailnet.NewConn(&tailnet.Options{ Addresses: []netip.Prefix{netip.PrefixFrom(tailnetIP, 128)}, @@ -243,6 +243,12 @@ func (a *agent) runCoordinator(ctx context.Context) { a.logger.Info(context.Background(), "connected to coordination server") break } + select { + case <-ctx.Done(): + return + default: + } + defer coordinator.Close() sendNodes, errChan := tailnet.ServeCoordinator(coordinator, a.network.UpdateNodes) a.network.SetNodeCallback(sendNodes) select { @@ -736,7 +742,7 @@ func (a *agent) handleReconnectingPTY(ctx context.Context, msg reconnectingPTYIn }() } // Resize the PTY to initial height + width. - err := rpty.ptty.Resize(uint16(msg.Height), uint16(msg.Width)) + err := rpty.ptty.Resize(msg.Height, msg.Width) if err != nil { // We can continue after this, it's not fatal! a.logger.Error(ctx, "resize reconnecting pty", slog.F("id", msg.ID), slog.Error(err)) diff --git a/agent/agent_test.go b/agent/agent_test.go index 20ace9a7dedf5..fd9d15e9ad06e 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -433,7 +433,7 @@ func TestAgent(t *testing.T) { require.Nil(t, netConn) }) - t.Run("Tailscale", func(t *testing.T) { + t.Run("Tailnet", func(t *testing.T) { t.Parallel() derpMap := runDERPAndStun(t, tailnet.Logger(slogtest.Make(t, nil))) conn := setupSSHSession(t, agent.Metadata{ @@ -481,6 +481,9 @@ func setupSSHCommand(t *testing.T, beforeArgs []string, afterArgs []string) *exe func setupSSHSession(t *testing.T, options agent.Metadata) *ssh.Session { sshClient, err := setupAgent(t, options, 0).SSHClient() require.NoError(t, err) + t.Cleanup(func() { + _ = sshClient.Close() + }) session, err := sshClient.NewSession() require.NoError(t, err) return session @@ -501,10 +504,13 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) }, CoordinatorDialer: func(ctx context.Context) (net.Conn, error) { clientConn, serverConn := net.Pipe() + t.Cleanup(func() { + _ = serverConn.Close() + _ = clientConn.Close() + }) go coordinator.ServeAgent(serverConn, agentID) return clientConn, nil }, - EnableTailnet: tailscale, Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug), ReconnectingPTYTimeout: ptyTimeout, }) @@ -523,8 +529,12 @@ func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) Logger: slogtest.Make(t, nil).Named("tailnet"), }) require.NoError(t, err) - clientConn, serverConn := net.Pipe() + t.Cleanup(func() { + _ = clientConn.Close() + _ = serverConn.Close() + _ = conn.Close() + }) go coordinator.ServeClient(serverConn, uuid.New(), agentID) sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error { return conn.UpdateNodes(node) diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index ef55712128a2f..e6ba976f3d05f 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -37,10 +37,6 @@ func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func return } _, err = conn.Write(data) - if errors.Is(err, io.EOF) { - errChan <- nil - return - } if err != nil { errChan <- xerrors.Errorf("write: %w", err) } From d9780246e9a3b3384228f08719cf20184cb8352e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 19:33:00 +0000 Subject: [PATCH 25/54] Rename ConnCoordinator to TailnetCoordinator --- coderd/coderd.go | 8 ++++---- coderd/provisionerjobs.go | 2 +- coderd/workspaceagents.go | 14 +++++++------- coderd/workspaceresources.go | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index cbeb47974e9f2..f658d4035a832 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -72,8 +72,8 @@ type Options struct { TURNServer *turnconn.Server TracerProvider *sdktrace.TracerProvider - ConnCoordinator *tailnet.Coordinator - DERPMap *tailcfg.DERPMap + TailnetCoordinator *tailnet.Coordinator + DERPMap *tailcfg.DERPMap } // New constructs a Coder API handler. @@ -100,8 +100,8 @@ func New(options *Options) *API { if options.PrometheusRegistry == nil { options.PrometheusRegistry = prometheus.NewRegistry() } - if options.ConnCoordinator == nil { - options.ConnCoordinator = tailnet.NewCoordinator() + if options.TailnetCoordinator == nil { + options.TailnetCoordinator = tailnet.NewCoordinator() } siteCacheDir := options.CacheDir diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index fad540c5dc2d9..ddae2b188f812 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -264,7 +264,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request, } } - apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading job agent.", diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 26bda735fc844..f1dff4bc1cddd 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -47,7 +47,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) { }) return } - apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -71,7 +71,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -118,7 +118,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) { workspaceAgent := httpmw.WorkspaceAgent(r) - apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -369,7 +369,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(api.ConnCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -458,7 +458,7 @@ func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (a }) conn.SetNodeCallback(sendNodes) go func() { - err := api.ConnCoordinator.ServeClient(serverConn, uuid.New(), agentID) + err := api.TailnetCoordinator.ServeClient(serverConn, uuid.New(), agentID) if err != nil { _ = conn.Close() } @@ -563,7 +563,7 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request return } defer conn.Close(websocket.StatusNormalClosure, "") - err = api.ConnCoordinator.ServeAgent(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), workspaceAgent.ID) + err = api.TailnetCoordinator.ServeAgent(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return @@ -589,7 +589,7 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R return } defer conn.Close(websocket.StatusNormalClosure, "") - err = api.ConnCoordinator.ServeClient(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID) + err = api.TailnetCoordinator.ServeClient(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), uuid.New(), workspaceAgent.ID) if err != nil { _ = conn.Close(websocket.StatusInternalError, err.Error()) return diff --git a/coderd/workspaceresources.go b/coderd/workspaceresources.go index 84a8828d1eaf7..8694fa47050ae 100644 --- a/coderd/workspaceresources.go +++ b/coderd/workspaceresources.go @@ -70,7 +70,7 @@ func (api *API) workspaceResource(rw http.ResponseWriter, r *http.Request) { } } - convertedAgent, err := convertWorkspaceAgent(api.ConnCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + convertedAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", From f9048959137531202f71146e0a0e0fdbf913db7b Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 19:36:55 +0000 Subject: [PATCH 26/54] Remove print statement from workspace agent test --- coderd/workspaceagents_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index afa5775fbb7d6..8a32ac823f969 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -4,7 +4,6 @@ import ( "bufio" "context" "encoding/json" - "fmt" "runtime" "strings" "testing" @@ -306,7 +305,6 @@ func TestWorkspaceAgentTailnet(t *testing.T) { agentCloser := agent.New(agent.Options{ FetchMetadata: agentClient.WorkspaceAgentMetadata, WebRTCDialer: agentClient.ListenWorkspaceAgent, - EnableTailnet: true, CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) @@ -327,7 +325,7 @@ func TestWorkspaceAgentTailnet(t *testing.T) { _ = session.Close() _ = sshClient.Close() _ = conn.Close() - fmt.Printf("\n\n\n\nOutput: %s\n\n\n\n", output) + require.Equal(t, "test", strings.TrimSpace(string(output))) } func TestWorkspaceAgentPTY(t *testing.T) { From 412590fade943711295f7557946465a5c5171dd7 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 21:23:37 +0000 Subject: [PATCH 27/54] Refactor wsconncache to use tailnet --- .vscode/settings.json | 1 + agent/agent.go | 9 +- agent/agent_test.go | 55 +-------- coderd/wsconncache/wsconncache_test.go | 54 +++++---- tailnet/conn.go | 9 +- tailnet/conn_test.go | 145 +++++++++--------------- tailnet/coordinator.go | 2 +- tailnet/tailnettest/tailnettest.go | 63 ++++++++++ tailnet/tailnettest/tailnettest_test.go | 18 +++ 9 files changed, 182 insertions(+), 174 deletions(-) create mode 100644 tailnet/tailnettest/tailnettest.go create mode 100644 tailnet/tailnettest/tailnettest_test.go diff --git a/.vscode/settings.json b/.vscode/settings.json index 07f5b0ccf729b..ad0c340a019d7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -90,6 +90,7 @@ "tailcfg", "tailexchange", "tailnet", + "tailnettest", "Tailscale", "TCGETS", "tcpip", diff --git a/agent/agent.go b/agent/agent.go index 30e446903b549..c16e0637db470 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -165,13 +165,20 @@ func (a *agent) run(ctx context.Context) { } }() - go a.runWebRTCNetworking(ctx) + if a.webrtcDialer != nil { + go a.runWebRTCNetworking(ctx) + } if metadata.DERPMap != nil { go a.runTailnet(ctx, metadata.DERPMap) } } func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { + a.closeMutex.Lock() + defer a.closeMutex.Unlock() + if a.isClosed() { + return + } if a.network != nil { return } diff --git a/agent/agent_test.go b/agent/agent_test.go index fd9d15e9ad06e..211cd9da347d0 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -3,13 +3,10 @@ package agent_test import ( "bufio" "context" - "crypto/tls" "encoding/json" "fmt" "io" "net" - "net/http" - "net/http/httptest" "net/netip" "os" "os/exec" @@ -21,13 +18,6 @@ import ( "time" "golang.org/x/xerrors" - "tailscale.com/derp" - "tailscale.com/derp/derphttp" - "tailscale.com/net/stun/stuntest" - "tailscale.com/tailcfg" - "tailscale.com/types/key" - tslogger "tailscale.com/types/logger" - "tailscale.com/types/nettype" scp "github.com/bramvdbogaerde/go-scp" "github.com/google/uuid" @@ -50,6 +40,7 @@ import ( "github.com/coder/coder/provisionersdk" "github.com/coder/coder/pty/ptytest" "github.com/coder/coder/tailnet" + "github.com/coder/coder/tailnet/tailnettest" "github.com/coder/coder/testutil" ) @@ -435,7 +426,7 @@ func TestAgent(t *testing.T) { t.Run("Tailnet", func(t *testing.T) { t.Parallel() - derpMap := runDERPAndStun(t, tailnet.Logger(slogtest.Make(t, nil))) + derpMap := tailnettest.RunDERPAndSTUN(t) conn := setupSSHSession(t, agent.Metadata{ DERPMap: derpMap, }) @@ -588,45 +579,3 @@ func assertWritePayload(t *testing.T, w io.Writer, payload []byte) { assert.NoError(t, err, "write payload") assert.Equal(t, len(payload), n, "payload length does not match") } - -func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) { - d := derp.NewServer(key.NewNode(), logf) - server := httptest.NewUnstartedServer(derphttp.Handler(d)) - server.Config.ErrorLog = tslogger.StdLogger(logf) - server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) - server.StartTLS() - - stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) - t.Cleanup(func() { - server.CloseClientConnections() - server.Close() - d.Close() - stunCleanup() - }) - - tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) - if !ok { - t.FailNow() - } - - return &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, - RegionCode: "test", - RegionName: "Testlandia", - Nodes: []*tailcfg.DERPNode{ - { - Name: "t2", - RegionID: 1, - IPv4: "127.0.0.1", - IPv6: "none", - STUNPort: stunAddr.Port, - DERPPort: tcpAddr.Port, - InsecureForTests: true, - }, - }, - }, - }, - } -} diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index bc990289ba469..f7a0938893ab6 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -7,13 +7,13 @@ import ( "net/http" "net/http/httptest" "net/http/httputil" + "net/netip" "net/url" "sync" "testing" "time" "github.com/google/uuid" - "github.com/pion/webrtc/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -23,10 +23,8 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/agent" "github.com/coder/coder/coderd/wsconncache" - "github.com/coder/coder/peer" - "github.com/coder/coder/peerbroker" - "github.com/coder/coder/peerbroker/proto" - "github.com/coder/coder/provisionersdk" + "github.com/coder/coder/tailnet" + "github.com/coder/coder/tailnet/tailnettest" ) func TestMain(m *testing.M) { @@ -41,7 +39,9 @@ func TestCache(t *testing.T) { return setupAgent(t, agent.Metadata{}, 0), nil }, 0) defer func() { + fmt.Printf("Closing cache...\n") _ = cache.Close() + fmt.Printf("Closed cache...\n") }() conn1, _, err := cache.Acquire(httptest.NewRequest(http.MethodGet, "/", nil), uuid.Nil) require.NoError(t, err) @@ -140,39 +140,47 @@ func TestCache(t *testing.T) { } func setupAgent(t *testing.T, metadata agent.Metadata, ptyTimeout time.Duration) agent.Conn { - client, server := provisionersdk.TransportPipe() + metadata.DERPMap = tailnettest.RunDERPAndSTUN(t) + + coordinator := tailnet.NewCoordinator() + agentID := uuid.New() closer := agent.New(agent.Options{ FetchMetadata: func(ctx context.Context) (agent.Metadata, error) { return metadata, nil }, - WebRTCDialer: func(ctx context.Context, logger slog.Logger) (*peerbroker.Listener, error) { - return peerbroker.Listen(server, func(ctx context.Context) ([]webrtc.ICEServer, *peer.ConnOptions, error) { - return nil, &peer.ConnOptions{ - Logger: slogtest.Make(t, nil).Named("server").Leveled(slog.LevelDebug), - }, nil + CoordinatorDialer: func(ctx context.Context) (net.Conn, error) { + clientConn, serverConn := net.Pipe() + t.Cleanup(func() { + _ = serverConn.Close() + _ = clientConn.Close() }) + go coordinator.ServeAgent(serverConn, agentID) + return clientConn, nil }, - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelInfo), ReconnectingPTYTimeout: ptyTimeout, }) t.Cleanup(func() { - _ = client.Close() - _ = server.Close() _ = closer.Close() }) - api := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) - stream, err := api.NegotiateConnection(context.Background()) - assert.NoError(t, err) - conn, err := peerbroker.Dial(stream, []webrtc.ICEServer{}, &peer.ConnOptions{ - Logger: slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug), + conn, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, + DERPMap: metadata.DERPMap, + Logger: slogtest.Make(t, nil).Named("tailnet").Leveled(slog.LevelDebug), }) require.NoError(t, err) + clientConn, serverConn := net.Pipe() t.Cleanup(func() { + _ = clientConn.Close() + _ = serverConn.Close() _ = conn.Close() }) - - return &agent.WebRTCConn{ - Negotiator: api, - Conn: conn, + go coordinator.ServeClient(serverConn, uuid.New(), agentID) + sendNode, _ := tailnet.ServeCoordinator(clientConn, func(node []*tailnet.Node) error { + return conn.UpdateNodes(node) + }) + conn.SetNodeCallback(sendNode) + return &agent.TailnetConn{ + Conn: conn, } } diff --git a/tailnet/conn.go b/tailnet/conn.go index 9984c41f1ad00..f37ef339f2b4f 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -237,6 +237,7 @@ type Conn struct { wireguardEngine wgengine.Engine listeners map[listenKey]*listener + lastMutex sync.Mutex // It's only possible to store these values via status functions, // so the values must be stored for retrieval later on. lastEndpoints []string @@ -261,10 +262,10 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { } } c.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) { - c.mutex.Lock() - defer c.mutex.Unlock() + c.lastMutex.Lock() c.lastPreferredDERP = ni.PreferredDERP c.lastDERPLatency = ni.DERPLatency + c.lastMutex.Unlock() callback(makeNode()) }) c.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { @@ -272,9 +273,9 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { for _, addr := range s.LocalAddrs { endpoints = append(endpoints, addr.Addr.String()) } - c.mutex.Lock() - defer c.mutex.Unlock() + c.lastMutex.Lock() c.lastEndpoints = endpoints + c.lastMutex.Unlock() callback(makeNode()) }) } diff --git a/tailnet/conn_test.go b/tailnet/conn_test.go index da7b798119bd5..32c3083604ae5 100644 --- a/tailnet/conn_test.go +++ b/tailnet/conn_test.go @@ -2,27 +2,17 @@ package tailnet_test import ( "context" - "crypto/tls" - "net" - "net/http" - "net/http/httptest" "net/netip" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" - "tailscale.com/derp" - "tailscale.com/derp/derphttp" - "tailscale.com/net/stun/stuntest" - "tailscale.com/tailcfg" - "tailscale.com/types/key" - tslogger "tailscale.com/types/logger" - "tailscale.com/types/nettype" "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/tailnet" + "github.com/coder/coder/tailnet/tailnettest" ) func TestMain(m *testing.M) { @@ -32,91 +22,62 @@ func TestMain(m *testing.M) { func TestTailnet(t *testing.T) { t.Parallel() logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) - derpMap := runDERPAndStun(t, tailnet.Logger(logger.Named("derp"))) - - w1IP := tailnet.IP() - w1, err := tailnet.NewConn(&tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, - Logger: logger.Named("w1"), - DERPMap: derpMap, - }) - require.NoError(t, err) - - w2, err := tailnet.NewConn(&tailnet.Options{ - Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, - Logger: logger.Named("w2"), - DERPMap: derpMap, - }) - require.NoError(t, err) - t.Cleanup(func() { - _ = w1.Close() - _ = w2.Close() - }) - w1.SetNodeCallback(func(node *tailnet.Node) { - w2.UpdateNodes([]*tailnet.Node{node}) - }) - w2.SetNodeCallback(func(node *tailnet.Node) { - w1.UpdateNodes([]*tailnet.Node{node}) + derpMap := tailnettest.RunDERPAndSTUN(t) + t.Run("InstantClose", func(t *testing.T) { + t.Parallel() + conn, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, + Logger: logger.Named("w1"), + DERPMap: derpMap, + }) + require.NoError(t, err) + err = conn.Close() + require.NoError(t, err) }) + t.Run("Connect", func(t *testing.T) { + t.Parallel() + w1IP := tailnet.IP() + w1, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)}, + Logger: logger.Named("w1"), + DERPMap: derpMap, + }) + require.NoError(t, err) - conn := make(chan struct{}) - go func() { - listener, err := w1.Listen("tcp", ":35565") - assert.NoError(t, err) - defer listener.Close() - nc, err := listener.Accept() - assert.NoError(t, err) - _ = nc.Close() - conn <- struct{}{} - }() - - nc, err := w2.DialContextTCP(context.Background(), netip.AddrPortFrom(w1IP, 35565)) - require.NoError(t, err) - _ = nc.Close() - <-conn + w2, err := tailnet.NewConn(&tailnet.Options{ + Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)}, + Logger: logger.Named("w2"), + DERPMap: derpMap, + }) + require.NoError(t, err) + t.Cleanup(func() { + _ = w1.Close() + _ = w2.Close() + }) + w1.SetNodeCallback(func(node *tailnet.Node) { + w2.UpdateNodes([]*tailnet.Node{node}) + }) + w2.SetNodeCallback(func(node *tailnet.Node) { + w1.UpdateNodes([]*tailnet.Node{node}) + }) - w1.Close() - w2.Close() -} + conn := make(chan struct{}) + go func() { + listener, err := w1.Listen("tcp", ":35565") + assert.NoError(t, err) + defer listener.Close() + nc, err := listener.Accept() + assert.NoError(t, err) + _ = nc.Close() + conn <- struct{}{} + }() -func runDERPAndStun(t *testing.T, logf tslogger.Logf) (derpMap *tailcfg.DERPMap) { - d := derp.NewServer(key.NewNode(), logf) - server := httptest.NewUnstartedServer(derphttp.Handler(d)) - server.Config.ErrorLog = tslogger.StdLogger(logf) - server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) - server.StartTLS() + nc, err := w2.DialContextTCP(context.Background(), netip.AddrPortFrom(w1IP, 35565)) + require.NoError(t, err) + _ = nc.Close() + <-conn - stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) - t.Cleanup(func() { - server.CloseClientConnections() - server.Close() - d.Close() - stunCleanup() + w1.Close() + w2.Close() }) - - tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) - if !ok { - t.FailNow() - } - - return &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - 1: { - RegionID: 1, - RegionCode: "test", - RegionName: "Testlandia", - Nodes: []*tailcfg.DERPNode{ - { - Name: "t2", - RegionID: 1, - IPv4: "127.0.0.1", - IPv6: "none", - STUNPort: stunAddr.Port, - DERPPort: tcpAddr.Port, - InsecureForTests: true, - }, - }, - }, - }, - } } diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index e6ba976f3d05f..5decbd0966cd9 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -13,7 +13,7 @@ import ( // ServeCoordinator matches the RW structure of a coordinator to exchange node messages. func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) { - errChan := make(chan error, 1) + errChan := make(chan error, 3) go func() { decoder := json.NewDecoder(conn) for { diff --git a/tailnet/tailnettest/tailnettest.go b/tailnet/tailnettest/tailnettest.go new file mode 100644 index 0000000000000..ce020bc0f42bc --- /dev/null +++ b/tailnet/tailnettest/tailnettest.go @@ -0,0 +1,63 @@ +package tailnettest + +import ( + "crypto/tls" + "net" + "net/http" + "net/http/httptest" + "testing" + + "tailscale.com/derp" + "tailscale.com/derp/derphttp" + "tailscale.com/net/stun/stuntest" + "tailscale.com/tailcfg" + "tailscale.com/types/key" + tslogger "tailscale.com/types/logger" + "tailscale.com/types/nettype" + + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/tailnet" +) + +// RunDERPAndSTUN creates a DERP mapping for tests. +func RunDERPAndSTUN(t *testing.T) *tailcfg.DERPMap { + logf := tailnet.Logger(slogtest.Make(t, nil)) + d := derp.NewServer(key.NewNode(), logf) + server := httptest.NewUnstartedServer(derphttp.Handler(d)) + server.Config.ErrorLog = tslogger.StdLogger(logf) + server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + server.StartTLS() + + stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{}) + t.Cleanup(func() { + server.CloseClientConnections() + server.Close() + d.Close() + stunCleanup() + }) + tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr) + if !ok { + t.FailNow() + } + + return &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: { + RegionID: 1, + RegionCode: "test", + RegionName: "Test", + Nodes: []*tailcfg.DERPNode{ + { + Name: "t2", + RegionID: 1, + IPv4: "127.0.0.1", + IPv6: "none", + STUNPort: stunAddr.Port, + DERPPort: tcpAddr.Port, + InsecureForTests: true, + }, + }, + }, + }, + } +} diff --git a/tailnet/tailnettest/tailnettest_test.go b/tailnet/tailnettest/tailnettest_test.go new file mode 100644 index 0000000000000..aebb018a9bcb2 --- /dev/null +++ b/tailnet/tailnettest/tailnettest_test.go @@ -0,0 +1,18 @@ +package tailnettest_test + +import ( + "testing" + + "go.uber.org/goleak" + + "github.com/coder/coder/tailnet/tailnettest" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} + +func TestRunDERPAndSTUN(t *testing.T) { + t.Parallel() + _ = tailnettest.RunDERPAndSTUN(t) +} From 95681d843a00c27bbdffeb12a3ce9eae3b5998d8 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 21:27:47 +0000 Subject: [PATCH 28/54] Remove STUN from unit tests --- coderd/coderdtest/coderdtest.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index d0a54ebb91a20..bbfc6bf4d8918 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -217,12 +217,6 @@ func newWithCloser(t *testing.T, options *Options) (*codersdk.Client, io.Closer) STUNPort: -1, InsecureForTests: true, HTTPForTests: true, - }, { - Name: "1b", - RegionID: 1, - STUNOnly: true, - HostName: "stun.l.google.com", - STUNPort: 19302, }}, }, }, From c6049676cdde89036811aba87f559fa88f0d0c2f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 21:29:51 +0000 Subject: [PATCH 29/54] Add migrate back to dump --- coderd/database/dump/main.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/coderd/database/dump/main.go b/coderd/database/dump/main.go index 10cdddb6ef811..20c4ac0c2e30a 100644 --- a/coderd/database/dump/main.go +++ b/coderd/database/dump/main.go @@ -2,12 +2,14 @@ package main import ( "bytes" + "database/sql" "fmt" "os" "os/exec" "path/filepath" "runtime" + "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/database/postgres" ) @@ -18,6 +20,16 @@ func main() { } defer closeFn() + db, err := sql.Open("postgres", connection) + if err != nil { + panic(err) + } + + err = database.MigrateUp(db) + if err != nil { + panic(err) + } + cmd := exec.Command( "docker", "run", From d24d022e0e0f870503302dbbe5fc6b12bc20c798 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 21 Aug 2022 21:56:29 +0000 Subject: [PATCH 30/54] chore: Upgrade to Go 1.19 This is required as part of #3505. --- .github/workflows/coder.yaml | 16 ++++++++-------- .vscode/settings.json | 3 +++ agent/reaper/reaper_test.go | 1 + cli/cliflag/cliflag.go | 3 +-- cli/cliflag/cliflag_test.go | 1 + cli/configssh.go | 2 +- cli/delete_test.go | 1 + cli/gitssh_test.go | 1 + cli/publickey_test.go | 1 + cli/root_test.go | 1 + cli/server.go | 11 ++++++++--- cli/server_test.go | 7 +++++-- cli/userstatus_test.go | 3 +-- coderd/audit/diff.go | 1 + coderd/audit/diff_test.go | 1 + coderd/authorize.go | 1 + coderd/autobuild/notify/notifier.go | 20 ++++++++++---------- coderd/autobuild/schedule/schedule.go | 25 +++++++++++++------------ coderd/azureidentity/azureidentity.go | 2 ++ coderd/database/pubsub_test.go | 7 +------ coderd/devtunnel/tunnel_test.go | 11 ++++++++--- coderd/features_internal_test.go | 1 + coderd/httpmw/authorize_test.go | 4 +++- coderd/httpmw/oauth2_test.go | 1 + coderd/httpmw/prometheus_test.go | 1 + coderd/httpmw/ratelimit_test.go | 4 +++- coderd/pagination_internal_test.go | 3 ++- coderd/provisionerjobs_internal_test.go | 1 + coderd/rbac/authz.go | 1 + coderd/rbac/authz_internal_test.go | 3 ++- coderd/rbac/builtin.go | 4 +++- coderd/rbac/builtin_test.go | 2 ++ coderd/roles_test.go | 2 +- coderd/telemetry/telemetry.go | 2 ++ coderd/templateversions_test.go | 2 +- coderd/userauth_test.go | 6 ++++-- coderd/users_test.go | 8 ++++---- coderd/workspaceapps_test.go | 2 ++ coderd/workspaces_test.go | 2 +- coderd/wsconncache/wsconncache_test.go | 6 ++++-- cryptorand/errors_test.go | 1 + cryptorand/numbers.go | 1 + dogfood/Dockerfile | 25 ++++++++++++------------- go.mod | 2 +- peer/conn_test.go | 1 + peerbroker/proxy.go | 13 ++++++++----- provisioner/terraform/provision_test.go | 3 +-- scripts/apitypings/main.go | 1 + site/site_test.go | 1 + 49 files changed, 136 insertions(+), 86 deletions(-) diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index c20411005d6c5..f64424d98c0d1 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -96,11 +96,11 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: golangci-lint uses: golangci/golangci-lint-action@v3.2.0 with: - version: v1.46.0 + version: v1.48.0 check-enterprise-imports: name: check/enterprise-imports @@ -199,7 +199,7 @@ jobs: version: "3.20.0" - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -283,7 +283,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -370,7 +370,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -453,7 +453,7 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - name: Echo Go Cache Paths id: go-cache-paths @@ -558,7 +558,7 @@ jobs: # Go is required for uploading the test results to datadog - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - uses: actions/setup-node@v3 with: @@ -616,7 +616,7 @@ jobs: # Go is required for uploading the test results to datadog - uses: actions/setup-go@v3 with: - go-version: "~1.18" + go-version: "~1.19" - uses: hashicorp/setup-terraform@v2 with: diff --git a/.vscode/settings.json b/.vscode/settings.json index f4d9255c0be80..81965c42613bd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "cSpell.words": [ "apps", "awsidentity", + "bodyclose", "buildinfo", "buildname", "circbuf", @@ -52,6 +53,7 @@ "ntqry", "OIDC", "oneof", + "paralleltest", "parameterscopeid", "pqtype", "prometheusmetrics", @@ -80,6 +82,7 @@ "tfjson", "tfplan", "tfstate", + "tparallel", "trimprefix", "turnconn", "typegen", diff --git a/agent/reaper/reaper_test.go b/agent/reaper/reaper_test.go index 88573636ee39c..867bcfa5749bc 100644 --- a/agent/reaper/reaper_test.go +++ b/agent/reaper/reaper_test.go @@ -29,6 +29,7 @@ func TestReap(t *testing.T) { // exited processes and passing the PIDs through the shared // channel. t.Run("OK", func(t *testing.T) { + t.Parallel() pids := make(reap.PidCh, 1) err := reaper.ForkReap( reaper.WithPIDCallback(pids), diff --git a/cli/cliflag/cliflag.go b/cli/cliflag/cliflag.go index 053ea948f04cf..843416c3ff3ea 100644 --- a/cli/cliflag/cliflag.go +++ b/cli/cliflag/cliflag.go @@ -6,8 +6,7 @@ // // Will produce the following usage docs: // -// -a, --address string The address to serve the API and dashboard (uses $CODER_ADDRESS). (default "127.0.0.1:3000") -// +// -a, --address string The address to serve the API and dashboard (uses $CODER_ADDRESS). (default "127.0.0.1:3000") package cliflag import ( diff --git a/cli/cliflag/cliflag_test.go b/cli/cliflag/cliflag_test.go index a5c9f532abe79..acdf7d6765fb5 100644 --- a/cli/cliflag/cliflag_test.go +++ b/cli/cliflag/cliflag_test.go @@ -14,6 +14,7 @@ import ( ) // Testcliflag cannot run in parallel because it uses t.Setenv. +// //nolint:paralleltest func TestCliflag(t *testing.T) { t.Run("StringDefault", func(t *testing.T) { diff --git a/cli/configssh.go b/cli/configssh.go index 08aeaf6c9d37a..a3f9a4517c8e0 100644 --- a/cli/configssh.go +++ b/cli/configssh.go @@ -558,7 +558,7 @@ func currentBinPath(w io.Writer) (string, error) { // diffBytes takes two byte slices and diffs them as if they were in a // file named name. -//nolint: revive // Color is an option, not a control coupling. +// nolint: revive // Color is an option, not a control coupling. func diffBytes(name string, b1, b2 []byte, color bool) ([]byte, error) { var buf bytes.Buffer var opts []write.Option diff --git a/cli/delete_test.go b/cli/delete_test.go index c8039d454f93e..a2ea15fd18121 100644 --- a/cli/delete_test.go +++ b/cli/delete_test.go @@ -15,6 +15,7 @@ import ( ) func TestDelete(t *testing.T) { + t.Parallel() t.Run("WithParameter", func(t *testing.T) { t.Parallel() client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true}) diff --git a/cli/gitssh_test.go b/cli/gitssh_test.go index ff15e145e4e64..4a6e571502754 100644 --- a/cli/gitssh_test.go +++ b/cli/gitssh_test.go @@ -22,6 +22,7 @@ import ( func TestGitSSH(t *testing.T) { t.Parallel() t.Run("Dial", func(t *testing.T) { + t.Parallel() client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true}) user := coderdtest.CreateFirstUser(t, client) diff --git a/cli/publickey_test.go b/cli/publickey_test.go index cd2ab3548650a..f0bef2c65359e 100644 --- a/cli/publickey_test.go +++ b/cli/publickey_test.go @@ -13,6 +13,7 @@ import ( func TestPublicKey(t *testing.T) { t.Parallel() t.Run("OK", func(t *testing.T) { + t.Parallel() client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) cmd, root := clitest.New(t, "publickey") diff --git a/cli/root_test.go b/cli/root_test.go index 5dc97bb060cf5..04a9b2c99ecda 100644 --- a/cli/root_test.go +++ b/cli/root_test.go @@ -15,6 +15,7 @@ import ( ) func TestRoot(t *testing.T) { + t.Parallel() t.Run("FormatCobraError", func(t *testing.T) { t.Parallel() diff --git a/cli/server.go b/cli/server.go index 126ea0bc8204b..9a9d464b4c1b8 100644 --- a/cli/server.go +++ b/cli/server.go @@ -475,8 +475,9 @@ func server() *cobra.Command { server := &http.Server{ // These errors are typically noise like "TLS: EOF". Vault does similar: // https://github.com/hashicorp/vault/blob/e2490059d0711635e529a4efcbaa1b26998d6e1c/command/server.go#L2714 - ErrorLog: log.New(io.Discard, "", 0), - Handler: coderAPI.Handler, + ErrorLog: log.New(io.Discard, "", 0), + Handler: coderAPI.Handler, + ReadHeaderTimeout: time.Minute, BaseContext: func(_ net.Listener) context.Context { return shutdownConnsCtx }, @@ -1080,7 +1081,11 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) { logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name)) - srv := &http.Server{Addr: addr, Handler: handler} + srv := &http.Server{ + Addr: addr, + Handler: handler, + ReadHeaderTimeout: time.Minute, + } go func() { err := srv.ListenAndServe() if err != nil && !xerrors.Is(err, http.ErrServerClosed) { diff --git a/cli/server_test.go b/cli/server_test.go index 66303339bbbdd..7dd732438114e 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -39,7 +39,7 @@ import ( ) // This cannot be ran in parallel because it uses a signal. -// nolint:paralleltest +// nolint:tparallel,paralleltest func TestServer(t *testing.T) { t.Run("Production", func(t *testing.T) { if runtime.GOOS != "linux" || testing.Short() { @@ -410,6 +410,7 @@ func TestServer(t *testing.T) { require.Eventually(t, func() bool { req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", randomPort), nil) assert.NoError(t, err) + // nolint:bodyclose res, err = http.DefaultClient.Do(req) return err == nil }, testutil.WaitShort, testutil.IntervalFast) @@ -461,7 +462,9 @@ func TestServer(t *testing.T) { } githubURL, err := accessURL.Parse("/api/v2/users/oauth2/github") require.NoError(t, err) - res, err := client.HTTPClient.Get(githubURL.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, githubURL.String(), nil) + require.NoError(t, err) + res, err := client.HTTPClient.Do(req) require.NoError(t, err) defer res.Body.Close() fakeURL, err := res.Location() diff --git a/cli/userstatus_test.go b/cli/userstatus_test.go index 700c08616d84b..aea26f8387228 100644 --- a/cli/userstatus_test.go +++ b/cli/userstatus_test.go @@ -12,6 +12,7 @@ import ( "github.com/coder/coder/codersdk" ) +// nolint:tparallel,paralleltest func TestUserStatus(t *testing.T) { t.Parallel() client := coderdtest.New(t, nil) @@ -20,7 +21,6 @@ func TestUserStatus(t *testing.T) { otherUser, err := other.User(context.Background(), codersdk.Me) require.NoError(t, err, "fetch user") - //nolint:paralleltest t.Run("StatusSelf", func(t *testing.T) { cmd, root := clitest.New(t, "users", "suspend", "me") clitest.SetupConfig(t, client, root) @@ -32,7 +32,6 @@ func TestUserStatus(t *testing.T) { require.ErrorContains(t, err, "cannot suspend yourself") }) - //nolint:paralleltest t.Run("StatusOther", func(t *testing.T) { require.Equal(t, otherUser.Status, codersdk.UserStatusActive, "start as active") diff --git a/coderd/audit/diff.go b/coderd/audit/diff.go index eb500891b99c4..efb8d87548f86 100644 --- a/coderd/audit/diff.go +++ b/coderd/audit/diff.go @@ -99,6 +99,7 @@ func diffValues[T any](left, right T, table Table) Map { } // convertDiffType converts external struct types to primitive types. +// //nolint:forcetypeassert func convertDiffType(left, right any) (newLeft, newRight any, changed bool) { switch typed := left.(type) { diff --git a/coderd/audit/diff_test.go b/coderd/audit/diff_test.go index fc9c41b7cb16d..0e90d8c30dcad 100644 --- a/coderd/audit/diff_test.go +++ b/coderd/audit/diff_test.go @@ -230,6 +230,7 @@ func runDiffTests[T audit.Auditable](t *testing.T, tests []diffTest[T]) { for _, test := range tests { t.Run(typName+"/"+test.name, func(t *testing.T) { + t.Parallel() require.Equal(t, test.exp, audit.Diff(test.left, test.right), diff --git a/coderd/authorize.go b/coderd/authorize.go index 855348aaf6184..981af5e868c36 100644 --- a/coderd/authorize.go +++ b/coderd/authorize.go @@ -31,6 +31,7 @@ func AuthorizeFilter[O rbac.Objecter](api *API, r *http.Request, action rbac.Act // This function will log appropriately, but the caller must return an // error to the api client. // Eg: +// // if !api.Authorize(...) { // httpapi.Forbidden(rw) // return diff --git a/coderd/autobuild/notify/notifier.go b/coderd/autobuild/notify/notifier.go index f7e1058aee3b9..e0db12af35475 100644 --- a/coderd/autobuild/notify/notifier.go +++ b/coderd/autobuild/notify/notifier.go @@ -20,14 +20,14 @@ type Notifier struct { } // Condition is a function that gets executed with a certain time. -// - It should return the deadline for the notification, as well as a -// callback function to execute once the time to the deadline is -// less than one of the notify attempts. If deadline is the zero -// time, callback will not be executed. -// - Callback is executed once for every time the difference between deadline -// and the current time is less than an element of countdown. -// - To enforce a minimum interval between consecutive callbacks, truncate -// the returned deadline to the minimum interval. +// - It should return the deadline for the notification, as well as a +// callback function to execute once the time to the deadline is +// less than one of the notify attempts. If deadline is the zero +// time, callback will not be executed. +// - Callback is executed once for every time the difference between deadline +// and the current time is less than an element of countdown. +// - To enforce a minimum interval between consecutive callbacks, truncate +// the returned deadline to the minimum interval. type Condition func(now time.Time) (deadline time.Time, callback func()) // Notify is a convenience function that initializes a new Notifier @@ -44,8 +44,8 @@ func Notify(cond Condition, interval time.Duration, countdown ...time.Duration) } // New returns a Notifier that calls cond once every time it polls. -// - Duplicate values are removed from countdown, and it is sorted in -// descending order. +// - Duplicate values are removed from countdown, and it is sorted in +// descending order. func New(cond Condition, countdown ...time.Duration) *Notifier { // Ensure countdown is sorted in descending order and contains no duplicates. ct := unique(countdown) diff --git a/coderd/autobuild/schedule/schedule.go b/coderd/autobuild/schedule/schedule.go index 43afcf2223a28..59433450c062a 100644 --- a/coderd/autobuild/schedule/schedule.go +++ b/coderd/autobuild/schedule/schedule.go @@ -28,13 +28,14 @@ var defaultParser = cron.NewParser(parserFormat) // - day of week e.g. 1 (required) // // Example Usage: -// local_sched, _ := schedule.Weekly("59 23 *") -// fmt.Println(sched.Next(time.Now().Format(time.RFC3339))) -// // Output: 2022-04-04T23:59:00Z // -// us_sched, _ := schedule.Weekly("CRON_TZ=US/Central 30 9 1-5") -// fmt.Println(sched.Next(time.Now()).Format(time.RFC3339)) -// // Output: 2022-04-04T14:30:00Z +// local_sched, _ := schedule.Weekly("59 23 *") +// fmt.Println(sched.Next(time.Now().Format(time.RFC3339))) +// // Output: 2022-04-04T23:59:00Z +// +// us_sched, _ := schedule.Weekly("CRON_TZ=US/Central 30 9 1-5") +// fmt.Println(sched.Next(time.Now()).Format(time.RFC3339)) +// // Output: 2022-04-04T14:30:00Z func Weekly(raw string) (*Schedule, error) { if err := validateWeeklySpec(raw); err != nil { return nil, xerrors.Errorf("validate weekly schedule: %w", err) @@ -115,12 +116,12 @@ var tMax = t0.Add(168 * time.Hour) // Min returns the minimum duration of the schedule. // This is calculated as follows: -// - Let t(0) be a given point in time (1970-01-01T01:01:01Z00:00) -// - Let t(max) be 168 hours after t(0). -// - Let t(1) be the next scheduled time after t(0). -// - Let t(n) be the next scheduled time after t(n-1). -// - Then, the minimum duration of s d(min) -// = min( t(n) - t(n-1) ∀ n ∈ N, t(n) < t(max) ) +// - Let t(0) be a given point in time (1970-01-01T01:01:01Z00:00) +// - Let t(max) be 168 hours after t(0). +// - Let t(1) be the next scheduled time after t(0). +// - Let t(n) be the next scheduled time after t(n-1). +// - Then, the minimum duration of s d(min) +// = min( t(n) - t(n-1) ∀ n ∈ N, t(n) < t(max) ) func (s Schedule) Min() time.Duration { durMin := tMax.Sub(t0) tPrev := s.Next(t0) diff --git a/coderd/azureidentity/azureidentity.go b/coderd/azureidentity/azureidentity.go index d80c60367d8f2..b1161edfc8278 100644 --- a/coderd/azureidentity/azureidentity.go +++ b/coderd/azureidentity/azureidentity.go @@ -52,8 +52,10 @@ func Validate(ctx context.Context, signature string, options x509.VerifyOptions) } data, err := io.ReadAll(res.Body) if err != nil { + _ = res.Body.Close() return "", xerrors.Errorf("read body %q: %w", certURL, err) } + _ = res.Body.Close() cert, err := x509.ParseCertificate(data) if err != nil { return "", xerrors.Errorf("parse certificate %q: %w", certURL, err) diff --git a/coderd/database/pubsub_test.go b/coderd/database/pubsub_test.go index 558b4e5650b07..c1377b69aa4ae 100644 --- a/coderd/database/pubsub_test.go +++ b/coderd/database/pubsub_test.go @@ -14,6 +14,7 @@ import ( "github.com/coder/coder/coderd/database/postgres" ) +// nolint:tparallel,paralleltest func TestPubsub(t *testing.T) { t.Parallel() @@ -22,10 +23,7 @@ func TestPubsub(t *testing.T) { return } - // nolint:paralleltest t.Run("Postgres", func(t *testing.T) { - // postgres.Open() seems to be creating race conditions when run in parallel. - // t.Parallel() ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() @@ -54,10 +52,7 @@ func TestPubsub(t *testing.T) { assert.Equal(t, string(message), data) }) - // nolint:paralleltest t.Run("PostgresCloseCancel", func(t *testing.T) { - // postgres.Open() seems to be creating race conditions when run in parallel. - // t.Parallel() ctx, cancelFunc := context.WithCancel(context.Background()) defer cancelFunc() connectionURL, closePg, err := postgres.Open() diff --git a/coderd/devtunnel/tunnel_test.go b/coderd/devtunnel/tunnel_test.go index 23dd92d966d54..9bf435ce3233a 100644 --- a/coderd/devtunnel/tunnel_test.go +++ b/coderd/devtunnel/tunnel_test.go @@ -52,6 +52,7 @@ func TestTunnel(t *testing.T) { defer cancelTun() server := http.Server{ + ReadHeaderTimeout: time.Minute, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Log("got request for", r.URL) // Going to use something _slightly_ exotic so that we can't accidentally get some @@ -100,8 +101,8 @@ func TestTunnel(t *testing.T) { // fakeTunnelServer is a fake version of the real dev tunnel server. It fakes 2 client interactions // that we want to test: -// 1. Responding to a POST /tun from the client -// 2. Sending an HTTP request down the wireguard connection +// 1. Responding to a POST /tun from the client +// 2. Sending an HTTP request down the wireguard connection // // Note that for 2, we don't implement a full proxy that accepts arbitrary requests, we just send // a test request over the Wireguard tunnel to make sure that we can listen. The proxy behavior is @@ -229,5 +230,9 @@ func (f *fakeTunnelServer) requestHTTP() (*http.Response, error) { Transport: transport, Timeout: testutil.WaitLong, } - return client.Get(fmt.Sprintf("http://[%s]:8090", clientIP)) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, fmt.Sprintf("http://[%s]:8090", clientIP), nil) + if err != nil { + return nil, err + } + return client.Do(req) } diff --git a/coderd/features_internal_test.go b/coderd/features_internal_test.go index d8480899c6d84..50c7e8f53e397 100644 --- a/coderd/features_internal_test.go +++ b/coderd/features_internal_test.go @@ -20,6 +20,7 @@ func TestEntitlements(t *testing.T) { rw := httptest.NewRecorder() entitlements(rw, r) resp := rw.Result() + defer resp.Body.Close() assert.Equal(t, http.StatusOK, resp.StatusCode) dec := json.NewDecoder(resp.Body) var result codersdk.Entitlements diff --git a/coderd/httpmw/authorize_test.go b/coderd/httpmw/authorize_test.go index 14ab9611066d0..dcae5bff96526 100644 --- a/coderd/httpmw/authorize_test.go +++ b/coderd/httpmw/authorize_test.go @@ -99,7 +99,9 @@ func TestExtractUserRoles(t *testing.T) { }) rtr.ServeHTTP(rw, req) - require.Equal(t, http.StatusOK, rw.Result().StatusCode) + resp := rw.Result() + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) }) } } diff --git a/coderd/httpmw/oauth2_test.go b/coderd/httpmw/oauth2_test.go index 771dcdd772f78..dd5c7c6bc7b35 100644 --- a/coderd/httpmw/oauth2_test.go +++ b/coderd/httpmw/oauth2_test.go @@ -32,6 +32,7 @@ func (*testOAuth2Provider) TokenSource(_ context.Context, _ *oauth2.Token) oauth return nil } +// nolint:bodyclose func TestOAuth2(t *testing.T) { t.Parallel() t.Run("NotSetup", func(t *testing.T) { diff --git a/coderd/httpmw/prometheus_test.go b/coderd/httpmw/prometheus_test.go index 1ab93cef06c71..97c557540674a 100644 --- a/coderd/httpmw/prometheus_test.go +++ b/coderd/httpmw/prometheus_test.go @@ -17,6 +17,7 @@ import ( func TestPrometheus(t *testing.T) { t.Parallel() t.Run("All", func(t *testing.T) { + t.Parallel() req := httptest.NewRequest("GET", "/", nil) req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, chi.NewRouteContext())) res := chimw.NewWrapResponseWriter(httptest.NewRecorder(), 0) diff --git a/coderd/httpmw/ratelimit_test.go b/coderd/httpmw/ratelimit_test.go index 132615d669ab7..3ed0178a1699a 100644 --- a/coderd/httpmw/ratelimit_test.go +++ b/coderd/httpmw/ratelimit_test.go @@ -26,7 +26,9 @@ func TestRateLimit(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) rec := httptest.NewRecorder() rtr.ServeHTTP(rec, req) - return rec.Result().StatusCode == http.StatusTooManyRequests + resp := rec.Result() + defer resp.Body.Close() + return resp.StatusCode == http.StatusTooManyRequests }, testutil.WaitShort, testutil.IntervalFast) }) } diff --git a/coderd/pagination_internal_test.go b/coderd/pagination_internal_test.go index 978cfab417b2e..d1e6bd107cfd8 100644 --- a/coderd/pagination_internal_test.go +++ b/coderd/pagination_internal_test.go @@ -1,6 +1,7 @@ package coderd import ( + "context" "encoding/json" "net/http" "net/http/httptest" @@ -93,7 +94,7 @@ func TestPagination(t *testing.T) { t.Run(c.Name, func(t *testing.T) { t.Parallel() rw := httptest.NewRecorder() - r, err := http.NewRequest("GET", "https://example.com", nil) + r, err := http.NewRequestWithContext(context.Background(), "GET", "https://example.com", nil) require.NoError(t, err, "new request") // Set query params diff --git a/coderd/provisionerjobs_internal_test.go b/coderd/provisionerjobs_internal_test.go index da593e68f91ce..80a94f88fcb1c 100644 --- a/coderd/provisionerjobs_internal_test.go +++ b/coderd/provisionerjobs_internal_test.go @@ -28,6 +28,7 @@ func TestProvisionerJobLogs_Unit(t *testing.T) { t.Parallel() t.Run("QueryPubSubDupes", func(t *testing.T) { + t.Parallel() logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) // mDB := mocks.NewStore(t) fDB := databasefake.New() diff --git a/coderd/rbac/authz.go b/coderd/rbac/authz.go index 4b99c29438bb4..d124aae29bc24 100644 --- a/coderd/rbac/authz.go +++ b/coderd/rbac/authz.go @@ -54,6 +54,7 @@ type RegoAuthorizer struct { } // Load the policy from policy.rego in this directory. +// //go:embed policy.rego var policy string diff --git a/coderd/rbac/authz_internal_test.go b/coderd/rbac/authz_internal_test.go index a3aca022c7ce8..b91130d0f4def 100644 --- a/coderd/rbac/authz_internal_test.go +++ b/coderd/rbac/authz_internal_test.go @@ -491,8 +491,8 @@ func TestAuthorizeDomain(t *testing.T) { } // TestAuthorizeLevels ensures level overrides are acting appropriately -//nolint:paralleltest func TestAuthorizeLevels(t *testing.T) { + t.Parallel() defOrg := uuid.New() unusedID := uuid.New() @@ -638,6 +638,7 @@ func testAuthorize(t *testing.T, name string, subject subject, sets ...[]authTes for _, cases := range sets { for _, c := range cases { t.Run(name, func(t *testing.T) { + t.Parallel() for _, a := range c.actions { ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) t.Cleanup(cancel) diff --git a/coderd/rbac/builtin.go b/coderd/rbac/builtin.go index bfa6ded0a5250..d2a52511df266 100644 --- a/coderd/rbac/builtin.go +++ b/coderd/rbac/builtin.go @@ -361,7 +361,9 @@ func ChangeRoleSet(from []string, to []string) (added []string, removed []string } // roleName is a quick helper function to return -// role_name:scopeID +// +// role_name:scopeID +// // If no scopeID is required, only 'role_name' is returned func roleName(name string, orgID string) string { if orgID == "" { diff --git a/coderd/rbac/builtin_test.go b/coderd/rbac/builtin_test.go index bbedc1e48527c..d357271943b8b 100644 --- a/coderd/rbac/builtin_test.go +++ b/coderd/rbac/builtin_test.go @@ -13,6 +13,7 @@ import ( ) // BenchmarkRBACFilter benchmarks the rbac.Filter method. +// // go test -bench BenchmarkRBACFilter -benchmem -memprofile memprofile.out -cpuprofile profile.out func BenchmarkRBACFilter(b *testing.B) { orgs := []uuid.UUID{ @@ -392,6 +393,7 @@ func TestIsOrgRole(t *testing.T) { // nolint:paralleltest for _, c := range testCases { t.Run(c.RoleName, func(t *testing.T) { + t.Parallel() orgID, ok := rbac.IsOrgRole(c.RoleName) require.Equal(t, c.OrgRole, ok, "match expected org role") require.Equal(t, c.OrgID, orgID, "match expected org id") diff --git a/coderd/roles_test.go b/coderd/roles_test.go index 034be6a6bb43a..27b5bc9df6fcf 100644 --- a/coderd/roles_test.go +++ b/coderd/roles_test.go @@ -112,7 +112,7 @@ func TestListRoles(t *testing.T) { orgAdmin := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleOrgAdmin(admin.OrganizationID)) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() + t.Cleanup(cancel) otherOrg, err := client.CreateOrganization(ctx, codersdk.CreateOrganizationRequest{ Name: "other", diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index d4b6d42db8de6..d44166635c0ff 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -133,6 +133,7 @@ func (r *remoteReporter) reportSync(snapshot *Snapshot) { r.options.Logger.Debug(r.ctx, "submit", slog.Error(err)) return } + defer resp.Body.Close() if resp.StatusCode != http.StatusAccepted { r.options.Logger.Debug(r.ctx, "bad response from telemetry server", slog.F("status", resp.StatusCode)) return @@ -261,6 +262,7 @@ func (r *remoteReporter) deployment() error { if err != nil { return xerrors.Errorf("perform request: %w", err) } + defer resp.Body.Close() if resp.StatusCode != http.StatusAccepted { return xerrors.Errorf("update deployment: %w", err) } diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index fa29c976de3ee..6187da3707a1f 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -818,7 +818,7 @@ func TestPaginatedTemplateVersions(t *testing.T) { // This test takes longer than a long time. ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong*2) - defer cancel() + t.Cleanup(cancel) // Populate database with template versions. total := 9 diff --git a/coderd/userauth_test.go b/coderd/userauth_test.go index f1768b588decb..a1e850a090217 100644 --- a/coderd/userauth_test.go +++ b/coderd/userauth_test.go @@ -76,6 +76,7 @@ func TestUserAuthMethods(t *testing.T) { }) } +// nolint:bodyclose func TestUserOAuth2Github(t *testing.T) { t.Parallel() t.Run("NotInAllowedOrganization", func(t *testing.T) { @@ -281,6 +282,7 @@ func TestUserOAuth2Github(t *testing.T) { }) } +// nolint:bodyclose func TestUserOIDC(t *testing.T) { t.Parallel() @@ -456,7 +458,7 @@ func oauth2Callback(t *testing.T, client *codersdk.Client) *http.Response { state := "somestate" oauthURL, err := client.URL.Parse("/api/v2/users/oauth2/github/callback?code=asd&state=" + state) require.NoError(t, err) - req, err := http.NewRequest("GET", oauthURL.String(), nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", oauthURL.String(), nil) require.NoError(t, err) req.AddCookie(&http.Cookie{ Name: codersdk.OAuth2StateKey, @@ -478,7 +480,7 @@ func oidcCallback(t *testing.T, client *codersdk.Client) *http.Response { state := "somestate" oauthURL, err := client.URL.Parse("/api/v2/users/oidc/callback?code=asd&state=" + state) require.NoError(t, err) - req, err := http.NewRequest("GET", oauthURL.String(), nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", oauthURL.String(), nil) require.NoError(t, err) req.AddCookie(&http.Cookie{ Name: codersdk.OAuth2StateKey, diff --git a/coderd/users_test.go b/coderd/users_test.go index bf372f01f182d..c813e7bcc71b1 100644 --- a/coderd/users_test.go +++ b/coderd/users_test.go @@ -477,7 +477,7 @@ func TestGrantSiteRoles(t *testing.T) { } ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() + t.Cleanup(cancel) var err error admin := coderdtest.New(t, nil) @@ -734,7 +734,7 @@ func TestUsersFilter(t *testing.T) { first := coderdtest.CreateFirstUser(t, client) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() + t.Cleanup(cancel) firstUser, err := client.User(ctx, codersdk.Me) require.NoError(t, err, "fetch me") @@ -1075,7 +1075,7 @@ func TestSuspendedPagination(t *testing.T) { coderdtest.CreateFirstUser(t, client) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() + t.Cleanup(cancel) me, err := client.User(ctx, codersdk.Me) require.NoError(t, err) @@ -1121,7 +1121,7 @@ func TestPaginatedUsers(t *testing.T) { // This test takes longer than a long time. ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong*2) - defer cancel() + t.Cleanup(cancel) me, err := client.User(ctx, codersdk.Me) require.NoError(t, err) diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 76aa3351062ee..d17a5e95ccbbb 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -27,6 +28,7 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { ln, err := net.Listen("tcp", ":0") require.NoError(t, err) server := http.Server{ + ReadHeaderTimeout: time.Minute, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, err := r.Cookie(codersdk.SessionTokenKey) assert.ErrorIs(t, err, http.ErrNoCookie) diff --git a/coderd/workspaces_test.go b/coderd/workspaces_test.go index 3c4acc52f46d0..db47eb796bcd5 100644 --- a/coderd/workspaces_test.go +++ b/coderd/workspaces_test.go @@ -402,7 +402,7 @@ func TestWorkspaceFilter(t *testing.T) { first := coderdtest.CreateFirstUser(t, client) ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() + t.Cleanup(cancel) users := make([]coderUser, 0) for i := 0; i < 10; i++ { diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index 46e7df383e552..452c516c8a342 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -94,6 +94,7 @@ func TestCache(t *testing.T) { require.True(t, valid) server := &http.Server{ + ReadHeaderTimeout: time.Minute, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }), @@ -131,8 +132,9 @@ func TestCache(t *testing.T) { proxy.Transport = conn.HTTPTransport() res := httptest.NewRecorder() proxy.ServeHTTP(res, req) - res.Result().Body.Close() - assert.Equal(t, http.StatusOK, res.Result().StatusCode) + resp := res.Result() + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) }() } wg.Wait() diff --git a/cryptorand/errors_test.go b/cryptorand/errors_test.go index f228ebd289302..ab0fabdd1e0f6 100644 --- a/cryptorand/errors_test.go +++ b/cryptorand/errors_test.go @@ -15,6 +15,7 @@ import ( // the rand.Reader. // // This test replaces the global rand.Reader, so cannot be parallelized +// //nolint:paralleltest func TestRandError(t *testing.T) { var origReader = rand.Reader diff --git a/cryptorand/numbers.go b/cryptorand/numbers.go index f840cd5c3ffd8..0059203ff30d0 100644 --- a/cryptorand/numbers.go +++ b/cryptorand/numbers.go @@ -110,6 +110,7 @@ func Int31n(max int32) (int32, error) { // set, regenerating v if necessary. n must be > 0. All input bits in v must be // fully random, you cannot cast a random uint8/uint16 for input into this // function. +// //nolint:varnamelen func UnbiasedModulo32(v uint32, n int32) (int32, error) { prod := uint64(v) * uint64(n) diff --git a/dogfood/Dockerfile b/dogfood/Dockerfile index ad03b25111fe1..4703a67e6356d 100644 --- a/dogfood/Dockerfile +++ b/dogfood/Dockerfile @@ -8,17 +8,17 @@ FROM ubuntu AS go RUN apt-get update && apt-get install --yes curl gcc # Install Go manually, so that we can control the version -ARG GOBORING_VERSION=1.18b7 -RUN mkdir --parents /usr/local/go /usr/local/goboring +ARG GO_VERSION=1.19 +RUN mkdir --parents /usr/local/go # Boring Go is needed to build FIPS-compliant binaries. RUN curl --silent --show-error --location \ - "https://storage.googleapis.com/go-boringcrypto/go${GOBORING_VERSION}.linux-amd64.tar.gz" \ - -o /usr/local/goboring.tar.gz + "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" \ + -o /usr/local/go.tar.gz -RUN tar --extract --gzip --directory=/usr/local/goboring --file=/usr/local/goboring.tar.gz --strip-components=1 +RUN tar --extract --gzip --directory=/usr/local/go --file=/usr/local/go.tar.gz --strip-components=1 -ENV PATH=$PATH:/usr/local/goboring/bin +ENV PATH=$PATH:/usr/local/go/bin # Install Go utilities. ARG GOPATH="/tmp/" @@ -196,7 +196,7 @@ RUN systemctl enable \ ARG CLOUD_SQL_PROXY_VERSION=1.26.0 \ DIVE_VERSION=0.10.0 \ DOCKER_GCR_VERSION=2.1.0 \ - GOLANGCI_LINT_VERSION=1.44.2 \ + GOLANGCI_LINT_VERSION=1.48.0 \ GRYPE_VERSION=0.24.0 \ HELM_VERSION=3.8.0 \ KUBE_LINTER_VERSION=0.2.5 \ @@ -290,14 +290,13 @@ RUN echo "PermitUserEnvironment yes" >>/etc/ssh/sshd_config && \ # We avoid copying the extracted directory since COPY slows to minutes when there # are a lot of small files. -COPY --from=go /usr/local/goboring.tar.gz /usr/local/goboring.tar.gz -RUN mkdir /usr/local/goboring && \ - tar --extract --gzip --directory=/usr/local/goboring --file=/usr/local/goboring.tar.gz --strip-components=1 +COPY --from=go /usr/local/go.tar.gz /usr/local/go.tar.gz +RUN mkdir /usr/local/go && \ + tar --extract --gzip --directory=/usr/local/go --file=/usr/local/go.tar.gz --strip-components=1 -ENV PATH=$PATH:/usr/local/goboring/bin +ENV PATH=$PATH:/usr/local/go/bin -# Ensure goboring is made the default Go -RUN update-alternatives --install /usr/local/bin/gofmt gofmt /usr/local/goboring/bin/gofmt 100 +RUN update-alternatives --install /usr/local/bin/gofmt gofmt /usr/local/go/bin/gofmt 100 COPY --from=go /tmp/bin /usr/local/bin diff --git a/go.mod b/go.mod index 4b8c596c4be95..38163dd09a596 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/coder/coder -go 1.18 +go 1.19 // Required until https://github.com/manifoldco/promptui/pull/169 is merged. replace github.com/manifoldco/promptui => github.com/kylecarbs/promptui v0.8.1-0.20201231190244-d8f2159af2b2 diff --git a/peer/conn_test.go b/peer/conn_test.go index 6e2de345f0d9e..992765b940c74 100644 --- a/peer/conn_test.go +++ b/peer/conn_test.go @@ -226,6 +226,7 @@ func TestConn(t *testing.T) { }() go func() { server := http.Server{ + ReadHeaderTimeout: time.Minute, Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(200) }), diff --git a/peerbroker/proxy.go b/peerbroker/proxy.go index ecaf135863515..3e3ccb441776b 100644 --- a/peerbroker/proxy.go +++ b/peerbroker/proxy.go @@ -40,16 +40,19 @@ type ProxyOptions struct { // PubSub is used to geodistribute WebRTC handshakes. All negotiation // messages are small in size (<=8KB), and we don't require delivery // guarantees because connections can always be renegotiated. -// ┌────────────────────┐ ┌─────────────────────────────┐ -// │ coderd │ │ coderd │ +// +// ┌────────────────────┐ ┌─────────────────────────────┐ +// │ coderd │ │ coderd │ +// // ┌─────────────────────┐ │//connect │ │ //listen │ // │ client │ │ │ │ │ ┌─────┐ // │ ├──►│Creates a stream ID │◄─►│Subscribe() to the │◄──┤agent│ // │NegotiateConnection()│ │and Publish() to the│ │channel. Parse the stream ID │ └─────┘ // └─────────────────────┘ │ channel: │ │from payloads to create new │ -// │ │ │NegotiateConnection() streams│ -// ││ │or write to existing ones. │ -// └────────────────────┘ └─────────────────────────────┘ +// +// │ │ │NegotiateConnection() streams│ +// ││ │or write to existing ones. │ +// └────────────────────┘ └─────────────────────────────┘ func ProxyDial(client proto.DRPCPeerBrokerClient, options ProxyOptions) (io.Closer, error) { proxyDial := &proxyDial{ channelID: options.ChannelID, diff --git a/provisioner/terraform/provision_test.go b/provisioner/terraform/provision_test.go index 7b3542c49e7c6..a9bd00cad9289 100644 --- a/provisioner/terraform/provision_test.go +++ b/provisioner/terraform/provision_test.go @@ -61,12 +61,11 @@ func setupProvisioner(t *testing.T, opts *provisionerServeOptions) (context.Cont } func TestProvision_Cancel(t *testing.T) { + t.Parallel() if runtime.GOOS == "windows" { t.Skip("This test uses interrupts and is not supported on Windows") } - t.Parallel() - cwd, err := os.Getwd() require.NoError(t, err) fakeBin := filepath.Join(cwd, "testdata", "bin", "terraform_fake_cancel.sh") diff --git a/scripts/apitypings/main.go b/scripts/apitypings/main.go index dcb0b962d1dba..8cea09d09f201 100644 --- a/scripts/apitypings/main.go +++ b/scripts/apitypings/main.go @@ -337,6 +337,7 @@ type TypescriptType struct { // typescriptType this function returns a typescript type for a given // golang type. // Eg: +// // []byte returns "string" func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) { switch ty := ty.(type) { diff --git a/site/site_test.go b/site/site_test.go index 72008b4941457..c04505bd03579 100644 --- a/site/site_test.go +++ b/site/site_test.go @@ -466,6 +466,7 @@ func TestServeAPIResponse(t *testing.T) { require.NoError(t, err) resp, err := http.DefaultClient.Do(req) require.NoError(t, err) + defer resp.Body.Close() var body struct { Code int `json:"code"` Message string `json:"message"` From 53ef2dfd95052066c6d98ee44f672835500658ae Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 22 Aug 2022 04:17:06 +0000 Subject: [PATCH 31/54] Fix reconnecting PTY tests --- agent/agent.go | 16 ++++++++++++++- agent/agent_test.go | 4 +++- agent/conn.go | 14 ++++++++++--- cli/agent.go | 1 - cli/configssh_test.go | 7 ++++--- cli/ssh.go | 14 ++++++------- cli/ssh_test.go | 21 +++++++++++--------- coderd/coderd_test.go | 7 ++++++- coderd/workspaceagents.go | 36 ---------------------------------- coderd/workspaceagents_test.go | 21 +++++++++++--------- coderd/workspaceapps_test.go | 7 ++++--- codersdk/workspaceagents.go | 3 +++ 12 files changed, 77 insertions(+), 74 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index c16e0637db470..41fa10d5180ed 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -4,6 +4,7 @@ import ( "context" "crypto/rand" "crypto/rsa" + "encoding/binary" "encoding/json" "errors" "fmt" @@ -219,8 +220,21 @@ func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { if err != nil { return } + // This cannot use a JSON decoder, since that can + // buffer additional data that is required for the PTY. + rawLen := make([]byte, 2) + _, err = conn.Read(rawLen) + if err != nil { + continue + } + length := binary.LittleEndian.Uint16(rawLen) + data := make([]byte, length) + _, err = conn.Read(data) + if err != nil { + continue + } var msg reconnectingPTYInit - err = json.NewDecoder(conn).Decode(&msg) + err = json.Unmarshal(data, &msg) if err != nil { continue } diff --git a/agent/agent_test.go b/agent/agent_test.go index 211cd9da347d0..4cdc6d2bb8cb4 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -274,7 +274,9 @@ func TestAgent(t *testing.T) { t.Skip("ConPTY appears to be inconsistent on Windows.") } - conn := setupAgent(t, agent.Metadata{}, 0) + conn := setupAgent(t, agent.Metadata{ + DERPMap: tailnettest.RunDERPAndSTUN(t), + }, 0) id := uuid.NewString() netConn, err := conn.ReconnectingPTY(id, 100, 100, "/bin/bash") require.NoError(t, err) diff --git a/agent/conn.go b/agent/conn.go index 06ff3f7998769..41c52b9133e8b 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -2,6 +2,7 @@ package agent import ( "context" + "encoding/binary" "encoding/json" "fmt" "io" @@ -138,11 +139,11 @@ type TailnetConn struct { *tailnet.Conn } -func (c *TailnetConn) Ping() (time.Duration, error) { +func (*TailnetConn) Ping() (time.Duration, error) { return 0, nil } -func (c *TailnetConn) CloseWithError(err error) error { +func (c *TailnetConn) CloseWithError(_ error) error { return c.Close() } @@ -168,6 +169,9 @@ func (c *TailnetConn) ReconnectingPTY(id string, height, width uint16, command s _ = conn.Close() return nil, err } + data = append(make([]byte, 2), data...) + binary.LittleEndian.PutUint16(data, uint16(len(data)-2)) + _, err = conn.Write(data) if err != nil { _ = conn.Close() @@ -202,5 +206,9 @@ func (c *TailnetConn) SSHClient() (*ssh.Client, error) { func (c *TailnetConn) DialContext(ctx context.Context, network string, addr string) (net.Conn, error) { _, rawPort, _ := net.SplitHostPort(addr) port, _ := strconv.Atoi(rawPort) - return c.Conn.DialContextTCP(ctx, netip.AddrPortFrom(tailnetIP, uint16(port))) + ipp := netip.AddrPortFrom(tailnetIP, uint16(port)) + if network == "udp" { + return c.Conn.DialContextUDP(ctx, ipp) + } + return c.Conn.DialContextTCP(ctx, ipp) } diff --git a/cli/agent.go b/cli/agent.go index ba9989ddde1b9..992de5ceaca6d 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -181,7 +181,6 @@ func workspaceAgent() *cobra.Command { // shells so "gitssh" works! "CODER_AGENT_TOKEN": client.SessionToken, }, - EnableTailnet: wireguard, CoordinatorDialer: client.ListenWorkspaceAgentTailnet, }) <-cmd.Context().Done() diff --git a/cli/configssh_test.go b/cli/configssh_test.go index 156b55a896e19..78df918d397bb 100644 --- a/cli/configssh_test.go +++ b/cli/configssh_test.go @@ -107,9 +107,10 @@ func TestConfigSSH(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent"), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + CoordinatorDialer: client.ListenWorkspaceAgentTailnet, + Logger: slogtest.Make(t, nil).Named("agent"), }) defer func() { _ = agentCloser.Close() diff --git a/cli/ssh.go b/cli/ssh.go index 3b06ea562c140..e8e83e0ad9a5a 100644 --- a/cli/ssh.go +++ b/cli/ssh.go @@ -274,34 +274,34 @@ func getWorkspaceAndAgent(ctx context.Context, cmd *cobra.Command, client *coder if len(agents) == 0 { return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, xerrors.Errorf("workspace %q has no agents", workspace.Name) } - var agent codersdk.WorkspaceAgent + var workspaceAgent codersdk.WorkspaceAgent if len(workspaceParts) >= 2 { for _, otherAgent := range agents { if otherAgent.Name != workspaceParts[1] { continue } - agent = otherAgent + workspaceAgent = otherAgent break } - if agent.ID == uuid.Nil { + if workspaceAgent.ID == uuid.Nil { return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, xerrors.Errorf("agent not found by name %q", workspaceParts[1]) } } - if agent.ID == uuid.Nil { + if workspaceAgent.ID == uuid.Nil { if len(agents) > 1 { if !shuffle { return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, xerrors.New("you must specify the name of an agent") } - agent, err = cryptorand.Element(agents) + workspaceAgent, err = cryptorand.Element(agents) if err != nil { return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, err } } else { - agent = agents[0] + workspaceAgent = agents[0] } } - return workspace, agent, nil + return workspace, workspaceAgent, nil } // Attempt to poll workspace autostop. We write a per-workspace lockfile to diff --git a/cli/ssh_test.go b/cli/ssh_test.go index 0865798453f3e..6f2965174ac77 100644 --- a/cli/ssh_test.go +++ b/cli/ssh_test.go @@ -89,9 +89,10 @@ func TestSSH(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent"), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + Logger: slogtest.Make(t, nil).Named("agent"), }) defer func() { _ = agentCloser.Close() @@ -110,9 +111,10 @@ func TestSSH(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent"), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + Logger: slogtest.Make(t, nil).Named("agent"), }) <-ctx.Done() _ = agentCloser.Close() @@ -178,9 +180,10 @@ func TestSSH(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = agentToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent"), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + Logger: slogtest.Make(t, nil).Named("agent"), }) defer agentCloser.Close() diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index ad9988267792a..a2bea4dbb20cb 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -266,13 +266,14 @@ func TestAuthorizeAllEndpoints(t *testing.T) { "POST:/api/v2/workspaceagents/azure-instance-identity": {NoAuthorize: true}, "POST:/api/v2/workspaceagents/google-instance-identity": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/gitsshkey": {NoAuthorize: true}, + "GET:/api/v2/workspaceagents/me/coordinate": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/iceservers": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/listen": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/metadata": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/turn": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/node": {NoAuthorize: true}, + "GET:/api/v2/workspaceagents/{workspaceagent}/coordinate": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/{workspaceagent}/iceservers": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/{workspaceagent}/turn": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/{workspaceagent}/derpmap": {NoAuthorize: true}, // These endpoints have more assertions. This is good, add more endpoints to assert if you can! @@ -338,6 +339,10 @@ func TestAuthorizeAllEndpoints(t *testing.T) { AssertAction: rbac.ActionCreate, AssertObject: workspaceExecObj, }, + "GET:/api/v2/workspaceagents/{workspaceagent}/turn": { + AssertAction: rbac.ActionCreate, + AssertObject: workspaceExecObj, + }, "GET:/api/v2/workspaceagents/{workspaceagent}/pty": { AssertAction: rbac.ActionCreate, AssertObject: workspaceExecObj, diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index f1dff4bc1cddd..17be85f9d657d 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -596,42 +596,6 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R } } -// postWorkspaceAgentNode sends networking information to a workspace agent node. -func (api *API) postWorkspaceAgentNode(rw http.ResponseWriter, r *http.Request) { - workspaceAgent := httpmw.WorkspaceAgentParam(r) - workspace := httpmw.WorkspaceParam(r) - if !api.Authorize(r, rbac.ActionUpdate, workspace) { - httpapi.ResourceNotFound(rw) - return - } - - var node tailnet.Node - if !httpapi.Read(rw, r, &node) { - return - } - data, err := json.Marshal(node) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Failed to marshal node data.", - Detail: err.Error(), - }) - return - } - agentIDBytes, _ := workspaceAgent.ID.MarshalText() - data = append(agentIDBytes, data...) - err = api.Pubsub.Publish("tailnet", data) - if err != nil { - httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Publish node data.", - Detail: err.Error(), - }) - return - } - httpapi.Write(rw, http.StatusOK, codersdk.Response{ - Message: "Published!", - }) -} - func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { apps := make([]codersdk.WorkspaceApp, 0) for _, dbApp := range dbApps { diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 8a32ac823f969..afe1411247313 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -110,9 +110,10 @@ func TestWorkspaceAgentListen(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) defer func() { _ = agentCloser.Close() @@ -243,9 +244,10 @@ func TestWorkspaceAgentTURN(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) defer func() { _ = agentCloser.Close() @@ -369,9 +371,10 @@ func TestWorkspaceAgentPTY(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug), }) defer func() { _ = agentCloser.Close() diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 75623ec033353..4a75e9fc78e99 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -82,9 +82,10 @@ func TestWorkspaceAppsProxyPath(t *testing.T) { agentClient := codersdk.New(client.URL) agentClient.SessionToken = authToken agentCloser := agent.New(agent.Options{ - FetchMetadata: agentClient.WorkspaceAgentMetadata, - WebRTCDialer: agentClient.ListenWorkspaceAgent, - Logger: slogtest.Make(t, nil).Named("agent"), + FetchMetadata: agentClient.WorkspaceAgentMetadata, + CoordinatorDialer: agentClient.ListenWorkspaceAgentTailnet, + WebRTCDialer: agentClient.ListenWorkspaceAgent, + Logger: slogtest.Make(t, nil).Named("agent"), }) t.Cleanup(func() { _ = agentCloser.Close() diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index 1c1b11f2238ff..8503e78ad80ad 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -280,12 +280,14 @@ func (c *Client) ListenWorkspaceAgentTailnet(ctx context.Context) (net.Conn, err httpClient := &http.Client{ Jar: jar, } + // nolint:bodyclose conn, _, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, }) if err != nil { return nil, err } + return websocket.NetConn(ctx, conn, websocket.MessageBinary), nil } @@ -344,6 +346,7 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg go func() { for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { logger.Debug(ctx, "connecting") + // nolint:bodyclose ws, _, err := websocket.Dial(ctx, coordinateURL.String(), &websocket.DialOptions{ HTTPClient: httpClient, // Need to disable compression to avoid a data-race. From fc4c2ec9fddaff9a7d5e4adee4eda906a5614064 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Tue, 23 Aug 2022 12:04:14 -0500 Subject: [PATCH 32/54] fix: update wireguard-go to fix devtunnel --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 889a8a0e82e2a..ce2b24580e473 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ replace github.com/golang/glog => github.com/coder/glog v1.0.1-0.20220322161911- // https://github.com/coder/kcp-go/commit/83c0904cec69dcf21ec10c54ea666bda18ada831 replace github.com/fatedier/kcp-go => github.com/coder/kcp-go v2.0.4-0.20220409183554-83c0904cec69+incompatible -replace golang.zx2c4.com/wireguard/tun/netstack => github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c +replace golang.zx2c4.com/wireguard/tun/netstack => github.com/coder/wireguard-go/tun/netstack v0.0.0-20220823170024-a78136eb0cab // https://github.com/pion/udp/pull/73 replace github.com/pion/udp => github.com/mafredri/udp v0.1.2-0.20220805105907-b2872e92e98d diff --git a/go.sum b/go.sum index 52d2b5a945d33..d2261dd13ea51 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 h1:Lv081uuydkOPNFwOSGv8fIWZeMXcIjQI1Wr8aZpLejE= github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1/go.mod h1:1AAccn2hLv0wT6q/MZ/bPIki+B3csbjq+P+nM/Xm2Oo= -github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c h1:EWRwdW6sGsNFiwDqQpwCgJDt4ilUwT11ZQcWV5WBNH8= -github.com/coder/wireguard-go/tun/netstack v0.0.0-20220803190501-a3df633de59c/go.mod h1:TCJ66NtXh3urJotTdoYQOHHkyE899vOQl5TuF+WLSes= +github.com/coder/wireguard-go/tun/netstack v0.0.0-20220823170024-a78136eb0cab h1:9yEvRWXXfyKzXu8AqywCi+tFZAoqCy4wVcsXwuvZNMc= +github.com/coder/wireguard-go/tun/netstack v0.0.0-20220823170024-a78136eb0cab/go.mod h1:TCJ66NtXh3urJotTdoYQOHHkyE899vOQl5TuF+WLSes= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= From c516b44007511454ecb6fa5a4eb58dc7ab62dd1f Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Tue, 23 Aug 2022 12:09:25 -0500 Subject: [PATCH 33/54] fix migration numbers --- coderd/authorize.go | 1 + .../{000037_tailnet.down.sql => 000038_tailnet.down.sql} | 0 .../migrations/{000037_tailnet.up.sql => 000038_tailnet.up.sql} | 0 3 files changed, 1 insertion(+) rename coderd/database/migrations/{000037_tailnet.down.sql => 000038_tailnet.down.sql} (100%) rename coderd/database/migrations/{000037_tailnet.up.sql => 000038_tailnet.up.sql} (100%) diff --git a/coderd/authorize.go b/coderd/authorize.go index b68e1b68544a3..ab81c4ee81f6a 100644 --- a/coderd/authorize.go +++ b/coderd/authorize.go @@ -49,6 +49,7 @@ func (api *API) Authorize(r *http.Request, action rbac.Action, object rbac.Objec // This function will log appropriately, but the caller must return an // error to the api client. // Eg: +// // if !h.Authorize(...) { // httpapi.Forbidden(rw) // return diff --git a/coderd/database/migrations/000037_tailnet.down.sql b/coderd/database/migrations/000038_tailnet.down.sql similarity index 100% rename from coderd/database/migrations/000037_tailnet.down.sql rename to coderd/database/migrations/000038_tailnet.down.sql diff --git a/coderd/database/migrations/000037_tailnet.up.sql b/coderd/database/migrations/000038_tailnet.up.sql similarity index 100% rename from coderd/database/migrations/000037_tailnet.up.sql rename to coderd/database/migrations/000038_tailnet.up.sql From 73a8d7eadfbe4ee28b1a290893bb784dde2a7d69 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Tue, 23 Aug 2022 12:17:43 -0500 Subject: [PATCH 34/54] linting --- coderd/workspaceagents.go | 70 ----------------------------------- enterprise/coderd/licenses.go | 13 ++++--- 2 files changed, 7 insertions(+), 76 deletions(-) diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 17be85f9d657d..5eb30f5c03058 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -23,9 +23,7 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" - "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/codersdk" - "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker" "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" @@ -468,74 +466,6 @@ func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (a }, nil } -// dialWorkspaceAgent connects to a workspace agent by ID. Only rely on -// r.Context() for cancellation if it's use is safe or r.Hijack() has -// not been performed. -func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Conn, error) { - client, server := provisionersdk.TransportPipe() - ctx, cancelFunc := context.WithCancel(context.Background()) - go func() { - _ = peerbroker.ProxyListen(ctx, server, peerbroker.ProxyOptions{ - ChannelID: agentID.String(), - Logger: api.Logger.Named("peerbroker-proxy-dial"), - Pubsub: api.Pubsub, - }) - _ = client.Close() - _ = server.Close() - }() - - peerClient := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) - stream, err := peerClient.NegotiateConnection(ctx) - if err != nil { - cancelFunc() - return nil, xerrors.Errorf("negotiate: %w", err) - } - options := &peer.ConnOptions{ - Logger: api.Logger.Named("agent-dialer"), - } - options.SettingEngine.SetSrflxAcceptanceMinWait(0) - options.SettingEngine.SetRelayAcceptanceMinWait(0) - // Use the ProxyDialer for the TURN server. - // This is required for connections where P2P is not enabled. - options.SettingEngine.SetICEProxyDialer(turnconn.ProxyDialer(func() (c net.Conn, err error) { - clientPipe, serverPipe := net.Pipe() - go func() { - <-ctx.Done() - _ = clientPipe.Close() - _ = serverPipe.Close() - }() - localAddress, _ := r.Context().Value(http.LocalAddrContextKey).(*net.TCPAddr) - remoteAddress := &net.TCPAddr{ - IP: net.ParseIP(r.RemoteAddr), - } - // By default requests have the remote address and port. - host, port, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - return nil, xerrors.Errorf("split remote address: %w", err) - } - remoteAddress.IP = net.ParseIP(host) - remoteAddress.Port, err = strconv.Atoi(port) - if err != nil { - return nil, xerrors.Errorf("convert remote port: %w", err) - } - api.TURNServer.Accept(clientPipe, remoteAddress, localAddress) - return serverPipe, nil - })) - peerConn, err := peerbroker.Dial(stream, append(api.ICEServers, turnconn.Proxy), options) - if err != nil { - cancelFunc() - return nil, xerrors.Errorf("dial: %w", err) - } - go func() { - <-peerConn.Closed() - cancelFunc() - }() - return &agent.WebRTCConn{ - Negotiator: peerClient, - Conn: peerConn, - }, nil -} - func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request) { workspace := httpmw.WorkspaceParam(r) if !api.Authorize(r, rbac.ActionRead, workspace) { diff --git a/enterprise/coderd/licenses.go b/enterprise/coderd/licenses.go index 630c1ffe18619..42df66ed99e6a 100644 --- a/enterprise/coderd/licenses.go +++ b/enterprise/coderd/licenses.go @@ -33,6 +33,7 @@ var ValidMethods = []string{"EdDSA"} // key20220812 is the Coder license public key with id 2022-08-12 used to validate licenses signed // by our signing infrastructure +// //go:embed keys/2022-08-12 var key20220812 []byte @@ -129,12 +130,12 @@ func (a *licenseAPI) handler() http.Handler { // postLicense adds a new Enterprise license to the cluster. We allow multiple different licenses // in the cluster at one time for several reasons: // -// 1. Upgrades --- if the license format changes from one version of Coder to the next, during a -// rolling update you will have different Coder servers that need different licenses to function. -// 2. Avoid abrupt feature breakage --- when an admin uploads a new license with different features -// we generally don't want the old features to immediately break without warning. With a grace -// period on the license, features will continue to work from the old license until its grace -// period, then the users will get a warning allowing them to gracefully stop using the feature. +// 1. Upgrades --- if the license format changes from one version of Coder to the next, during a +// rolling update you will have different Coder servers that need different licenses to function. +// 2. Avoid abrupt feature breakage --- when an admin uploads a new license with different features +// we generally don't want the old features to immediately break without warning. With a grace +// period on the license, features will continue to work from the old license until its grace +// period, then the users will get a warning allowing them to gracefully stop using the feature. func (a *licenseAPI) postLicense(rw http.ResponseWriter, r *http.Request) { if !a.auth.Authorize(r, rbac.ActionCreate, rbac.ResourceLicense) { httpapi.Forbidden(rw) From 3820dc9f4cfb4e4cc69689428a3339d6919ebe21 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 23 Aug 2022 17:47:10 +0000 Subject: [PATCH 35/54] Return early for status if endpoints are empty --- tailnet/conn.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tailnet/conn.go b/tailnet/conn.go index f37ef339f2b4f..24908003533d1 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -269,6 +269,9 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { callback(makeNode()) }) c.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { + if err != nil { + return + } endpoints := make([]string, 0, len(s.LocalAddrs)) for _, addr := range s.LocalAddrs { endpoints = append(endpoints, addr.Addr.String()) From 9f991d79261c7170bd64e4f932a9c0f5342aaf10 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 23 Aug 2022 18:23:58 -0500 Subject: [PATCH 36/54] Update cli/server.go Co-authored-by: Colin Adler --- cli/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/server.go b/cli/server.go index b051d9839d5e8..525d07167b9d5 100644 --- a/cli/server.go +++ b/cli/server.go @@ -316,7 +316,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { RegionName: "Coder", Nodes: []*tailcfg.DERPNode{{ Name: "1a", - RegionID: 1, + RegionID: derpServerRegionID, STUNOnly: true, HostName: "stun.l.google.com", STUNPort: 19302, From a518e260ce533cf6e9fed72858c523a198b6a186 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 23 Aug 2022 18:24:05 -0500 Subject: [PATCH 37/54] Update cli/server.go Co-authored-by: Colin Adler --- cli/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/server.go b/cli/server.go index 525d07167b9d5..c24fcddf1fc02 100644 --- a/cli/server.go +++ b/cli/server.go @@ -322,7 +322,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { STUNPort: 19302, }, { Name: "1b", - RegionID: 1, + RegionID: derpServerRegionID, HostName: accessURLParsed.Hostname(), DERPPort: accessURLPort, STUNPort: -1, From cfb679afa7b12dcb3ed2fe497545878168692bfa Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 24 Aug 2022 00:15:17 +0000 Subject: [PATCH 38/54] Fix frontend entites --- coderd/wsconncache/wsconncache_test.go | 2 -- site/src/testHelpers/entities.ts | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/coderd/wsconncache/wsconncache_test.go b/coderd/wsconncache/wsconncache_test.go index e3c4853adf4d4..80f187ba15ab7 100644 --- a/coderd/wsconncache/wsconncache_test.go +++ b/coderd/wsconncache/wsconncache_test.go @@ -39,9 +39,7 @@ func TestCache(t *testing.T) { return setupAgent(t, agent.Metadata{}, 0), nil }, 0) defer func() { - fmt.Printf("Closing cache...\n") _ = cache.Close() - fmt.Printf("Closed cache...\n") }() conn1, _, err := cache.Acquire(httptest.NewRequest(http.MethodGet, "/", nil), uuid.Nil) require.NoError(t, err) diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index 5c28f574704f5..67ebc6f8c2069 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -288,9 +288,8 @@ export const MockWorkspaceAgent: TypesGen.WorkspaceAgent = { resource_id: "", status: "connected", updated_at: "", - wireguard_public_key: "", - disco_public_key: "", - ipv6: "", + preferred_derp: 0, + latency: {}, } export const MockWorkspaceAgentDisconnected: TypesGen.WorkspaceAgent = { From 5a53494f9b3b599084b4c0aba7d2079367db2a7a Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 24 Aug 2022 00:17:51 +0000 Subject: [PATCH 39/54] Fix agent bicopy --- agent/agent_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index e3f658dbe0b65..84e1e2303e317 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -14,6 +14,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -463,6 +464,7 @@ func setupSSHCommand(t *testing.T, beforeArgs []string, afterArgs []string) *exe agentConn := setupAgent(t, agent.Metadata{}, 0) listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) + waitGroup := sync.WaitGroup{} go func() { defer listener.Close() for { @@ -475,11 +477,16 @@ func setupSSHCommand(t *testing.T, beforeArgs []string, afterArgs []string) *exe _ = conn.Close() return } - go agent.Bicopy(context.Background(), conn, ssh) + waitGroup.Add(1) + go func() { + agent.Bicopy(context.Background(), conn, ssh) + waitGroup.Done() + }() } }() t.Cleanup(func() { _ = listener.Close() + waitGroup.Wait() }) tcpAddr, valid := listener.Addr().(*net.TCPAddr) require.True(t, valid) From f2bd8fb1b7902b6b0f7d14cc9acd816b7bb7ee19 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 24 Aug 2022 00:31:04 +0000 Subject: [PATCH 40/54] Fix race condition for the last node --- tailnet/conn.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tailnet/conn.go b/tailnet/conn.go index 24908003533d1..0588ee968a4ab 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -265,8 +265,9 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { c.lastMutex.Lock() c.lastPreferredDERP = ni.PreferredDERP c.lastDERPLatency = ni.DERPLatency + node := makeNode() c.lastMutex.Unlock() - callback(makeNode()) + callback(node) }) c.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) { if err != nil { @@ -278,8 +279,9 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { } c.lastMutex.Lock() c.lastEndpoints = endpoints + node := makeNode() c.lastMutex.Unlock() - callback(makeNode()) + callback(node) }) } From e82c076c830052d5447d6569f5ed2e54f3922bcc Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 24 Aug 2022 00:44:58 +0000 Subject: [PATCH 41/54] Fix down migration --- coderd/database/migrations/000038_tailnet.down.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/coderd/database/migrations/000038_tailnet.down.sql b/coderd/database/migrations/000038_tailnet.down.sql index e69de29bb2d1d..e889d94f0934b 100644 --- a/coderd/database/migrations/000038_tailnet.down.sql +++ b/coderd/database/migrations/000038_tailnet.down.sql @@ -0,0 +1,4 @@ +ALTER TABLE workspace_agents + ADD COLUMN wireguard_node_ipv6 inet NOT NULL DEFAULT '::/128', + ADD COLUMN wireguard_node_public_key varchar(128) NOT NULL DEFAULT 'nodekey:0000000000000000000000000000000000000000000000000000000000000000', + ADD COLUMN wireguard_disco_public_key varchar(128) NOT NULL DEFAULT 'discokey:0000000000000000000000000000000000000000000000000000000000000000'; From 9458070d3f06979a0dd11dd6ff8f8a38bcfaee1f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 30 Aug 2022 03:27:06 +0000 Subject: [PATCH 42/54] Fix connection RBAC --- coderd/coderdtest/authtest.go | 10 ++++++---- coderd/workspaceagents.go | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/coderd/coderdtest/authtest.go b/coderd/coderdtest/authtest.go index f88fd21f20849..cc516904ce33a 100644 --- a/coderd/coderdtest/authtest.go +++ b/coderd/coderdtest/authtest.go @@ -167,6 +167,7 @@ func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) { // skipRoutes allows skipping routes from being checked. skipRoutes := map[string]string{ "POST:/api/v2/users/logout": "Logging out deletes the API Key for other routes", + "GET:/derp": "This requires a WebSocket upgrade!", } assertRoute := map[string]RouteCheck{ @@ -193,11 +194,8 @@ func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) { "GET:/api/v2/workspaceagents/me/listen": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/metadata": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/me/turn": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/me/derp": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/me/wireguardlisten": {NoAuthorize: true}, - "POST:/api/v2/workspaceagents/me/keys": {NoAuthorize: true}, + "GET:/api/v2/workspaceagents/me/coordinate": {NoAuthorize: true}, "GET:/api/v2/workspaceagents/{workspaceagent}/iceservers": {NoAuthorize: true}, - "GET:/api/v2/workspaceagents/{workspaceagent}/derp": {NoAuthorize: true}, // These endpoints have more assertions. This is good, add more endpoints to assert if you can! "GET:/api/v2/organizations/{organization}": {AssertObject: rbac.ResourceOrganization.InOrg(a.Admin.OrganizationID)}, @@ -270,6 +268,10 @@ func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) { AssertAction: rbac.ActionCreate, AssertObject: workspaceExecObj, }, + "GET:/api/v2/workspaceagents/{workspaceagent}/coordinate": { + AssertAction: rbac.ActionCreate, + AssertObject: workspaceExecObj, + }, "GET:/api/v2/workspaces/": { StatusCode: http.StatusOK, AssertAction: rbac.ActionRead, diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 22959d1a31e73..a1f45203715da 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -513,6 +513,12 @@ func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request // After accept a PubSub starts listening for new connection node updates // which are written to the WebSocket. func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.Request) { + workspace := httpmw.WorkspaceParam(r) + if !api.Authorize(r, rbac.ActionCreate, workspace.ExecutionRBAC()) { + httpapi.ResourceNotFound(rw) + return + } + api.websocketWaitMutex.Lock() api.websocketWaitGroup.Add(1) api.websocketWaitMutex.Unlock() From 2382dc2c4b66bf0f97336726174d459085aeed9e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 30 Aug 2022 03:31:12 +0000 Subject: [PATCH 43/54] Fix migration numbers --- .../{000038_tailnet.down.sql => 000039_tailnet.down.sql} | 0 .../migrations/{000038_tailnet.up.sql => 000039_tailnet.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000038_tailnet.down.sql => 000039_tailnet.down.sql} (100%) rename coderd/database/migrations/{000038_tailnet.up.sql => 000039_tailnet.up.sql} (100%) diff --git a/coderd/database/migrations/000038_tailnet.down.sql b/coderd/database/migrations/000039_tailnet.down.sql similarity index 100% rename from coderd/database/migrations/000038_tailnet.down.sql rename to coderd/database/migrations/000039_tailnet.down.sql diff --git a/coderd/database/migrations/000038_tailnet.up.sql b/coderd/database/migrations/000039_tailnet.up.sql similarity index 100% rename from coderd/database/migrations/000038_tailnet.up.sql rename to coderd/database/migrations/000039_tailnet.up.sql From 04367d97a89ae1b2ee46309b8e515714f2287da8 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 30 Aug 2022 21:55:14 +0000 Subject: [PATCH 44/54] Fix forwarding TCP to a local port --- tailnet/conn.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tailnet/conn.go b/tailnet/conn.go index 0588ee968a4ab..cb90a83efd488 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -461,7 +461,11 @@ func (c *Conn) forwardTCPToLocal(conn net.Conn, port uint16) { _, err := io.Copy(conn, server) connClosed <- err }() - err = <-connClosed + select { + case err = <-connClosed: + case <-c.closed: + return + } if err != nil { c.logger.Debug(ctx, "proxy connection closed with error", slog.Error(err)) } From db65752d5005c87656ec20c2f4c8e2199e40aadc Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 00:54:03 +0000 Subject: [PATCH 45/54] Implement ping for tailnet --- agent/agent_test.go | 11 ++++++----- agent/conn.go | 20 ++++++++++++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index e24e47a1d0b2f..fa671cd01723b 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -473,13 +473,14 @@ func TestAgent(t *testing.T) { t.Run("Tailnet", func(t *testing.T) { t.Parallel() derpMap := tailnettest.RunDERPAndSTUN(t) - conn := setupSSHSession(t, agent.Metadata{ + conn := setupAgent(t, agent.Metadata{ DERPMap: derpMap, - }) + }, 0) defer conn.Close() - output, err := conn.CombinedOutput("echo test") - require.NoError(t, err) - t.Log(string(output)) + require.Eventually(t, func() bool { + _, err := conn.Ping() + return err == nil + }, testutil.WaitMedium, testutil.IntervalFast) }) } diff --git a/agent/conn.go b/agent/conn.go index 41c52b9133e8b..e311668369237 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -15,6 +15,8 @@ import ( "golang.org/x/crypto/ssh" "golang.org/x/xerrors" + "tailscale.com/ipn/ipnstate" + "tailscale.com/tailcfg" "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker/proto" @@ -139,8 +141,22 @@ type TailnetConn struct { *tailnet.Conn } -func (*TailnetConn) Ping() (time.Duration, error) { - return 0, nil +func (c *TailnetConn) Ping() (time.Duration, error) { + errCh := make(chan error, 1) + durCh := make(chan time.Duration, 1) + c.Conn.Ping(tailnetIP, tailcfg.PingICMP, func(pr *ipnstate.PingResult) { + if pr.Err != "" { + errCh <- xerrors.New(pr.Err) + return + } + durCh <- time.Duration(pr.LatencySeconds * float64(time.Second)) + }) + select { + case err := <-errCh: + return 0, err + case dur := <-durCh: + return dur, nil + } } func (c *TailnetConn) CloseWithError(_ error) error { From 3f48e18131d2ff94960aae691a30645e8c325214 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 01:11:21 +0000 Subject: [PATCH 46/54] Rename to ForceHTTP --- cli/server.go | 12 ++++++------ coderd/coderd_test.go | 12 ++++++------ coderd/coderdtest/coderdtest.go | 2 +- go.mod | 12 +++++++----- go.sum | 16 ++++++++-------- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/cli/server.go b/cli/server.go index 9c4eae57f7cf5..6dd29295a4187 100644 --- a/cli/server.go +++ b/cli/server.go @@ -344,12 +344,12 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { HostName: "stun.l.google.com", STUNPort: 19302, }, { - Name: "1b", - RegionID: derpServerRegionID, - HostName: accessURLParsed.Hostname(), - DERPPort: accessURLPort, - STUNPort: -1, - HTTPForTests: accessURLParsed.Scheme == "http", + Name: "1b", + RegionID: derpServerRegionID, + HostName: accessURLParsed.Hostname(), + DERPPort: accessURLPort, + STUNPort: -1, + ForceHTTP: accessURLParsed.Scheme == "http", }}, }, 2: { diff --git a/coderd/coderd_test.go b/coderd/coderd_test.go index f54832ae6db8c..7e9175e73392b 100644 --- a/coderd/coderd_test.go +++ b/coderd/coderd_test.go @@ -62,12 +62,12 @@ func TestDERP(t *testing.T) { RegionCode: "cdr", RegionName: "Coder", Nodes: []*tailcfg.DERPNode{{ - Name: "1a", - RegionID: 1, - HostName: client.URL.Hostname(), - DERPPort: derpPort, - STUNPort: -1, - HTTPForTests: true, + Name: "1a", + RegionID: 1, + HostName: client.URL.Hostname(), + DERPPort: derpPort, + STUNPort: -1, + ForceHTTP: true, }}, }, }, diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 6dd1d964d2fd5..8970058ffb74d 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -229,7 +229,7 @@ func newWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c DERPPort: derpPort, STUNPort: -1, InsecureForTests: true, - HTTPForTests: true, + ForceHTTP: true, }}, }, }, diff --git a/go.mod b/go.mod index fd8eddb94d97b..c88d4aced6ea1 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,6 @@ replace github.com/chzyer/readline => github.com/kylecarbs/readline v0.0.0-20220 // Required until https://github.com/briandowns/spinner/pull/136 is merged. replace github.com/briandowns/spinner => github.com/kylecarbs/spinner v1.18.2-0.20220329160715-20702b5af89e -replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 - // Required until https://github.com/fergusstrange/embedded-postgres/pull/75 is merged. replace github.com/fergusstrange/embedded-postgres => github.com/kylecarbs/embedded-postgres v1.17.1-0.20220615202325-461532cecd3a @@ -46,6 +44,10 @@ replace github.com/pion/udp => github.com/mafredri/udp v0.1.2-0.20220805105907-b // https://github.com/hashicorp/hc-install/pull/68 replace github.com/hashicorp/hc-install => github.com/mafredri/hc-install v0.4.1-0.20220727132613-e91868e28445 +// There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here: +// https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main +replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220831012541-a77bda274fd6 + require ( cdr.dev/slog v1.4.2-0.20220525200111-18dce5c2cd5f cloud.google.com/go/compute v1.7.0 @@ -131,12 +133,12 @@ require ( go.uber.org/atomic v1.9.0 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 - golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 golang.org/x/net v0.0.0-20220630215102-69896b714898 golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f - golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/tools v0.1.11 @@ -270,7 +272,7 @@ require ( github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 // indirect github.com/tcnksm/go-httpstat v0.2.0 // indirect github.com/tdewolff/parse/v2 v2.6.0 // indirect - github.com/u-root/uio v0.0.0-20210528151154-e40b768296a7 // indirect + github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 // indirect github.com/vektah/gqlparser/v2 v2.4.4 // indirect github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 // indirect github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect diff --git a/go.sum b/go.sum index 7139938a35b60..fb393d3457085 100644 --- a/go.sum +++ b/go.sum @@ -352,8 +352,8 @@ github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1 h1:UqBrPWSYvRI2s5RtOu github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY= github.com/coder/retry v1.3.0/go.mod h1:tXuRgZgWjUnU5LZPT4lJh4ew2elUhexhlnXzrJWdyFY= -github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1 h1:Lv081uuydkOPNFwOSGv8fIWZeMXcIjQI1Wr8aZpLejE= -github.com/coder/tailscale v1.1.1-0.20220802200410-cba8e836c5f1/go.mod h1:1AAccn2hLv0wT6q/MZ/bPIki+B3csbjq+P+nM/Xm2Oo= +github.com/coder/tailscale v1.1.1-0.20220831012541-a77bda274fd6 h1://ApBDDh58hFwMe0AzlgqJrGhzu6Rjk8fQXrR+mbhYE= +github.com/coder/tailscale v1.1.1-0.20220831012541-a77bda274fd6/go.mod h1:MO+tWkQp2YIF3KBnnej/mQvgYccRS5Xk/IrEpZ4Z3BU= github.com/coder/wireguard-go/tun/netstack v0.0.0-20220823170024-a78136eb0cab h1:9yEvRWXXfyKzXu8AqywCi+tFZAoqCy4wVcsXwuvZNMc= github.com/coder/wireguard-go/tun/netstack v0.0.0-20220823170024-a78136eb0cab/go.mod h1:TCJ66NtXh3urJotTdoYQOHHkyE899vOQl5TuF+WLSes= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= @@ -1811,8 +1811,8 @@ github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoi github.com/tommy-muehle/go-mnd/v2 v2.4.0/go.mod h1:WsUAkMJMYww6l/ufffCD3m+P7LEvr8TnZn9lwVDlgzw= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= -github.com/u-root/uio v0.0.0-20210528151154-e40b768296a7 h1:XMAtQHwKjWHIRwg+8Nj/rzUomQY1q6cM3ncA0wP8GU4= -github.com/u-root/uio v0.0.0-20210528151154-e40b768296a7/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= +github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4 h1:hl6sK6aFgTLISijk6xIzeqnPzQcsLqqvL6vEfTPinME= +github.com/u-root/uio v0.0.0-20220204230159-dac05f7d2cb4/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -2054,8 +2054,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd h1:zVFyTKZN/Q7mNRWSs1GOYnHM9NiFSJ54YVRsD0rNWT4= -golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp/typeparams v0.0.0-20220328175248-053ad81199eb h1:fP6C8Xutcp5AlakmT/SkQot0pMicROAsEX7OfNPuG10= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -2385,8 +2385,8 @@ golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d h1:/m5NbqQelATgoSPVC2Z23sR4kVNokFwDDyWh/3rGY+I= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From 94658f58f01168cb773c52a9a901e530fb369cfa Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 02:51:28 +0000 Subject: [PATCH 47/54] Add external derpmapping --- cli/server.go | 87 +++++++++++++---------------------------- tailnet/conn.go | 12 ------ tailnet/coordinator.go | 15 +++++++ tailnet/derpmap.go | 60 ++++++++++++++++++++++++++++ tailnet/derpmap_test.go | 60 ++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 71 deletions(-) create mode 100644 tailnet/derpmap.go create mode 100644 tailnet/derpmap_test.go diff --git a/cli/server.go b/cli/server.go index 6dd29295a4187..a43aeaf649be3 100644 --- a/cli/server.go +++ b/cli/server.go @@ -66,6 +66,7 @@ import ( "github.com/coder/coder/provisionerd" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk/proto" + "github.com/coder/coder/tailnet" ) // nolint:gocyclo @@ -78,7 +79,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { derpServerRegionID int derpServerRegionCode string derpServerRegionName string - derpServerSTUNURLs []string + derpServerSTUNAddrs []string derpConfigURL string promEnabled bool promAddress string @@ -326,62 +327,29 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { validatedAutoImportTemplates[i] = v } + derpMap, err := tailnet.NewDERPMap(ctx, &tailcfg.DERPRegion{ + RegionID: derpServerRegionID, + RegionCode: derpServerRegionCode, + RegionName: derpServerRegionName, + Nodes: []*tailcfg.DERPNode{{ + Name: fmt.Sprintf("%db", derpServerRegionID), + RegionID: derpServerRegionID, + HostName: accessURLParsed.Hostname(), + DERPPort: accessURLPort, + STUNPort: -1, + ForceHTTP: accessURLParsed.Scheme == "http", + }}, + }, derpServerSTUNAddrs, derpConfigURL) + if err != nil { + return xerrors.Errorf("create derp map: %w", err) + } + options := &coderd.Options{ - AccessURL: accessURLParsed, - ICEServers: iceServers, - Logger: logger.Named("coderd"), - Database: databasefake.New(), - DERPMap: &tailcfg.DERPMap{ - Regions: map[int]*tailcfg.DERPRegion{ - derpServerRegionID: { - RegionID: derpServerRegionID, - RegionCode: "coder", - RegionName: "Coder", - Nodes: []*tailcfg.DERPNode{{ - Name: "1a", - RegionID: derpServerRegionID, - STUNOnly: true, - HostName: "stun.l.google.com", - STUNPort: 19302, - }, { - Name: "1b", - RegionID: derpServerRegionID, - HostName: accessURLParsed.Hostname(), - DERPPort: accessURLPort, - STUNPort: -1, - ForceHTTP: accessURLParsed.Scheme == "http", - }}, - }, - 2: { - RegionID: 2, - RegionCode: "nyc", - RegionName: "New York City", - Nodes: []*tailcfg.DERPNode{ - { - Name: "2c", - RegionID: 2, - HostName: "derp1c.tailscale.com", - IPv4: "104.248.8.210", - IPv6: "2604:a880:800:10::7a0:e001", - }, - }, - }, - 3: { - RegionID: 3, - RegionCode: "sin", - RegionName: "Singapore", - Nodes: []*tailcfg.DERPNode{ - { - Name: "3a", - RegionID: 3, - HostName: "derp3.tailscale.com", - IPv4: "68.183.179.66", - IPv6: "2400:6180:0:d1::67d:8001", - }, - }, - }, - }, - }, + AccessURL: accessURLParsed, + ICEServers: iceServers, + Logger: logger.Named("coderd"), + Database: databasefake.New(), + DERPMap: derpMap, Pubsub: database.NewPubsubInMemory(), CacheDir: cacheDir, GoogleTokenValidator: googleTokenValidator, @@ -774,14 +742,15 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { cliflag.DurationVarP(root.Flags(), &autobuildPollInterval, "autobuild-poll-interval", "", "CODER_AUTOBUILD_POLL_INTERVAL", time.Minute, "Specifies the interval at which to poll for and execute automated workspace build operations.") cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.") cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.") - cliflag.StringVarP(root.Flags(), &derpConfigURL, "derp-config-url", "", "CODER_DERP_CONFIG_URL", "", "Specifies a URL to periodically fetch a DERP map. See: https://tailscale.com/kb/1118/custom-derp-servers/") + cliflag.StringVarP(root.Flags(), &derpConfigURL, "derp-config-url", "", "CODER_DERP_CONFIG_URL", "", + "Specifies a URL to periodically fetch a DERP map. See: https://tailscale.com/kb/1118/custom-derp-servers/") cliflag.BoolVarP(root.Flags(), &derpServerEnabled, "derp-server-enable", "", "CODER_DERP_SERVER_ENABLE", true, "Specifies whether to enable or disable the embedded DERP server.") cliflag.IntVarP(root.Flags(), &derpServerRegionID, "derp-server-region-id", "", "CODER_DERP_SERVER_REGION_ID", 999, "Specifies the region ID to use for the embedded DERP server.") cliflag.StringVarP(root.Flags(), &derpServerRegionCode, "derp-server-region-code", "", "CODER_DERP_SERVER_REGION_CODE", "coder", "Specifies the region code that is displayed in the Coder UI for the embedded DERP server.") cliflag.StringVarP(root.Flags(), &derpServerRegionName, "derp-server-region-name", "", "CODER_DERP_SERVER_REGION_NAME", "Coder Embedded DERP", "Specifies the region name that is displayed in the Coder UI for the embedded DERP server.") - cliflag.StringArrayVarP(root.Flags(), &derpServerSTUNURLs, "derp-server-stun-urls", "", "CODER_DERP_SERVER_STUN_URLS", []string{ + cliflag.StringArrayVarP(root.Flags(), &derpServerSTUNAddrs, "derp-server-stun-addresses", "", "CODER_DERP_SERVER_STUN_ADDRESSES", []string{ "stun.l.google.com:19302", - }, "Specify URLs for STUN servers to establish P2P connections. Set empty to disable P2P connections entirely.") + }, "Specify addresses for STUN servers to establish P2P connections. Set empty to disable P2P connections entirely.") cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") cliflag.BoolVarP(root.Flags(), &pprofEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.") diff --git a/tailnet/conn.go b/tailnet/conn.go index cb90a83efd488..9f1f5d7526ea6 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -370,18 +370,6 @@ func (c *Conn) Close() error { return nil } -// Node represents a node in the network. -type Node struct { - ID tailcfg.NodeID `json:"id"` - Key key.NodePublic `json:"key"` - DiscoKey key.DiscoPublic `json:"disco"` - PreferredDERP int `json:"preferred_derp"` - DERPLatency map[string]float64 `json:"derp_latency"` - Addresses []netip.Prefix `json:"addresses"` - AllowedIPs []netip.Prefix `json:"allowed_ips"` - Endpoints []string `json:"endpoints"` -} - // This and below is taken _mostly_ verbatim from Tailscale: // https://github.com/tailscale/tailscale/blob/c88bd53b1b7b2fcf7ba302f2e53dd1ce8c32dad4/tsnet/tsnet.go#L459-L494 diff --git a/tailnet/coordinator.go b/tailnet/coordinator.go index 5decbd0966cd9..db8dffa0ebaef 100644 --- a/tailnet/coordinator.go +++ b/tailnet/coordinator.go @@ -5,12 +5,27 @@ import ( "errors" "io" "net" + "net/netip" "sync" "github.com/google/uuid" "golang.org/x/xerrors" + "tailscale.com/tailcfg" + "tailscale.com/types/key" ) +// Node represents a node in the network. +type Node struct { + ID tailcfg.NodeID `json:"id"` + Key key.NodePublic `json:"key"` + DiscoKey key.DiscoPublic `json:"disco"` + PreferredDERP int `json:"preferred_derp"` + DERPLatency map[string]float64 `json:"derp_latency"` + Addresses []netip.Prefix `json:"addresses"` + AllowedIPs []netip.Prefix `json:"allowed_ips"` + Endpoints []string `json:"endpoints"` +} + // ServeCoordinator matches the RW structure of a coordinator to exchange node messages. func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) { errChan := make(chan error, 3) diff --git a/tailnet/derpmap.go b/tailnet/derpmap.go new file mode 100644 index 0000000000000..36ff02b89f9dc --- /dev/null +++ b/tailnet/derpmap.go @@ -0,0 +1,60 @@ +package tailnet + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strconv" + + "golang.org/x/xerrors" + "tailscale.com/tailcfg" +) + +// NewDERPMap constructs a DERPMap from a set of STUN addresses and optionally a remote +// URL to fetch a mapping from e.g. https://controlplane.tailscale.com/derpmap/default. +func NewDERPMap(ctx context.Context, region *tailcfg.DERPRegion, stunAddrs []string, remoteURL string) (*tailcfg.DERPMap, error) { + for index, stunAddr := range stunAddrs { + host, rawPort, err := net.SplitHostPort(stunAddr) + if err != nil { + return nil, xerrors.Errorf("split host port for %q: %w", stunAddr, err) + } + port, err := strconv.Atoi(rawPort) + if err != nil { + return nil, xerrors.Errorf("parse port for %q: %w", stunAddr, err) + } + region.Nodes = append(region.Nodes, &tailcfg.DERPNode{ + Name: fmt.Sprintf("%dstun%d", region.RegionID, index), + RegionID: region.RegionID, + HostName: host, + STUNOnly: true, + STUNPort: port, + }) + } + + derpMap := &tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{}, + } + if remoteURL != "" { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL, nil) + if err != nil { + return nil, xerrors.Errorf("create request: %w", err) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + return nil, xerrors.Errorf("get derpmap: %w", err) + } + defer res.Body.Close() + err = json.NewDecoder(res.Body).Decode(&derpMap) + if err != nil { + return nil, xerrors.Errorf("fetch derpmap: %w", err) + } + } + _, conflicts := derpMap.Regions[region.RegionID] + if conflicts { + return nil, xerrors.Errorf("the default region ID conflicts with a remote region from %q", remoteURL) + } + derpMap.Regions[region.RegionID] = region + return derpMap, nil +} diff --git a/tailnet/derpmap_test.go b/tailnet/derpmap_test.go new file mode 100644 index 0000000000000..252ccef907c20 --- /dev/null +++ b/tailnet/derpmap_test.go @@ -0,0 +1,60 @@ +package tailnet_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + "tailscale.com/tailcfg" + + "github.com/coder/coder/tailnet" +) + +func TestNewDERPMap(t *testing.T) { + t.Parallel() + t.Run("WithoutRemoteURL", func(t *testing.T) { + t.Parallel() + derpMap, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{ + RegionID: 1, + Nodes: []*tailcfg.DERPNode{{}}, + }, []string{"stun.google.com:2345"}, "") + require.NoError(t, err) + require.Len(t, derpMap.Regions[1].Nodes, 2) + }) + t.Run("RemoteURL", func(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, _ := json.Marshal(&tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: {}, + }, + }) + _, _ = w.Write(data) + })) + t.Cleanup(server.Close) + derpMap, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{ + RegionID: 2, + }, []string{}, server.URL) + require.NoError(t, err) + require.Len(t, derpMap.Regions, 2) + }) + t.Run("RemoteConflicts", func(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, _ := json.Marshal(&tailcfg.DERPMap{ + Regions: map[int]*tailcfg.DERPRegion{ + 1: {}, + }, + }) + _, _ = w.Write(data) + })) + t.Cleanup(server.Close) + _, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{ + RegionID: 1, + }, []string{}, server.URL) + require.Error(t, err) + }) +} From fd29e7ce7773a57f9193a6784902b5dab936c96f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 03:27:16 +0000 Subject: [PATCH 48/54] Expose DERP region names to the API --- coderd/provisionerjobs.go | 2 +- coderd/workspaceagents.go | 30 +++++++++++++++++++++++------- coderd/workspaceresources.go | 2 +- codersdk/workspaceresources.go | 12 +++++++----- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index ddae2b188f812..19e802fc35440 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -264,7 +264,7 @@ func (api *API) provisionerJobResources(rw http.ResponseWriter, r *http.Request, } } - apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading job agent.", diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index a1f45203715da..9ea83defdb9a5 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -10,12 +10,14 @@ import ( "net/http" "net/netip" "strconv" + "strings" "time" "github.com/google/uuid" "github.com/hashicorp/yamux" "golang.org/x/xerrors" "nhooyr.io/websocket" + "tailscale.com/tailcfg" "cdr.dev/slog" "github.com/coder/coder/agent" @@ -46,7 +48,7 @@ func (api *API) workspaceAgent(rw http.ResponseWriter, r *http.Request) { }) return } - apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -70,7 +72,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -121,7 +123,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) { func (api *API) workspaceAgentMetadata(rw http.ResponseWriter, r *http.Request) { workspaceAgent := httpmw.WorkspaceAgent(r) - apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -376,7 +378,7 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { httpapi.ResourceNotFound(rw) return } - apiAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) + apiAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, workspaceAgent, nil, api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", @@ -554,7 +556,7 @@ func convertApps(dbApps []database.WorkspaceApp) []codersdk.WorkspaceApp { return apps } -func convertWorkspaceAgent(coordinator *tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { +func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator *tailnet.Coordinator, dbAgent database.WorkspaceAgent, apps []codersdk.WorkspaceApp, agentInactiveDisconnectTimeout time.Duration) (codersdk.WorkspaceAgent, error) { var envs map[string]string if dbAgent.EnvironmentVariables.Valid { err := json.Unmarshal(dbAgent.EnvironmentVariables.RawMessage, &envs) @@ -578,8 +580,22 @@ func convertWorkspaceAgent(coordinator *tailnet.Coordinator, dbAgent database.Wo } node := coordinator.Node(dbAgent.ID) if node != nil { - workspaceAgent.PreferredDERP = node.PreferredDERP - workspaceAgent.DERPLatency = node.DERPLatency + workspaceAgent.DERPLatency = map[string]codersdk.DERPRegion{} + for rawRegion, latency := range node.DERPLatency { + regionParts := strings.SplitN(rawRegion, "-", 2) + regionID, err := strconv.Atoi(regionParts[0]) + if err != nil { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("convert derp region id %q: %w", rawRegion, err) + } + region, found := derpMap.Regions[regionID] + if !found { + return codersdk.WorkspaceAgent{}, xerrors.Errorf("region %d not found in derpmap", regionID) + } + workspaceAgent.DERPLatency[region.RegionName] = codersdk.DERPRegion{ + Preferred: node.PreferredDERP == regionID, + LatencyMilliseconds: latency * 1000, + } + } } if dbAgent.FirstConnectedAt.Valid { diff --git a/coderd/workspaceresources.go b/coderd/workspaceresources.go index 8694fa47050ae..416390132f48a 100644 --- a/coderd/workspaceresources.go +++ b/coderd/workspaceresources.go @@ -70,7 +70,7 @@ func (api *API) workspaceResource(rw http.ResponseWriter, r *http.Request) { } } - convertedAgent, err := convertWorkspaceAgent(api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) + convertedAgent, err := convertWorkspaceAgent(api.DERPMap, api.TailnetCoordinator, agent, convertApps(dbApps), api.AgentInactiveDisconnectTimeout) if err != nil { httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error reading workspace agent.", diff --git a/codersdk/workspaceresources.go b/codersdk/workspaceresources.go index cdc98a69568bd..463448e6cda7f 100644 --- a/codersdk/workspaceresources.go +++ b/codersdk/workspaceresources.go @@ -35,6 +35,11 @@ type WorkspaceResourceMetadata struct { Sensitive bool `json:"sensitive"` } +type DERPRegion struct { + Preferred bool `json:"preferred"` + LatencyMilliseconds float64 `json:"latency_ms"` +} + type WorkspaceAgent struct { ID uuid.UUID `json:"id"` CreatedAt time.Time `json:"created_at"` @@ -52,11 +57,8 @@ type WorkspaceAgent struct { StartupScript string `json:"startup_script,omitempty"` Directory string `json:"directory,omitempty"` Apps []WorkspaceApp `json:"apps"` - // PreferredDERP represents the connected region. - PreferredDERP int `json:"preferred_derp"` - // Maps DERP region to MS latency. - // Fetch the DERP mapping to extract region names! - DERPLatency map[string]float64 `json:"latency"` + // DERPLatency is mapped by region name (e.g. "New York City", "Seattle"). + DERPLatency map[string]DERPRegion `json:"latency"` } type WorkspaceAgentResourceMetadata struct { From 77dda4a8c865ab70f315889847d555091bae6b0f Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 14:22:09 +0000 Subject: [PATCH 49/54] Add global option to enable Tailscale networking for web --- cli/server.go | 1 + coderd/coderd.go | 7 +++- coderd/workspaceagents.go | 70 ++++++++++++++++++++++++++++++++++ site/src/api/typesGenerated.ts | 9 ++++- 4 files changed, 84 insertions(+), 3 deletions(-) diff --git a/cli/server.go b/cli/server.go index a43aeaf649be3..2c160ef7757b0 100644 --- a/cli/server.go +++ b/cli/server.go @@ -355,6 +355,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { GoogleTokenValidator: googleTokenValidator, SecureAuthCookie: secureAuthCookie, SSHKeygenAlgorithm: sshKeygenAlgorithm, + TailscaleEnable: tailscaleEnable, TURNServer: turnServer, TracerProvider: tracerProvider, Telemetry: telemetry.NewNoop(), diff --git a/coderd/coderd.go b/coderd/coderd.go index 5ef526b4de4e9..e2a7abf48b9d3 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -73,6 +73,7 @@ type Options struct { LicenseHandler http.Handler FeaturesService FeaturesService + TailscaleEnable bool TailnetCoordinator *tailnet.Coordinator DERPMap *tailcfg.DERPMap } @@ -130,7 +131,11 @@ func New(options *Options) *API { Logger: options.Logger, }, } - api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0) + if options.TailscaleEnable { + api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgentTailnet, 0) + } else { + api.workspaceAgentCache = wsconncache.New(api.dialWorkspaceAgent, 0) + } api.derpServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger)) oauthConfigs := &httpmw.OAuth2Configs{ Github: options.GithubOAuth2Config, diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 9ea83defdb9a5..ba854036d436f 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -26,7 +26,9 @@ import ( "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/tracing" + "github.com/coder/coder/coderd/turnconn" "github.com/coder/coder/codersdk" + "github.com/coder/coder/peer" "github.com/coder/coder/peerbroker" "github.com/coder/coder/peerbroker/proto" "github.com/coder/coder/provisionersdk" @@ -445,6 +447,74 @@ func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { _, _ = io.Copy(ptNetConn, wsNetConn) } +// dialWorkspaceAgent connects to a workspace agent by ID. Only rely on +// r.Context() for cancellation if it's use is safe or r.Hijack() has +// not been performed. +func (api *API) dialWorkspaceAgent(r *http.Request, agentID uuid.UUID) (agent.Conn, error) { + client, server := provisionersdk.TransportPipe() + ctx, cancelFunc := context.WithCancel(context.Background()) + go func() { + _ = peerbroker.ProxyListen(ctx, server, peerbroker.ProxyOptions{ + ChannelID: agentID.String(), + Logger: api.Logger.Named("peerbroker-proxy-dial"), + Pubsub: api.Pubsub, + }) + _ = client.Close() + _ = server.Close() + }() + + peerClient := proto.NewDRPCPeerBrokerClient(provisionersdk.Conn(client)) + stream, err := peerClient.NegotiateConnection(ctx) + if err != nil { + cancelFunc() + return nil, xerrors.Errorf("negotiate: %w", err) + } + options := &peer.ConnOptions{ + Logger: api.Logger.Named("agent-dialer"), + } + options.SettingEngine.SetSrflxAcceptanceMinWait(0) + options.SettingEngine.SetRelayAcceptanceMinWait(0) + // Use the ProxyDialer for the TURN server. + // This is required for connections where P2P is not enabled. + options.SettingEngine.SetICEProxyDialer(turnconn.ProxyDialer(func() (c net.Conn, err error) { + clientPipe, serverPipe := net.Pipe() + go func() { + <-ctx.Done() + _ = clientPipe.Close() + _ = serverPipe.Close() + }() + localAddress, _ := r.Context().Value(http.LocalAddrContextKey).(*net.TCPAddr) + remoteAddress := &net.TCPAddr{ + IP: net.ParseIP(r.RemoteAddr), + } + // By default requests have the remote address and port. + host, port, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return nil, xerrors.Errorf("split remote address: %w", err) + } + remoteAddress.IP = net.ParseIP(host) + remoteAddress.Port, err = strconv.Atoi(port) + if err != nil { + return nil, xerrors.Errorf("convert remote port: %w", err) + } + api.TURNServer.Accept(clientPipe, remoteAddress, localAddress) + return serverPipe, nil + })) + peerConn, err := peerbroker.Dial(stream, append(api.ICEServers, turnconn.Proxy), options) + if err != nil { + cancelFunc() + return nil, xerrors.Errorf("dial: %w", err) + } + go func() { + <-peerConn.Closed() + cancelFunc() + }() + return &agent.WebRTCConn{ + Negotiator: peerClient, + Conn: peerConn, + }, nil +} + func (api *API) dialWorkspaceAgentTailnet(r *http.Request, agentID uuid.UUID) (agent.Conn, error) { clientConn, serverConn := net.Pipe() go func() { diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index fd3a26de5df96..4bf0f188308ac 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -140,6 +140,12 @@ export interface CreateWorkspaceRequest { readonly parameter_values?: CreateParameterRequest[] } +// From codersdk/workspaceresources.go +export interface DERPRegion { + readonly preferred: boolean + readonly latency_ms: number +} + // From codersdk/features.go export interface Entitlements { readonly features: Record @@ -470,8 +476,7 @@ export interface WorkspaceAgent { readonly startup_script?: string readonly directory?: string readonly apps: WorkspaceApp[] - readonly preferred_derp: number - readonly latency: Record + readonly latency: Record } // From codersdk/workspaceagents.go From b9f2b772ee7142d1bbd196e3c2f98dba32d630f5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 14:24:43 +0000 Subject: [PATCH 50/54] Mark DERP flags hidden while testing --- cli/server.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cli/server.go b/cli/server.go index 2c160ef7757b0..f6f5534f3604c 100644 --- a/cli/server.go +++ b/cli/server.go @@ -752,6 +752,15 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command { cliflag.StringArrayVarP(root.Flags(), &derpServerSTUNAddrs, "derp-server-stun-addresses", "", "CODER_DERP_SERVER_STUN_ADDRESSES", []string{ "stun.l.google.com:19302", }, "Specify addresses for STUN servers to establish P2P connections. Set empty to disable P2P connections entirely.") + + // Mark hidden while this feature is in testing! + _ = root.Flags().MarkHidden("derp-config-url") + _ = root.Flags().MarkHidden("derp-server-enable") + _ = root.Flags().MarkHidden("derp-server-region-id") + _ = root.Flags().MarkHidden("derp-server-region-code") + _ = root.Flags().MarkHidden("derp-server-region-name") + _ = root.Flags().MarkHidden("derp-server-stun-addresses") + cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.") cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.") cliflag.BoolVarP(root.Flags(), &pprofEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.") From b18b823460f7cac23b2179ac75a42432e92c4fb2 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 20:49:56 +0000 Subject: [PATCH 51/54] Update DERP map on reconnect --- agent/agent.go | 1 + tailnet/conn.go | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/agent/agent.go b/agent/agent.go index 919269f0e6c1b..da19c4ac61de7 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -181,6 +181,7 @@ func (a *agent) runTailnet(ctx context.Context, derpMap *tailcfg.DERPMap) { return } if a.network != nil { + a.network.SetDERPMap(derpMap) return } var err error diff --git a/tailnet/conn.go b/tailnet/conn.go index 9f1f5d7526ea6..84aed3fc1ae81 100644 --- a/tailnet/conn.go +++ b/tailnet/conn.go @@ -285,6 +285,13 @@ func (c *Conn) SetNodeCallback(callback func(node *Node)) { }) } +// SetDERPMap updates the DERPMap of a connection. +func (c *Conn) SetDERPMap(derpMap *tailcfg.DERPMap) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.wireguardEngine.SetDERPMap(derpMap) +} + // UpdateNodes connects with a set of peers. This can be constantly updated, // and peers will continually be reconnected as necessary. func (c *Conn) UpdateNodes(nodes []*Node) error { From 7b774c90dc2834233a52d2d24f27ac56d44f7ea5 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 20:59:15 +0000 Subject: [PATCH 52/54] Add close func to workspace agents --- agent/conn.go | 8 ++++++++ codersdk/workspaceagents.go | 19 +++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/agent/conn.go b/agent/conn.go index e311668369237..0e95e97e21254 100644 --- a/agent/conn.go +++ b/agent/conn.go @@ -139,6 +139,7 @@ func (c *WebRTCConn) Close() error { type TailnetConn struct { *tailnet.Conn + CloseFunc func() } func (c *TailnetConn) Ping() (time.Duration, error) { @@ -163,6 +164,13 @@ func (c *TailnetConn) CloseWithError(_ error) error { return c.Close() } +func (c *TailnetConn) Close() error { + if c.CloseFunc != nil { + c.CloseFunc() + } + return c.Conn.Close() +} + type reconnectingPTYInit struct { ID string Height uint16 diff --git a/codersdk/workspaceagents.go b/codersdk/workspaceagents.go index cc03d09040a5e..816d4ea4a01e9 100644 --- a/codersdk/workspaceagents.go +++ b/codersdk/workspaceagents.go @@ -319,18 +319,6 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg if err != nil { return nil, xerrors.Errorf("create tailnet: %w", err) } - go func() { - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - logger.Info(ctx, "ipn", slog.F("status", conn.Status())) - } - }() coordinateURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/workspaceagents/%s/coordinate", agentID)) if err != nil { @@ -347,7 +335,10 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg httpClient := &http.Client{ Jar: jar, } + ctx, cancelFunc := context.WithCancel(ctx) + closed := make(chan struct{}) go func() { + defer close(closed) for retrier := retry.New(50*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { logger.Debug(ctx, "connecting") // nolint:bodyclose @@ -380,6 +371,10 @@ func (c *Client) DialWorkspaceAgentTailnet(ctx context.Context, logger slog.Logg }() return &agent.TailnetConn{ Conn: conn, + CloseFunc: func() { + cancelFunc() + <-closed + }, }, nil } From b62b46c96b90ce18d4d1ceb02be4cde5f2ab288a Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 23:39:15 +0000 Subject: [PATCH 53/54] Fix race condition in upstream dependency --- go.mod | 3 +++ go.sum | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index c88d4aced6ea1..2b8480f592d66 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,9 @@ replace github.com/pion/udp => github.com/mafredri/udp v0.1.2-0.20220805105907-b // https://github.com/hashicorp/hc-install/pull/68 replace github.com/hashicorp/hc-install => github.com/mafredri/hc-install v0.4.1-0.20220727132613-e91868e28445 +// https://github.com/tcnksm/go-httpstat/pull/29 +replace github.com/tcnksm/go-httpstat => github.com/kylecarbs/go-httpstat v0.0.0-20220831233600-c91452099472 + // There are a few minor changes we make to Tailscale that we're slowly upstreaming. Compare here: // https://github.com/tailscale/tailscale/compare/main...coder:tailscale:main replace tailscale.com => github.com/coder/tailscale v1.1.1-0.20220831012541-a77bda274fd6 diff --git a/go.sum b/go.sum index fb393d3457085..f479632bf4fb2 100644 --- a/go.sum +++ b/go.sum @@ -1210,6 +1210,8 @@ github.com/kulti/thelper v0.4.0/go.mod h1:vMu2Cizjy/grP+jmsvOFDx1kYP6+PD1lqg4Yu5 github.com/kunwardeep/paralleltest v1.0.3/go.mod h1:vLydzomDFpk7yu5UX02RmP0H8QfRPOV/oFhWN85Mjb4= github.com/kylecarbs/embedded-postgres v1.17.1-0.20220615202325-461532cecd3a h1:uOnis+HNE6e6eR17YlqzKk51GDahd7E/FacnZxS8h8w= github.com/kylecarbs/embedded-postgres v1.17.1-0.20220615202325-461532cecd3a/go.mod h1:0B+3bPsMvcNgR9nN+bdM2x9YaNYDnf3ksUqYp1OAub0= +github.com/kylecarbs/go-httpstat v0.0.0-20220831233600-c91452099472 h1:KXbxoQY9tOxgacpw0vbHWfIb56Xuzgi0Oql5yr6RYaA= +github.com/kylecarbs/go-httpstat v0.0.0-20220831233600-c91452099472/go.mod h1:MdOqT7wdglCBuU45KzMIvO+xdKlCGHPUWwdTxytqHBU= github.com/kylecarbs/opencensus-go v0.23.1-0.20220307014935-4d0325a68f8b h1:1Y1X6aR78kMEQE1iCjQodB3lA7VO4jB88Wf8ZrzXSsA= github.com/kylecarbs/opencensus-go v0.23.1-0.20220307014935-4d0325a68f8b/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= github.com/kylecarbs/readline v0.0.0-20220211054233-0d62993714c8/go.mod h1:n/KX1BZoN1m9EwoXkn/xAV4fd3k8c++gGBsgLONaPOY= @@ -1788,8 +1790,6 @@ github.com/tailscale/goupnp v1.0.1-0.20210804011211-c64d0f06ea05/go.mod h1:PdCqy github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 h1:zrsUcqrG2uQSPhaUPjUQwozcRdDdSxxqhNgNZ3drZFk= github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0= -github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/tdewolff/parse/v2 v2.6.0 h1:f2D7w32JtqjCv6SczWkfwK+m15et42qEtDnZXHoNY70= github.com/tdewolff/parse/v2 v2.6.0/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= From 0606551cc554c5f86c0435cd2a136e294be2349e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 31 Aug 2022 23:43:49 +0000 Subject: [PATCH 54/54] Fix feature columns race condition --- cli/features.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cli/features.go b/cli/features.go index 1995153275eaf..f430534330816 100644 --- a/cli/features.go +++ b/cli/features.go @@ -13,8 +13,6 @@ import ( "github.com/coder/coder/codersdk" ) -var featureColumns = []string{"Name", "Entitlement", "Enabled", "Limit", "Actual"} - func features() *cobra.Command { cmd := &cobra.Command{ Short: "List features", @@ -29,8 +27,9 @@ func features() *cobra.Command { func featuresList() *cobra.Command { var ( - columns []string - outputFormat string + featureColumns = []string{"Name", "Entitlement", "Enabled", "Limit", "Actual"} + columns []string + outputFormat string ) cmd := &cobra.Command{