Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat(devtunnel): support geodistributed tunnels #2711

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 1, 2022
Merged

feat(devtunnel): support geodistributed tunnels #2711

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
42fb7af
feat(devtunnel): support geodistributed tunnels
coadler Jun 29, 2022
9e94ec1
fixup! feat(devtunnel): support geodistributed tunnels
coadler Jun 30, 2022
8a7c64f
Merge branch 'main' into colin/tunnelv2
coadler Jun 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions coderd/devtunnel/servers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package devtunnel

import (
"runtime"
"sync"
"time"

"github.com/go-ping/ping"
"golang.org/x/exp/slices"
"golang.org/x/sync/errgroup"

"github.com/coder/coder/cryptorand"
)

type Region struct {
ID int
LocationName string
Nodes []Node
}

type Node struct {
ID int `json:"id"`
HostnameHTTPS string `json:"hostname_https"`
HostnameWireguard string `json:"hostname_wireguard"`
WireguardPort uint16 `json:"wireguard_port"`

AvgLatency time.Duration `json:"avg_latency"`
}

var Regions = []Region{
{
ID: 1,
LocationName: "US East Pittsburgh",
Nodes: []Node{
{
ID: 1,
HostnameHTTPS: "pit-1.try.coder.app",
HostnameWireguard: "pit-1.try.coder.app",
WireguardPort: 55551,
Comment on lines +38 to +39
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These two fields could be returned by the server instead when it registers

},
},
},
}

func FindClosestNode() (Node, error) {
nodes := []Node{}

for _, region := range Regions {
// Pick a random node from each region.
i, err := cryptorand.Intn(len(region.Nodes))
if err != nil {
return Node{}, err
}
nodes = append(nodes, region.Nodes[i])
}

var (
nodesMu sync.Mutex
eg = errgroup.Group{}
)
for i, node := range nodes {
i, node := i, node
eg.Go(func() error {
pinger, err := ping.NewPinger(node.HostnameHTTPS)
if err != nil {
return err
}

if runtime.GOOS == "windows" {
pinger.SetPrivileged(true)
}

pinger.Count = 5
err = pinger.Run()
if err != nil {
return err
}

nodesMu.Lock()
nodes[i].AvgLatency = pinger.Statistics().AvgRtt
nodesMu.Unlock()
return nil
})
}

err := eg.Wait()
if err != nil {
return Node{}, err
}

slices.SortFunc(nodes, func(i, j Node) bool {
return i.AvgLatency < j.AvgLatency
})
return nodes[0], nil
}
126 changes: 90 additions & 36 deletions coderd/devtunnel/tunnel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@ import (
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"

"cdr.dev/slog"
"github.com/coder/coder/cryptorand"
)

const (
EndpointWireguard = "wg-tunnel-udp.coder.app"
EndpointHTTPS = "wg-tunnel.coder.app"
var (
v0EndpointHTTPS = "wg-tunnel.coder.app"

ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
ServerUUID = "fcad0000-0000-4000-8000-000000000001"
v0ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0="
v0ServerIP = netip.AddrFrom16(uuid.MustParse("fcad0000-0000-4000-8000-000000000001"))
)

type Tunnel struct {
Expand All @@ -39,47 +39,54 @@ type Tunnel struct {
}

type Config struct {
Version int `json:"version"`
ID uuid.UUID `json:"id"`
PrivateKey device.NoisePrivateKey `json:"private_key"`
PublicKey device.NoisePublicKey `json:"public_key"`

Tunnel Node `json:"tunnel"`
}
type configExt struct {
Version int `json:"-"`
ID uuid.UUID `json:"id"`
PrivateKey device.NoisePrivateKey `json:"-"`
PublicKey device.NoisePublicKey `json:"public_key"`

Tunnel Node `json:"-"`
}

// NewWithConfig calls New with the given config. For documentation, see New.
func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel, <-chan error, error) {
routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
server, routineEnd, err := startUpdateRoutine(ctx, logger, cfg)
if err != nil {
return nil, nil, xerrors.Errorf("start update routine: %w", err)
}

tun, tnet, err := netstack.CreateNetTUN(
[]netip.Addr{netip.AddrFrom16(cfg.ID)},
[]netip.Addr{server.ClientIP},
[]netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})},
1280,
)
if err != nil {
return nil, nil, xerrors.Errorf("create net TUN: %w", err)
}

wgip, err := net.ResolveIPAddr("ip", EndpointWireguard)
wgip, err := net.ResolveIPAddr("ip", cfg.Tunnel.HostnameWireguard)
if err != nil {
return nil, nil, xerrors.Errorf("resolve endpoint: %w", err)
}

dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelSilent, ""))
err = dev.IpcSet(fmt.Sprintf(`private_key=%s
public_key=%s
endpoint=%s:55555
endpoint=%s:%d
persistent_keepalive_interval=21
allowed_ip=%s/128`,
hex.EncodeToString(cfg.PrivateKey[:]),
encodeBase64ToHex(ServerPublicKey),
server.ServerPublicKey,
wgip.IP.String(),
netip.AddrFrom16(uuid.MustParse(ServerUUID)).String(),
cfg.Tunnel.WireguardPort,
server.ServerIP.String(),
))
if err != nil {
return nil, nil, xerrors.Errorf("configure wireguard ipc: %w", err)
Expand Down Expand Up @@ -110,7 +117,7 @@ allowed_ip=%s/128`,
}()

return &Tunnel{
URL: fmt.Sprintf("https://%s.%s", cfg.ID, EndpointHTTPS),
URL: fmt.Sprintf("https://%s", server.Hostname),
Listener: wgListen,
}, ch, nil
}
Expand All @@ -129,11 +136,11 @@ func New(ctx context.Context, logger slog.Logger) (*Tunnel, <-chan error, error)
return NewWithConfig(ctx, logger, cfg)
}

func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-chan struct{}, error) {
func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (ServerResponse, <-chan struct{}, error) {
// Ensure we send the first config before spawning in the background.
_, err := sendConfigToServer(ctx, cfg)
res, err := sendConfigToServer(ctx, cfg)
if err != nil {
return nil, xerrors.Errorf("send config to server: %w", err)
return ServerResponse{}, nil, xerrors.Errorf("send config to server: %w", err)
}

endCh := make(chan struct{})
Expand All @@ -156,29 +163,67 @@ func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-
}
}
}()
return endCh, nil
return res, endCh, nil
}

type ServerResponse struct {
Hostname string `json:"hostname"`
ServerIP netip.Addr `json:"server_ip"`
ServerPublicKey string `json:"server_public_key"` // hex
ClientIP netip.Addr `json:"client_ip"`
}

func sendConfigToServer(ctx context.Context, cfg Config) (created bool, err error) {
func sendConfigToServer(ctx context.Context, cfg Config) (ServerResponse, error) {
raw, err := json.Marshal(configExt(cfg))
if err != nil {
return false, xerrors.Errorf("marshal config: %w", err)
return ServerResponse{}, xerrors.Errorf("marshal config: %w", err)
}

req, err := http.NewRequestWithContext(ctx, "POST", "https://"+EndpointHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return false, xerrors.Errorf("new request: %w", err)
var req *http.Request
switch cfg.Version {
case 0:
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+v0EndpointHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
}

case 1:
req, err = http.NewRequestWithContext(ctx, "POST", "https://"+cfg.Tunnel.HostnameHTTPS+"/tun", bytes.NewReader(raw))
if err != nil {
return ServerResponse{}, xerrors.Errorf("new request: %w", err)
}

default:
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
}

res, err := http.DefaultClient.Do(req)
if err != nil {
return false, xerrors.Errorf("do request: %w", err)
return ServerResponse{}, xerrors.Errorf("do request: %w", err)
}
defer res.Body.Close()

var resp ServerResponse
switch cfg.Version {
case 0:
_, _ = io.Copy(io.Discard, res.Body)
resp.Hostname = fmt.Sprintf("%s.%s", cfg.ID, v0EndpointHTTPS)
resp.ServerIP = v0ServerIP
resp.ServerPublicKey = encodeBase64ToHex(v0ServerPublicKey)
resp.ClientIP = netip.AddrFrom16(cfg.ID)

case 1:
err := json.NewDecoder(res.Body).Decode(&resp)
if err != nil {
return ServerResponse{}, xerrors.Errorf("decode response: %w", err)
}

_, _ = io.Copy(io.Discard, res.Body)
_ = res.Body.Close()
default:
_, _ = io.Copy(io.Discard, res.Body)
return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version)
}

return res.StatusCode == http.StatusCreated, nil
return resp, nil
}

func cfgPath() (string, error) {
Expand Down Expand Up @@ -227,6 +272,15 @@ func readOrGenerateConfig() (Config, error) {
return Config{}, xerrors.Errorf("unmarshal config: %w", err)
}

if cfg.Version == 0 {
cfg.Tunnel = Node{
ID: 0,
HostnameHTTPS: "wg-tunnel.coder.app",
HostnameWireguard: "wg-tunnel-udp.coder.app",
WireguardPort: 55555,
}
}

return cfg, nil
}

Expand All @@ -235,25 +289,25 @@ func GenerateConfig() (Config, error) {
if err != nil {
return Config{}, xerrors.Errorf("generate private key: %w", err)
}

pub := priv.PublicKey()

node, err := FindClosestNode()
if err != nil {
region := Regions[0]
n, _ := cryptorand.Intn(len(region.Nodes))
node = region.Nodes[n]
_, _ = fmt.Println("Error picking closest dev tunnel:", err)
_, _ = fmt.Println("Defaulting to", Regions[0].LocationName)
}

return Config{
ID: newUUID(),
Version: 1,
PrivateKey: device.NoisePrivateKey(priv),
PublicKey: device.NoisePublicKey(pub),
Tunnel: node,
}, nil
}

func newUUID() uuid.UUID {
u := uuid.New()
// 0xfc is the IPV6 prefix for internal networks.
u[0] = 0xfc
u[1] = 0xca

return u
}

func writeConfig(cfg Config) error {
cfgFi, err := cfgPath()
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ require (
github.com/go-chi/chi/v5 v5.0.7
github.com/go-chi/httprate v0.5.3
github.com/go-chi/render v1.0.1
github.com/go-ping/ping v1.1.0
github.com/go-playground/validator/v10 v10.11.0
github.com/gofrs/flock v0.8.1
github.com/gohugoio/hugo v0.101.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -689,6 +689,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
Expand Down