From 42fb7aff5741e4fc29a6c1b0e0d9d655869e56e7 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Tue, 28 Jun 2022 21:16:45 -0500 Subject: [PATCH 1/2] feat(devtunnel): support geodistributed tunnels --- coderd/devtunnel/servers.go | 95 ++++++++++++++++++++++++++++ coderd/devtunnel/tunnel.go | 121 +++++++++++++++++++++++++----------- go.mod | 1 + go.sum | 2 + 4 files changed, 183 insertions(+), 36 deletions(-) create mode 100644 coderd/devtunnel/servers.go diff --git a/coderd/devtunnel/servers.go b/coderd/devtunnel/servers.go new file mode 100644 index 0000000000000..c8c9996d02512 --- /dev/null +++ b/coderd/devtunnel/servers.go @@ -0,0 +1,95 @@ +package devtunnel + +import ( + "runtime" + "sync" + "time" + + "github.com/go-ping/ping" + "golang.org/x/exp/slices" + "golang.org/x/sync/errgroup" + + "github.com/coder/coder/cryptorand" +) + +type TunnelRegion struct { + ID int + LocationName string + Nodes []TunnelNode +} + +type TunnelNode struct { + ID int `json:"id"` + HostnameHTTPS string `json:"hostname_https"` + HostnameWireguard string `json:"hostname_wireguard"` + WireguardPort uint16 `json:"wireguard_port"` + + AvgLatency time.Duration `json:"avg_latency"` +} + +var TunnelRegions = []TunnelRegion{ + { + ID: 1, + LocationName: "US East Pittsburgh", + Nodes: []TunnelNode{ + { + ID: 1, + HostnameHTTPS: "pit-1.try.coder.app", + HostnameWireguard: "pit-1.try.coder.app", + WireguardPort: 55551, + }, + }, + }, +} + +func PickTunnelNode() (TunnelNode, error) { + nodes := []TunnelNode{} + + for _, region := range TunnelRegions { + // Pick a random node from each region. + i, err := cryptorand.Intn(len(region.Nodes)) + if err != nil { + return TunnelNode{}, err + } + nodes = append(nodes, region.Nodes[i]) + } + + var ( + nodesMu sync.Mutex + eg = errgroup.Group{} + ) + for i, node := range nodes { + i, node := i, node + eg.Go(func() error { + pinger, err := ping.NewPinger(node.HostnameHTTPS) + if err != nil { + return err + } + + if runtime.GOOS == "windows" { + pinger.SetPrivileged(true) + } + + pinger.Count = 5 + err = pinger.Run() + if err != nil { + return err + } + + nodesMu.Lock() + nodes[i].AvgLatency = pinger.Statistics().AvgRtt + nodesMu.Unlock() + return nil + }) + } + + err := eg.Wait() + if err != nil { + return TunnelNode{}, err + } + + slices.SortFunc(nodes, func(i, j TunnelNode) bool { + return i.AvgLatency < j.AvgLatency + }) + return nodes[0], nil +} diff --git a/coderd/devtunnel/tunnel.go b/coderd/devtunnel/tunnel.go index a5b8be7d02c4f..177cdbb7ab0b2 100644 --- a/coderd/devtunnel/tunnel.go +++ b/coderd/devtunnel/tunnel.go @@ -25,12 +25,11 @@ import ( "cdr.dev/slog" ) -const ( - EndpointWireguard = "wg-tunnel-udp.coder.app" - EndpointHTTPS = "wg-tunnel.coder.app" +var ( + v0EndpointHTTPS = "wg-tunnel.coder.app" - ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0=" - ServerUUID = "fcad0000-0000-4000-8000-000000000001" + v0ServerPublicKey = "+KNSMwed/IlqoesvTMSBNsHFaKVLrmmaCkn0bxIhUg0=" + v0ServerIP = netip.AddrFrom16(uuid.MustParse("fcad0000-0000-4000-8000-000000000001")) ) type Tunnel struct { @@ -39,25 +38,31 @@ type Tunnel struct { } type Config struct { + Version int `json:"version"` ID uuid.UUID `json:"id"` PrivateKey device.NoisePrivateKey `json:"private_key"` PublicKey device.NoisePublicKey `json:"public_key"` + + Tunnel TunnelNode `json:"tunnel"` } type configExt struct { + Version int `json:"-"` ID uuid.UUID `json:"id"` PrivateKey device.NoisePrivateKey `json:"-"` PublicKey device.NoisePublicKey `json:"public_key"` + + Tunnel TunnelNode `json:"-"` } // NewWithConfig calls New with the given config. For documentation, see New. func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel, <-chan error, error) { - routineEnd, err := startUpdateRoutine(ctx, logger, cfg) + server, routineEnd, err := startUpdateRoutine(ctx, logger, cfg) if err != nil { return nil, nil, xerrors.Errorf("start update routine: %w", err) } tun, tnet, err := netstack.CreateNetTUN( - []netip.Addr{netip.AddrFrom16(cfg.ID)}, + []netip.Addr{server.ClientIP}, []netip.Addr{netip.AddrFrom4([4]byte{1, 1, 1, 1})}, 1280, ) @@ -65,7 +70,7 @@ func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel return nil, nil, xerrors.Errorf("create net TUN: %w", err) } - wgip, err := net.ResolveIPAddr("ip", EndpointWireguard) + wgip, err := net.ResolveIPAddr("ip", cfg.Tunnel.HostnameWireguard) if err != nil { return nil, nil, xerrors.Errorf("resolve endpoint: %w", err) } @@ -73,13 +78,14 @@ func NewWithConfig(ctx context.Context, logger slog.Logger, cfg Config) (*Tunnel dev := device.NewDevice(tun, conn.NewDefaultBind(), device.NewLogger(device.LogLevelSilent, "")) err = dev.IpcSet(fmt.Sprintf(`private_key=%s public_key=%s -endpoint=%s:55555 +endpoint=%s:%d persistent_keepalive_interval=21 allowed_ip=%s/128`, hex.EncodeToString(cfg.PrivateKey[:]), - encodeBase64ToHex(ServerPublicKey), + server.ServerPublicKey, wgip.IP.String(), - netip.AddrFrom16(uuid.MustParse(ServerUUID)).String(), + cfg.Tunnel.WireguardPort, + server.ServerIP.String(), )) if err != nil { return nil, nil, xerrors.Errorf("configure wireguard ipc: %w", err) @@ -110,7 +116,7 @@ allowed_ip=%s/128`, }() return &Tunnel{ - URL: fmt.Sprintf("https://%s.%s", cfg.ID, EndpointHTTPS), + URL: fmt.Sprintf("https://%s", server.Hostname), Listener: wgListen, }, ch, nil } @@ -129,11 +135,11 @@ func New(ctx context.Context, logger slog.Logger) (*Tunnel, <-chan error, error) return NewWithConfig(ctx, logger, cfg) } -func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<-chan struct{}, error) { +func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (ServerResponse, <-chan struct{}, error) { // Ensure we send the first config before spawning in the background. - _, err := sendConfigToServer(ctx, cfg) + res, err := sendConfigToServer(ctx, cfg) if err != nil { - return nil, xerrors.Errorf("send config to server: %w", err) + return ServerResponse{}, nil, xerrors.Errorf("send config to server: %w", err) } endCh := make(chan struct{}) @@ -156,29 +162,67 @@ func startUpdateRoutine(ctx context.Context, logger slog.Logger, cfg Config) (<- } } }() - return endCh, nil + return res, endCh, nil +} + +type ServerResponse struct { + Hostname string `json:"hostname"` + ServerIP netip.Addr `json:"server_ip"` + ServerPublicKey string `json:"server_public_key"` // hex + ClientIP netip.Addr `json:"client_ip"` } -func sendConfigToServer(ctx context.Context, cfg Config) (created bool, err error) { +func sendConfigToServer(ctx context.Context, cfg Config) (ServerResponse, error) { raw, err := json.Marshal(configExt(cfg)) if err != nil { - return false, xerrors.Errorf("marshal config: %w", err) + return ServerResponse{}, xerrors.Errorf("marshal config: %w", err) } - req, err := http.NewRequestWithContext(ctx, "POST", "https://"+EndpointHTTPS+"/tun", bytes.NewReader(raw)) - if err != nil { - return false, xerrors.Errorf("new request: %w", err) + var req *http.Request + switch cfg.Version { + case 0: + req, err = http.NewRequestWithContext(ctx, "POST", "https://"+v0EndpointHTTPS+"/tun", bytes.NewReader(raw)) + if err != nil { + return ServerResponse{}, xerrors.Errorf("new request: %w", err) + } + + case 1: + req, err = http.NewRequestWithContext(ctx, "POST", "https://"+cfg.Tunnel.HostnameHTTPS+"/tun", bytes.NewReader(raw)) + if err != nil { + return ServerResponse{}, xerrors.Errorf("new request: %w", err) + } + + default: + return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version) } res, err := http.DefaultClient.Do(req) if err != nil { - return false, xerrors.Errorf("do request: %w", err) + return ServerResponse{}, xerrors.Errorf("do request: %w", err) } + defer res.Body.Close() + + var resp ServerResponse + switch cfg.Version { + case 0: + _, _ = io.Copy(io.Discard, res.Body) + resp.Hostname = fmt.Sprintf("%s.%s", cfg.ID, v0EndpointHTTPS) + resp.ServerIP = v0ServerIP + resp.ServerPublicKey = encodeBase64ToHex(v0ServerPublicKey) + resp.ClientIP = netip.AddrFrom16(cfg.ID) + + case 1: + err := json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + return ServerResponse{}, xerrors.Errorf("decode response: %w", err) + } - _, _ = io.Copy(io.Discard, res.Body) - _ = res.Body.Close() + default: + _, _ = io.Copy(io.Discard, res.Body) + return ServerResponse{}, xerrors.Errorf("unknown config version: %d", cfg.Version) + } - return res.StatusCode == http.StatusCreated, nil + return resp, nil } func cfgPath() (string, error) { @@ -227,6 +271,15 @@ func readOrGenerateConfig() (Config, error) { return Config{}, xerrors.Errorf("unmarshal config: %w", err) } + if cfg.Version == 0 { + cfg.Tunnel = TunnelNode{ + ID: 0, + HostnameHTTPS: "wg-tunnel.coder.app", + HostnameWireguard: "wg-tunnel-udp.coder.app", + WireguardPort: 55555, + } + } + return cfg, nil } @@ -235,25 +288,21 @@ func GenerateConfig() (Config, error) { if err != nil { return Config{}, xerrors.Errorf("generate private key: %w", err) } - pub := priv.PublicKey() + node, err := PickTunnelNode() + if err != nil { + return Config{}, xerrors.Errorf("pick tunnel node: %w", err) + } + return Config{ - ID: newUUID(), + Version: 1, PrivateKey: device.NoisePrivateKey(priv), PublicKey: device.NoisePublicKey(pub), + Tunnel: node, }, nil } -func newUUID() uuid.UUID { - u := uuid.New() - // 0xfc is the IPV6 prefix for internal networks. - u[0] = 0xfc - u[1] = 0xca - - return u -} - func writeConfig(cfg Config) error { cfgFi, err := cfgPath() if err != nil { diff --git a/go.mod b/go.mod index 281a492c35f26..342aad47ab374 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/httprate v0.5.3 github.com/go-chi/render v1.0.1 + github.com/go-ping/ping v1.1.0 github.com/go-playground/validator/v10 v10.11.0 github.com/gofrs/flock v0.8.1 github.com/gohugoio/hugo v0.101.0 diff --git a/go.sum b/go.sum index 8d87f80957542..8976449c6af0c 100644 --- a/go.sum +++ b/go.sum @@ -689,6 +689,8 @@ github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dp github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw= +github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk= github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= From 9e94ec1a6d35a442aeeceee64babdaafc185fb77 Mon Sep 17 00:00:00 2001 From: Colin Adler Date: Thu, 30 Jun 2022 18:36:30 -0500 Subject: [PATCH 2/2] fixup! feat(devtunnel): support geodistributed tunnels --- coderd/devtunnel/servers.go | 22 +++++++++++----------- coderd/devtunnel/tunnel.go | 15 ++++++++++----- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/coderd/devtunnel/servers.go b/coderd/devtunnel/servers.go index c8c9996d02512..6fc347a2db06e 100644 --- a/coderd/devtunnel/servers.go +++ b/coderd/devtunnel/servers.go @@ -12,13 +12,13 @@ import ( "github.com/coder/coder/cryptorand" ) -type TunnelRegion struct { +type Region struct { ID int LocationName string - Nodes []TunnelNode + Nodes []Node } -type TunnelNode struct { +type Node struct { ID int `json:"id"` HostnameHTTPS string `json:"hostname_https"` HostnameWireguard string `json:"hostname_wireguard"` @@ -27,11 +27,11 @@ type TunnelNode struct { AvgLatency time.Duration `json:"avg_latency"` } -var TunnelRegions = []TunnelRegion{ +var Regions = []Region{ { ID: 1, LocationName: "US East Pittsburgh", - Nodes: []TunnelNode{ + Nodes: []Node{ { ID: 1, HostnameHTTPS: "pit-1.try.coder.app", @@ -42,14 +42,14 @@ var TunnelRegions = []TunnelRegion{ }, } -func PickTunnelNode() (TunnelNode, error) { - nodes := []TunnelNode{} +func FindClosestNode() (Node, error) { + nodes := []Node{} - for _, region := range TunnelRegions { + for _, region := range Regions { // Pick a random node from each region. i, err := cryptorand.Intn(len(region.Nodes)) if err != nil { - return TunnelNode{}, err + return Node{}, err } nodes = append(nodes, region.Nodes[i]) } @@ -85,10 +85,10 @@ func PickTunnelNode() (TunnelNode, error) { err := eg.Wait() if err != nil { - return TunnelNode{}, err + return Node{}, err } - slices.SortFunc(nodes, func(i, j TunnelNode) bool { + slices.SortFunc(nodes, func(i, j Node) bool { return i.AvgLatency < j.AvgLatency }) return nodes[0], nil diff --git a/coderd/devtunnel/tunnel.go b/coderd/devtunnel/tunnel.go index 177cdbb7ab0b2..6a2b4e1c45fb0 100644 --- a/coderd/devtunnel/tunnel.go +++ b/coderd/devtunnel/tunnel.go @@ -23,6 +23,7 @@ import ( "golang.zx2c4.com/wireguard/wgctrl/wgtypes" "cdr.dev/slog" + "github.com/coder/coder/cryptorand" ) var ( @@ -43,7 +44,7 @@ type Config struct { PrivateKey device.NoisePrivateKey `json:"private_key"` PublicKey device.NoisePublicKey `json:"public_key"` - Tunnel TunnelNode `json:"tunnel"` + Tunnel Node `json:"tunnel"` } type configExt struct { Version int `json:"-"` @@ -51,7 +52,7 @@ type configExt struct { PrivateKey device.NoisePrivateKey `json:"-"` PublicKey device.NoisePublicKey `json:"public_key"` - Tunnel TunnelNode `json:"-"` + Tunnel Node `json:"-"` } // NewWithConfig calls New with the given config. For documentation, see New. @@ -272,7 +273,7 @@ func readOrGenerateConfig() (Config, error) { } if cfg.Version == 0 { - cfg.Tunnel = TunnelNode{ + cfg.Tunnel = Node{ ID: 0, HostnameHTTPS: "wg-tunnel.coder.app", HostnameWireguard: "wg-tunnel-udp.coder.app", @@ -290,9 +291,13 @@ func GenerateConfig() (Config, error) { } pub := priv.PublicKey() - node, err := PickTunnelNode() + node, err := FindClosestNode() if err != nil { - return Config{}, xerrors.Errorf("pick tunnel node: %w", err) + region := Regions[0] + n, _ := cryptorand.Intn(len(region.Nodes)) + node = region.Nodes[n] + _, _ = fmt.Println("Error picking closest dev tunnel:", err) + _, _ = fmt.Println("Defaulting to", Regions[0].LocationName) } return Config{