8000 chore: add easy NAT integration tests part 2 by deansheather · Pull Request #13312 · coder/coder · GitHub
[go: up one dir, main page]
Skip to content

chore: add easy NAT integration tests part 2 #13312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 24, 2024
Merged

chore: add easy NAT integration tests part 2 #13312

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d31bd8c
chore: add easy NAT integration tests part 2
deansheather May 20, 2024
9b67a9c
Refactor clientNumber
deansheather May 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
chore: add easy NAT integration tests part 2
  • Loading branch information
@deansheather
deansheather committed May 24, 2024
commit d31bd8c23433570064d0f19d6469e2c07d5e5273
42 changes: 32 additions & 10 deletions tailnet/test/integration/integration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ type TestTopology struct {
Server ServerStarter
// StartClient gets called in each client subprocess. It's expected to
// create the tailnet.Conn and ensure connectivity to it's peer.
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn
StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID uuid.UUID, peerID uuid.UUID) *tailnet.Conn

// RunTests is the main test function. It's called in each of the client
// subprocesses. If tests can only run once, they should check the client ID
Expand Down Expand Up @@ -264,13 +264,18 @@ http {

// StartClientDERP creates a client connection to the server for coordination
// and creates a tailnet.Conn which will only use DERP to connect to the peer.
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
listenPort := uint16(client1Port)
if clientNumber == 2 {
listenPort = client2Port
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

clientNumber feels pretty ambiguous. I think something like type ClientRole int would be clearer. From a quick reading it's hard to tell which is which.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Refactored into a Client struct that contains all the details that gets passed around instead. The client processes still use a "number" so they can select the correct struct as "me", but it's a const now.

return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
DERPMap: derpMap,
BlockEndpoints: true,
Logger: logger,
DERPForceWebSockets: false,
ListenPort: listenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
Expand All @@ -279,13 +284,18 @@ func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, myID,

// StartClientDERPWebSockets does the same thing as StartClientDERP but will
// only use DERP WebSocket fallback.
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
listenPort := uint16(client1Port)
if clientNumber == 2 {
listenPort = client2Port
}
return startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
DERPMap: derpMap,
BlockEndpoints: true,
Logger: logger,
DERPForceWebSockets: true,
ListenPort: listenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
Expand All @@ -295,13 +305,18 @@ func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.
// StartClientDirect does the same thing as StartClientDERP but disables
// BlockEndpoints (which enables Direct connections), and waits for a direct
// connection to be established between the two peers.
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, myID, peerID uuid.UUID) *tailnet.Conn {
func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, clientNumber int, myID, peerID uuid.UUID) *tailnet.Conn {
listenPort := uint16(client1Port)
if clientNumber == 2 {
listenPort = client2Port
}
conn := startClientOptions(t, logger, serverURL, myID, peerID, &tailnet.Options{
Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IPFromUUID(myID), 128)},
DERPMap: basicDERPMap(t, serverURL),
DERPMap: derpMap,
BlockEndpoints: false,
Logger: logger,
DERPForceWebSockets: true,
ListenPort: listenPort,
// These tests don't have internet connection, so we need to force
// magicsock to do anything.
ForceNetworkUp: true,
Expand Down Expand Up @@ -365,10 +380,17 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, my
return conn
}

func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
serverURL, err := url.Parse(serverURLStr)
if err != nil {
return nil, xerrors.Errorf("parse server URL %q: %w", serverURLStr, err)
}

portStr := serverURL.Port()
port, err := strconv.Atoi(portStr)
require.NoError(t, err, "parse server port")
if err != nil {
return nil, xerrors.Errorf("parse port %q: %w", portStr, err)
}

hostname := serverURL.Hostname()
ipv4 := ""
Expand Down Expand Up @@ -399,7 +421,7 @@ func basicDERPMap(t *testing.T, serverURL *url.URL) *tailcfg.DERPMap {
},
},
},
}
}, nil
}

// ExecBackground starts a subprocess with the given flags and returns a
Expand Down
148 changes: 123 additions & 25 deletions tailnet/test/integration/integration_test.go
8000
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,28 @@
package integration_test

import (
"context"
"encoding/json"
"flag"
"fmt"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime"
"strconv"
"syscall"
"testing"
"time"

"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/net/stun/stuntest"
"tailscale.com/tailcfg"
"tailscale.com/types/nettype"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
Expand All @@ -30,17 +39,22 @@ const runTestEnv = "CODER_TAILNET_TESTS"
var (
isSubprocess = flag.Bool("subprocess", false, "Signifies that this is a test subprocess")
testID = flag.String("test-name", "", "Which test is being run")
role = flag.String("role", "", "The role of the test subprocess: server, client")
role = flag.String("role", "", "The role of the test subprocess: server, stun, client")

// Role: server
serverListenAddr = flag.String("server-listen-addr", "", "The address to listen on for the server")

// Role: stun
stunListenAddr = flag.String("stun-listen-addr", "", "The address to listen on for the STUN server")

// Role: client
clientName = flag.String("client-name", "", "The name of the client for logs")
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
clientMyID = flag.String("client-id", "", "The id of the client")
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
clientName = flag.String("client-name", "", "The name of the client for logs")
clientNumber = flag.Int("client-number", 0, "The number of the client")
clientMyID = flag.String("client-id", "", "The id of the client")
clientPeerID = flag.String("client-peer-id", "", "The id of the other client")
clientServerURL = flag.String("client-server-url", "", "The url to connect to the server")
clientDERPMapPath = flag.String("client-derp-map-path", "", "The path to the DERP map file to use on this client")
clientRunTests = flag.Bool("client-run-tests", false, "Run the tests in the client subprocess")
)

func TestMain(m *testing.M) {
Expand Down Expand Up @@ -87,7 +101,7 @@ var topologies = []integration.TestTopology{
// endpoints to connect as routing is enabled between client 1 and
// client 2.
Name: "EasyNATDirect",
SetupNetworking: integration.SetupNetworkingEasyNAT,
SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
Server: integration.SimpleServerOptions{},
StartClient: integration.StartClientDirect,
RunTests: integration.TestSuite,
Expand Down Expand Up @@ -143,17 +157,41 @@ func TestIntegration(t *testing.T) {
log := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
networking := topo.SetupNetworking(t, log)

// Fork the three child processes.
// Useful for debugging network namespaces by avoiding cleanup.
// t.Cleanup(func() {
// time.Sleep(time.Minute * 15)
// })

closeServer := startServerSubprocess(t, topo.Name, networking)

closeSTUN := func() error { return nil }
if networking.STUN.ListenAddr != "" {
closeSTUN = startSTUNSubprocess(t, topo.Name, networking)
}

// Write the DERP maps to a file.
tempDir := t.TempDir()
client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json")
client1DERPMap, err := networking.Client1.ResolveDERPMap()
require.NoError(t, err, "resolve client 1 DERP map")
err = writeDERPMapToFile(client1DERPMapPath, client1DERPMap)
require.NoError(t, err, "write client 1 DERP map")
client2DERPMapPath := filepath.Join(tempDir, "client2-derp-map.json")
client2DERPMap, err := networking.Client2.ResolveDERPMap()
require.NoError(t, err, "resolve client 2 DERP map")
err = writeDERPMapToFile(client2DERPMapPath, client2DERPMap)
require.NoError(t, err, "write client 2 DERP map")

// client1 runs the tests.
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1)
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2)
client1ErrCh, _ := startClientSubprocess(t, topo.Name, networking, 1, client1DERPMapPath)
_, closeClient2 := startClientSubprocess(t, topo.Name, networking, 2, client2DERPMapPath)

// Wait for client1 to exit.
require.NoError(t, <-client1ErrCh, "client 1 exited")

// Close client2 and the server.
require.NoError(t, closeClient2(), "client 2 exited")
require.NoError(t, closeSTUN(), "stun exited")
require.NoError(t, closeServer(), "server exited")
})
}
Expand All @@ -169,10 +207,11 @@ func handleTestSubprocess(t *testing.T) {
}
}
require.NotEmptyf(t, topo.Name, "unknown test topology %q", *testID)
require.Contains(t, []string{"server", "stun", "client"}, *role, "unknown role %q", *role)

testName := topo.Name + "/"
if *role == "server" {
testName += "server"
if *role == "server" || *role == "stun" {
testName += *role
} else {
testName += *clientName
}
Expand All @@ -185,18 +224,34 @@ func handleTestSubprocess(t *testing.T) {
topo.Server.StartServer(t, logger, *serverListenAddr)
// no exit

case "stun":
launchSTUNServer(t, *stunListenAddr)
// no exit

case "client":
logger = logger.Named(*clientName)
if *clientNumber != 1 && *clientNumber != 2 {
t.Fatalf("invalid client number %d", clientNumber)
}
serverURL, err := url.Parse(*clientServerURL)
require.NoErrorf(t, err, "parse server url %q", *clientServerURL)
myID, err := uuid.Parse(*clientMyID)
require.NoErrorf(t, err, "parse client id %q", *clientMyID)
peerID, err := uuid.Parse(*clientPeerID)
require.NoErrorf(t, err, "parse peer id %q", *clientPeerID)

// Load the DERP map.
var derpMap tailcfg.DERPMap
derpMapPath := *clientDERPMapPath
f, err := os.Open(derpMapPath)
require.NoErrorf(t, err, "open DERP map %q", derpMapPath)
err = json.NewDecoder(f).Decode(&derpMap)
_ = f.Close()
require.NoErrorf(t, err, "decode DERP map %q", derpMapPath)

waitForServerAvailable(t, serverURL)

conn := topo.StartClient(t, logger, serverURL, myID, peerID)
conn := topo.StartClient(t, logger, serverURL, &derpMap, *clientNumber, myID, peerID)

if *clientRunTests {
// Wait for connectivity.
Expand All @@ -218,6 +273,23 @@ func handleTestSubprocess(t *testing.T) {
})
}

type forcedAddrPacketListener struct {
addr string
}

var _ nettype.PacketListener = forcedAddrPacketListener{}

func (ln forcedAddrPacketListener) ListenPacket(ctx context.Context, network, _ string) (net.PacketConn, error) {
return nettype.Std{}.ListenPacket(ctx, network, ln.addr)
}

func launchSTUNServer(t *testing.T, listenAddr string) {
ln := forcedAddrPacketListener{addr: listenAddr}
addr, cleanup := stuntest.ServeWithPacketListener(t, ln)
t.Cleanup(cleanup)
assert.Equal(t, listenAddr, addr.String(), "listen address should match forced addr")
}

func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
const delay = 100 * time.Millisecond
const reqTimeout = 2 * time.Second
Expand Down Expand Up @@ -247,45 +319,55 @@ func waitForServerAvailable(t *testing.T, serverURL *url.URL) {
}

func startServerSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
_, closeFn := startSubprocess(t, "server", networking.ProcessServer.NetNS, []string{
_, closeFn := startSubprocess(t, "server", networking.Server.Process.NetNS, []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=server",
"--server-listen-addr=" + networking.ServerListenAddr,
"--server-listen-addr=" + networking.Server.ListenAddr,
})
return closeFn
}

func startSTUNSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking) func() error {
_, closeFn := startSubprocess(t, "stun", networking.STUN.Process.NetNS, []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=stun",
"--stun-listen-addr=" + networking.STUN.ListenAddr,
})
return closeFn
}

func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int) (<-chan error, func() error) {
func startClientSubprocess(t *testing.T, topologyName string, networking integration.TestNetworking, clientNumber int, derpMapPath string) (<-chan error, func() error) {
require.True(t, clientNumber == 1 || clientNumber == 2)

var (
clientName = fmt.Sprintf("client%d", clientNumber)
myID = integration.Client1ID
peerID = integration.Client2ID
accessURL = networking.ServerAccessURLClient1
netNS = networking.ProcessClient1.NetNS
clientName = fmt.Sprintf("client%d", clientNumber)
myID = integration.Client1ID
peerID = integration.Client2ID
clientConfig = networking.Client1
)
if clientNumber == 2 {
myID, peerID = peerID, myID
accessURL = networking.ServerAccessURLClient2
netNS = networking.ProcessClient2.NetNS
clientConfig = networking.Client2
}

flags := []string{
"--subprocess",
"--test-name=" + topologyName,
"--role=client",
"--client-name=" + clientName,
"--client-server-url=" + accessURL,
"--client-number=" + strconv.Itoa(clientNumber),
"--client-server-url=" + clientConfig.ServerAccessURL,
"--client-id=" + myID.String(),
"--client-peer-id=" + peerID.String(),
"--client-derp-map-path=" + derpMapPath,
}
if clientNumber == 1 {
flags = append(flags, "--client-run-tests")
}

return startSubprocess(t, clientName, netNS, flags)
return startSubprocess(t, clientName, clientConfig.Process.NetNS, flags)
}

// startSubprocess launches the test binary with the same flags as the test, but
Expand All @@ -298,3 +380,19 @@ func startSubprocess(t *testing.T, processName string, netNS *os.File, flags []s
args := append(os.Args[1:], append([]string{"-test.v=true"}, flags...)...)
return integration.ExecBackground(t, processName, netNS, name, args)
}

func writeDERPMapToFile(path string, derpMap *tailcfg.DERPMap) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()

enc := json.NewEncoder(f)
enc.SetIndent("", " ")
err = enc.Encode(derpMap)
if err != nil {
return err
}
return nil
}
Loading
0