Integration tests utilities improvements (#4832)

go.mod

@@ -89,7 +89,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.7
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.14.0
+	github.com/onsi/gomega v1.17.0
 	github.com/opencontainers/runc v1.0.3
 	github.com/opencontainers/selinux v1.8.3
 	github.com/otiai10/copy v1.6.0
@@ -106,6 +106,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/urfave/cli v1.22.4
+	github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
 	go.etcd.io/etcd/etcdutl/v3 v3.5.0

go.sum

@@ -853,8 +853,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
-github.com/onsi/gomega v1.14.0 h1:ep6kpPVwmr/nTbklSx2nrLNSIO62DoYAhnPNIMhK8gI=
 github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
+github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
+github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=

pkg/etcd/etcd_int_test.go

@@ -31,7 +31,7 @@ var _ = Describe("etcd snapshots", func() {
 	When("a new etcd is created", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl", "get pods -A")
 			}, "180s", "5s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("saves an etcd snapshot", func() {
@@ -54,7 +54,7 @@ var _ = Describe("etcd snapshots", func() {
 	})
 	When("saving a custom name", func() {
 		It("saves an etcd snapshot with a custom name", func() {
-			Expect(testutil.K3sCmd("etcd-snapshot", "save", "--name", "ALIVEBEEF")).
+			Expect(testutil.K3sCmd("etcd-snapshot", "save --name ALIVEBEEF")).
 				To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF"))
 		})
 		It("deletes that snapshot", func() {
@@ -69,13 +69,13 @@ var _ = Describe("etcd snapshots", func() {
 	})
 	When("using etcd snapshot prune", func() {
 		It("saves 3 different snapshots", func() {
-			Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
+			Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
 				To(ContainSubstring("saved"))
 			time.Sleep(1 * time.Second)
-			Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
+			Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
 				To(ContainSubstring("saved"))
 			time.Sleep(1 * time.Second)
-			Expect(testutil.K3sCmd("etcd-snapshot", "save", "-name", "PRUNE_TEST")).
+			Expect(testutil.K3sCmd("etcd-snapshot", "save -name PRUNE_TEST")).
 				To(ContainSubstring("saved"))
 			time.Sleep(1 * time.Second)
 		})
@@ -88,7 +88,7 @@ var _ = Describe("etcd snapshots", func() {
 			Expect(sepLines).To(HaveLen(3))
 		})
 		It("prunes snapshots down to 2", func() {
-			Expect(testutil.K3sCmd("etcd-snapshot", "prune", "--snapshot-retention", "2", "--name", "PRUNE_TEST")).
+			Expect(testutil.K3sCmd("etcd-snapshot", "prune --snapshot-retention 2 --name PRUNE_TEST")).
 				To(ContainSubstring("Removing local snapshot"))
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
@@ -112,7 +112,8 @@ var _ = Describe("etcd snapshots", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(server)).To(Succeed())
+		Expect(testutil.K3sKillServer(server, false)).To(Succeed())
+		Expect(testutil.K3sCleanup(server, true)).To(Succeed())
 	}
 })
 

scripts/test

@@ -38,8 +38,8 @@ echo "Did test-run-sonobuoy $?"
 
 # ---
 
-test-run-sonobuoy etcd
 test-run-sonobuoy mysql
 test-run-sonobuoy postgres
+test-run-sonobuoy etcd skip-parallel
 
 exit 0

scripts/test-helpers

@@ -554,7 +554,7 @@ run-test() {
     local delay=15
     (
       set +x
-      while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-4} ]; do
+      while [ $(count-running-tests) -ge ${MAX_CONCURRENT_TESTS:-3} ]; do
           sleep $delay
       done
     )
@@ -619,21 +619,6 @@ e2e-test() {
 
 # ---
 
-run-e2e-tests() {
-    label=PARALLEL \
-        logName=e2e-STATUS-${ARCH}-parallel.log \
-        e2e-test ${sonobuoyParallelArgs[@]}
-    echo "Exit code $? for parallel start"
-
-    label=SERIAL \
-        logName=e2e-STATUS-${ARCH}-serial.log \
-        e2e-test ${sonobuoySerialArgs[@]}
-    echo "Exit code $? for serial start"
-}
-export -f run-e2e-tests
-
-# ---
-
 test-run-sonobuoy() {
     local suffix
     if [ "$1" ]; then
@@ -643,7 +628,18 @@ test-run-sonobuoy() {
 
     cleanup-test-env
     . ./scripts/test-setup-sonobuoy$suffix
-    run-e2e-tests
+    
+    if [ "$2" != "skip-parallel" ]; then
+        label=PARALLEL \
+            logName=e2e-STATUS-${ARCH}-parallel.log \
+            e2e-test ${sonobuoyParallelArgs[@]}
+        echo "Exit code $? for parallel start"
+    fi
+
+    label=SERIAL \
+        logName=e2e-STATUS-${ARCH}-serial.log \
+        e2e-test ${sonobuoySerialArgs[@]}
+    echo "Exit code $? for serial start"
 }
 export -f test-run-sonobuoy
 

scripts/test-setup-sonobuoy-etcd

@@ -2,8 +2,8 @@
 
 . ./scripts/test-setup-sonobuoy
 
-export NUM_SERVERS=2
-export NUM_AGENTS=0
+export NUM_SERVERS=1
+export NUM_AGENTS=1
 export SERVER_1_ARGS="--cluster-init"
 
 server-post-hook() {

tests/integration/custometcdargs/custometcdargs_int_test.go

@@ -51,7 +51,7 @@ var _ = Describe("custom etcd args", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
+		Expect(testutil.K3sKillServer(customEtcdArgsServer, true)).To(Succeed())
 	}
 })
 

tests/integration/dualstack/dualstack_int_test.go

@@ -52,7 +52,8 @@ var _ = Describe("dual stack", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() && os.Getenv("CI") != "true" {
-		Expect(testutil.K3sKillServer(dualStackServer)).To(Succeed())
+		Expect(testutil.K3sKillServer(dualStackServer, false)).To(Succeed())
+		Expect(testutil.K3sCleanup(dualStackServer, true)).To(Succeed())
 	}
 })
 

tests/integration/localstorage/localstorage_int_test.go

@@ -32,27 +32,27 @@ var _ = Describe("local storage", func() {
 	When("a new local storage is created", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl get pods -A")
 			}, "90s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("creates a new pvc", func() {
-			result, err := testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pvc.yaml")
+			result, err := testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pvc.yaml")
 			Expect(result).To(ContainSubstring("persistentvolumeclaim/local-path-pvc created"))
 			Expect(err).NotTo(HaveOccurred())
 		})
 		It("creates a new pod", func() {
-			Expect(testutil.K3sCmd("kubectl", "create", "-f", "./testdata/localstorage_pod.yaml")).
+			Expect(testutil.K3sCmd("kubectl create -f ./testdata/localstorage_pod.yaml")).
 				To(ContainSubstring("pod/volume-test created"))
 		})
 		It("shows storage up in kubectl", func() {
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pvc")
+				return testutil.K3sCmd("kubectl get --namespace=default pvc")
 			}, "45s", "1s").Should(MatchRegexp(`local-path-pvc.+Bound`))
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
+				return testutil.K3sCmd("kubectl get --namespace=default pv")
 			}, "10s", "1s").Should(MatchRegexp(`pvc.+1Gi.+Bound`))
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "--namespace=default", "pod")
+				return testutil.K3sCmd("kubectl get --namespace=default pod")
 			}, "10s", "1s").Should(MatchRegexp(`volume-test.+Running`))
 		})
 		It("has proper folder permissions", func() {
@@ -61,7 +61,7 @@ var _ = Describe("local storage", func() {
 			Expect(err).ToNot(HaveOccurred())
 			Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0701"))
 
-			pvResult, err := testutil.K3sCmd("kubectl", "get", "--namespace=default", "pv")
+			pvResult, err := testutil.K3sCmd("kubectl get --namespace=default pv")
 			Expect(err).ToNot(HaveOccurred())
 			reg, err := regexp.Compile(`pvc[^\s]+`)
 			Expect(err).ToNot(HaveOccurred())
@@ -71,9 +71,9 @@ var _ = Describe("local storage", func() {
 			Expect(fmt.Sprintf("%04o", fileStat.Mode().Perm())).To(Equal("0777"))
 		})
 		It("deletes properly", func() {
-			Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "--force", "pod", "volume-test")).
+			Expect(testutil.K3sCmd("kubectl delete --namespace=default --force pod volume-test")).
 				To(ContainSubstring("pod \"volume-test\" force deleted"))
-			Expect(testutil.K3sCmd("kubectl", "delete", "--namespace=default", "pvc", "local-path-pvc")).
+			Expect(testutil.K3sCmd("kubectl delete --namespace=default pvc local-path-pvc")).
 				To(ContainSubstring("persistentvolumeclaim \"local-path-pvc\" deleted"))
 		})
 	})
@@ -81,7 +81,8 @@ var _ = Describe("local storage", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(localStorageServer)).To(Succeed())
+		Expect(testutil.K3sKillServer(localStorageServer, false)).To(Succeed())
+		Expect(testutil.K3sCleanup(localStorageServer, true)).To(Succeed())
 	}
 })
 

tests/integration/secretsencryption/secretsencryption_int_test.go

@@ -35,11 +35,11 @@ var _ = Describe("secrets encryption rotation", func() {
 	When("A server starts with secrets encryption", func() {
 		It("starts up with no problems", func() {
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl get pods -A")
 			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("it creates a encryption key", func() {
-			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("Encryption Status: Enabled"))
 			Expect(result).To(ContainSubstring("Current Rotation Stage: start"))
@@ -47,10 +47,10 @@ var _ = Describe("secrets encryption rotation", func() {
 	})
 	When("A server rotates encryption keys", func() {
 		It("it prepares to rotate", func() {
-			Expect(testutil.K3sCmd("secrets-encrypt", "prepare", "-d", secretsEncryptionDataDir)).
+			Expect(testutil.K3sCmd("secrets-encrypt prepare -d", secretsEncryptionDataDir)).
 				To(ContainSubstring("prepare completed successfully"))
 
-			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("Current Rotation Stage: prepare"))
 			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
@@ -62,19 +62,19 @@ var _ = Describe("secrets encryption rotation", func() {
 		})
 		It("restarts the server", func() {
 			var err error
-			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl get pods -A")
 			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 		})
 		It("rotates the keys", func() {
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("secrets-encrypt", "rotate", "-d", secretsEncryptionDataDir)
+				return testutil.K3sCmd("secrets-encrypt rotate -d", secretsEncryptionDataDir)
 			}, "10s", "2s").Should(ContainSubstring("rotate completed successfully"))
 
-			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("Current Rotation Stage: rotate"))
 			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
@@ -86,21 +86,21 @@ var _ = Describe("secrets encryption rotation", func() {
 		})
 		It("restarts the server", func() {
 			var err error
-			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl get pods -A")
 			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
-			Expect(testutil.K3sCmd("secrets-encrypt", "reencrypt", "-d", secretsEncryptionDataDir)).
+			Expect(testutil.K3sCmd("secrets-encrypt reencrypt -d", secretsEncryptionDataDir)).
 				To(ContainSubstring("reencryption started"))
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+				return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			}, "30s", "2s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
-			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			reg, err := regexp.Compile(`AES-CBC.+aescbckey.*`)
 			Expect(err).ToNot(HaveOccurred())
@@ -111,29 +111,29 @@ var _ = Describe("secrets encryption rotation", func() {
 	})
 	When("A server disables encryption", func() {
 		It("it triggers the disable", func() {
-			Expect(testutil.K3sCmd("secrets-encrypt", "disable", "-d", secretsEncryptionDataDir)).
+			Expect(testutil.K3sCmd("secrets-encrypt disable -d", secretsEncryptionDataDir)).
 				To(ContainSubstring("secrets-encryption disabled"))
 
-			result, err := testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
 		})
 		It("restarts the server", func() {
 			var err error
-			Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+			Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
 			secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
 			Expect(err).ToNot(HaveOccurred())
 			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+				return testutil.K3sCmd("kubectl get pods -A")
 			}, "180s", "1s").Should(MatchRegexp("kube-system.+coredns.+1\\/1.+Running"))
 			time.Sleep(10 * time.Second)
 		})
 		It("reencrypts the keys", func() {
-			result, err := testutil.K3sCmd("secrets-encrypt", "reencrypt", "-f", "--skip", "-d", secretsEncryptionDataDir)
+			result, err := testutil.K3sCmd("secrets-encrypt reencrypt -f --skip -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("reencryption started"))
 
-			result, err = testutil.K3sCmd("secrets-encrypt", "status", "-d", secretsEncryptionDataDir)
+			result, err = testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(result).To(ContainSubstring("Encryption Status: Disabled"))
 		})
@@ -142,7 +142,7 @@ var _ = Describe("secrets encryption rotation", func() {
 
 var _ = AfterSuite(func() {
 	if !testutil.IsExistingServer() {
-		Expect(testutil.K3sKillServer(secretsEncryptionServer)).To(Succeed())
+		Expect(testutil.K3sKillServer(secretsEncryptionServer, true)).To(Succeed())
 		Expect(os.RemoveAll(secretsEncryptionDataDir)).To(Succeed())
 	}
 })

tests/util/cmd.go → tests/util/integration.go

@@ -8,9 +8,11 @@ import (
 	"os/exec"
 	"os/user"
 	"strings"
+	"syscall"
 
 	"github.com/rancher/k3s/pkg/flock"
 	"github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
 )
 
 // Compile-time variable
@@ -63,16 +65,20 @@ func IsExistingServer() bool {
 }
 
 // K3sCmd launches the provided K3s command via exec. Command blocks until finished.
-// Command output from both Stderr and Stdout is provided via string.
+// Command output from both Stderr and Stdout is provided via string. Input can
+// be a single string with space separated args, or multiple string args
 //   cmdEx1, err := K3sCmd("etcd-snapshot", "ls")
+//   cmdEx2, err := K3sCmd("kubectl get pods -A")
 //   cmdEx2, err := K3sCmd("kubectl", "get", "pods", "-A")
-func K3sCmd(cmdName string, cmdArgs ...string) (string, error) {
+func K3sCmd(inputArgs ...string) (string, error) {
 	if !IsRoot() {
 		return "", errors.New("integration tests must be run as sudo/root")
 	}
 	k3sBin := findK3sExecutable()
-	// Only run sudo if not root
-	k3sCmd := append([]string{cmdName}, cmdArgs...)
+	var k3sCmd []string
+	for _, arg := range inputArgs {
+		k3sCmd = append(k3sCmd, strings.Fields(arg)...)
+	}
 	cmd := exec.Command(k3sBin, k3sCmd...)
 	byteOut, err := cmd.CombinedOutput()
 	return string(byteOut), err
@@ -145,17 +151,53 @@ func K3sStartServer(inputArgs ...string) (*K3sServer, error) {
 
 	k3sCmd := append([]string{"server"}, cmdArgs...)
 	cmd := exec.Command(k3sBin, k3sCmd...)
+	// Give the server a new group id so we can kill it and its children later
+	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
 	cmdOut, _ := cmd.StderrPipe()
 	cmd.Stderr = os.Stderr
 	err = cmd.Start()
 	return &K3sServer{cmd, bufio.NewScanner(cmdOut), k3sLock}, err
 }
 
-// K3sKillServer terminates the running K3s server and unlocks the file for
-// other tests
-func K3sKillServer(server *K3sServer) error {
-	if err := server.cmd.Process.Kill(); err != nil {
+// K3sKillServer terminates the running K3s server and its children
+// and unlocks the file for other tests
+func K3sKillServer(server *K3sServer, releaseLock bool) error {
+	pgid, err := syscall.Getpgid(server.cmd.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil {
+		return err
+	}
+	if releaseLock {
+		return flock.Release(server.lock)
+	}
+	return nil
+}
+
+// K3sCleanup attempts to cleanup networking and files leftover from an integration test
+func K3sCleanup(server *K3sServer, releaseLock bool) error {
+	if cni0Link, err := netlink.LinkByName("cni0"); err == nil {
+		links, _ := netlink.LinkList()
+		for _, link := range links {
+			if link.Attrs().MasterIndex == cni0Link.Attrs().Index {
+				netlink.LinkDel(link)
+			}
+		}
+		netlink.LinkDel(cni0Link)
+	}
+
+	if flannel1, err := netlink.LinkByName("flannel.1"); err == nil {
+		netlink.LinkDel(flannel1)
+	}
+	if flannelV6, err := netlink.LinkByName("flannel-v6.1"); err == nil {
+		netlink.LinkDel(flannelV6)
+	}
+	if err := os.RemoveAll("/var/lib/rancher/k3s"); err != nil {
 		return err
 	}
+	if releaseLock {
 		return flock.Release(server.lock)
+	}
+	return nil
 }