Migrate E2E autoimport to Docker test

.github/workflows/e2e.yaml

@@ -45,7 +45,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        etest: [autoimport, startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
+        etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
       max-parallel: 3
     steps:
       - name: "Checkout"
@@ -160,8 +160,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade]
+        dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade]
         arch: [amd64, arm64]
+        exclude:
+          - dtest: autoimport
+            arch: arm64
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     env:
       CHANNEL: ${{ needs.build-go-tests.outputs.channel }}
@@ -200,9 +203,12 @@ jobs:
         chmod +x ./dist/artifacts/${{ matrix.dtest }}.test
         mv ./dist/artifacts/${{ matrix.dtest }}.test ./tests/docker/${{ matrix.dtest }}/
         cd ./tests/docker/${{ matrix.dtest }}
+        
+        # These tests use rancher/systemd-node and have different flags.
+        CI_TESTS="autoimport hardened snapshotrestore token"
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
-        elif [ ${{ matrix.dtest }} = "snapshotrestore" ] || [ ${{ matrix.dtest }} = "hardened" ]; then
+        elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then
           ./${{ matrix.dtest }}.test -ci 
         else
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE

tests/e2e/autoimport/autoimport_test.go → tests/docker/autoimport/autoimport_test.go

@@ -2,71 +2,42 @@ package autoimport
 
 import (
 	"flag"
-	"os"
 	"testing"
 
 	"github.com/k3s-io/k3s/tests"
-	"github.com/k3s-io/k3s/tests/e2e"
+	"github.com/k3s-io/k3s/tests/docker"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
 
-// Valid nodeOS:
-// bento/ubuntu-24.04, opensuse/Leap-15.6.x86_64
-// eurolinux-vagrant/rocky-8, eurolinux-vagrant/rocky-9,
-var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system")
-var serverCount = flag.Int("serverCount", 1, "number of server nodes")
-var agentCount = flag.Int("agentCount", 0, "number of agent nodes")
 var ci = flag.Bool("ci", false, "running on CI")
-var local = flag.Bool("local", false, "deploy a locally built K3s binary")
 
-// Environment Variables Info:
-// E2E_RELEASE_VERSION=v1.23.1+k3s2 (default: latest commit from master)
-// E2E_REGISTRY: true/false (default: false)
-
-func Test_E2EAutoImport(t *testing.T) {
+func Test_DockerAutoImport(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
 	suiteConfig, reporterConfig := GinkgoConfiguration()
 	RunSpecs(t, "Auto Import Test Suite", suiteConfig, reporterConfig)
 }
 
-var tc *e2e.TestConfig
-
-var _ = ReportAfterEach(e2e.GenReport)
+var tc *docker.TestConfig
 
 var _ = Describe("Verify Create", Ordered, func() {
-	Context("Cluster :", func() {
-		It("Starts up with no issues", func() {
+	Context("Setup Cluster", func() {
+		It("should provision server", func() {
 			var err error
-			if *local {
-				tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
-			} else {
-				tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
-			}
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
-			By("CLUSTER CONFIG")
-			By("OS: " + *nodeOS)
-			By(tc.Status())
-		})
-
-		It("Checks node and pod status", func() {
-			By("Fetching Nodes status")
-			Eventually(func(g Gomega) {
-				nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
-				g.Expect(err).NotTo(HaveOccurred())
-				for _, node := range nodes {
-					g.Expect(node.Status).Should(Equal("Ready"))
-				}
-			}, "620s", "5s").Should(Succeed())
-			e2e.DumpPods(tc.KubeconfigFile)
-
+			tc, err = docker.NewTestConfig("rancher/systemd-node")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(tc.ProvisionServers(1)).To(Succeed())
 			Eventually(func() error {
-				return tests.AllPodsUp(tc.KubeconfigFile)
-			}, "620s", "5s").Should(Succeed())
-			e2e.DumpPods(tc.KubeconfigFile)
+				return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, tc.KubeconfigFile)
+			}, "60s", "5s").Should(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(tc.KubeconfigFile, tc.GetNodeNames())
+			}, "40s", "5s").Should(Succeed())
 		})
+	})
 
+	Context("Add images that should be imported by containerd automatically", func() {
 		It("Create a folder in agent/images", func() {
 			cmd := `mkdir /var/lib/rancher/k3s/agent/images`
 			_, err := tc.Servers[0].RunCmdOnNode(cmd)
@@ -74,7 +45,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 		})
 
 		It("Create file for auto import and search in the image store", func() {
-			cmd := `echo docker.io/library/redis:latest | sudo tee /var/lib/rancher/k3s/agent/images/testautoimport.txt`
+			cmd := `echo docker.io/library/redis:latest | tee /var/lib/rancher/k3s/agent/images/testautoimport.txt`
 			_, err := tc.Servers[0].RunCmdOnNode(cmd)
 			Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
 
@@ -98,7 +69,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 		})
 
 		It("Create, remove and create again a file", func() {
-			cmd := `echo docker.io/library/busybox:latest | sudo tee /var/lib/rancher/k3s/agent/images/bb.txt`
+			cmd := `echo docker.io/library/busybox:latest | tee /var/lib/rancher/k3s/agent/images/bb.txt`
 			_, err := tc.Servers[0].RunCmdOnNode(cmd)
 			Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
 
@@ -118,7 +89,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 				g.Expect(tc.Servers[0].RunCmdOnNode(cmd)).Should(ContainSubstring("io.cri-containerd.pinned=pinned"))
 			}, "620s", "5s").Should(Succeed())
 
-			cmd = `echo docker.io/library/busybox:latest | sudo tee /var/lib/rancher/k3s/agent/images/bb.txt`
+			cmd = `echo docker.io/library/busybox:latest | tee /var/lib/rancher/k3s/agent/images/bb.txt`
 			_, err = tc.Servers[0].RunCmdOnNode(cmd)
 			Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
 
@@ -150,16 +121,10 @@ var _ = Describe("Verify Create", Ordered, func() {
 		})
 
 		It("Restarts normally", func() {
-			errRestart := e2e.RestartCluster(append(tc.Servers, tc.Agents...))
-			Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly")
-
-			Eventually(func(g Gomega) {
-				nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
-				g.Expect(err).NotTo(HaveOccurred())
-				for _, node := range nodes {
-					g.Expect(node.Status).Should(Equal("Ready"))
-				}
-			}, "620s", "5s").Should(Succeed())
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(tc.KubeconfigFile, tc.GetNodeNames())
+			}, "60s", "5s").Should(Succeed())
 		})
 
 		It("Verify bb.txt image and see if are pinned", func() {
@@ -183,16 +148,10 @@ var _ = Describe("Verify Create", Ordered, func() {
 		})
 
 		It("Restarts normally", func() {
-			errRestart := e2e.RestartCluster(append(tc.Servers, tc.Agents...))
-			Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly")
-
-			Eventually(func(g Gomega) {
-				nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
-				g.Expect(err).NotTo(HaveOccurred())
-				for _, node := range nodes {
-					g.Expect(node.Status).Should(Equal("Ready"))
-				}
-			}, "620s", "5s").Should(Succeed())
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(tc.KubeconfigFile, tc.GetNodeNames())
+			}, "60s", "5s").Should(Succeed())
 		})
 
 		It("Verify if bb.txt image is unpinned", func() {
@@ -212,12 +171,7 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
-
-	if !failed {
-		Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
-	}
-	if !failed || *ci {
-		Expect(e2e.DestroyCluster()).To(Succeed())
-		Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
+	if *ci || (tc != nil && !failed) {
+		Expect(tc.Cleanup()).To(Succeed())
 	}
 })

tests/docker/test-helpers.go

@@ -23,7 +23,7 @@ type TestConfig struct {
 	K3sImage       string
 	DBType         string
 	SkipStart      bool
-	Servers        []Server
+	Servers        []DockerNode
 	Agents         []DockerNode
 	ServerYaml     string
 	AgentYaml      string
@@ -32,12 +32,8 @@ type TestConfig struct {
 type DockerNode struct {
 	Name string
 	IP   string
-}
-
-type Server struct {
-	DockerNode
-	Port int
-	URL  string
+	Port int    // Not filled by agent nodes
+	URL  string // Not filled by agent nodes
 }
 
 // NewTestConfig initializes the test environment and returns the configuration
@@ -130,10 +126,8 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			}
 			joinServer = fmt.Sprintf("--server %s", config.Servers[0].URL)
 		}
-		newServer := Server{
-			DockerNode: DockerNode{
-				Name: name,
-			},
+		newServer := DockerNode{
+			Name: name,
 			Port: port,
 		}
 
@@ -536,7 +530,7 @@ func getEnvOrDefault(key, defaultValue string) string {
 }
 
 // VerifyValidVersion checks for invalid version strings
-func VerifyValidVersion(node Server, binary string) error {
+func VerifyValidVersion(node DockerNode, binary string) error {
 	output, err := node.RunCmdOnNode(binary + " version")
 	if err != nil {
 		return err
@@ -613,3 +607,14 @@ func (config TestConfig) DeployWorkload(workload string) (string, error) {
 	}
 	return "", nil
 }
+
+// RestartCluster restarts the k3s service on each node given
+func RestartCluster(nodes []DockerNode) error {
+	for _, node := range nodes {
+		cmd := "systemctl restart k3s* --all"
+		if _, err := node.RunCmdOnNode(cmd); err != nil {
+			return err
+		}
+	}
+	return nil
+}

tests/e2e/autoimport/Vagrantfile

-ENV['VAGRANT_NO_PARALLEL'] = 'no'
-NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
-  ["server-0"])
-NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
-  ['bento/ubuntu-24.04'])
-GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
-RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
-GOCOVER = (ENV['E2E_GOCOVER'] || "")
-NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
-NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 2048).to_i
-# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
-NETWORK_PREFIX = "10.10.10"
-install_type = ""
-
-def provision(vm, role, role_num, node_num)
-  vm.box = NODE_BOXES[node_num]
-  vm.hostname = role
-  # An expanded netmask is required to allow VM<-->VM communication, virtualbox defaults to /32
-  node_ip = "#{NETWORK_PREFIX}.#{100+node_num}"
-  vm.network "private_network", ip: node_ip, netmask: "255.255.255.0"
-
-  scripts_location = Dir.exist?("./scripts") ? "./scripts" : "../scripts"
-  vagrant_defaults = File.exist?("./vagrantdefaults.rb") ? "./vagrantdefaults.rb" : "../vagrantdefaults.rb"
-  load vagrant_defaults
-
-  defaultOSConfigure(vm)
-  addCoverageDir(vm, role, GOCOVER)
-  install_type = getInstallType(vm, RELEASE_VERSION, GITHUB_BRANCH)
-
-  if role.include?("server") && role_num == 0
-    dockerInstall(vm)
-
-    vm.provision 'k3s-primary-server', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "server "
-      k3s.config = <<~YAML
-        token: vagrant
-        node-external-ip: #{NETWORK_PREFIX}.100
-        flannel-iface: eth1
-        cluster-init: true
-      YAML
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
-      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
-    end
-  
-  elsif role.include?("server") && role_num != 0
-    vm.provision 'k3s-secondary-server', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "server"
-      k3s.config = <<~YAML
-        server: "https://#{NETWORK_PREFIX}.100:6443"
-        token: vagrant
-        node-external-ip: #{node_ip}
-        flannel-iface: eth1
-      YAML
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 K3S_TOKEN=vagrant #{install_type}]
-      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
-    end
-  end
-  
-  if role.include?("agent")
-    vm.provision 'k3s-agent', type: 'k3s', run: 'once' do |k3s|
-      k3s.args = "agent"
-      k3s.config = <<~YAML
-        server: "https://#{NETWORK_PREFIX}.100:6443"
-        token: vagrant
-        node-external-ip: #{node_ip}
-        flannel-iface: eth1
-      YAML
-      k3s.env = %W[K3S_KUBECONFIG_MODE=0644 #{install_type}]
-      k3s.config_mode = '0644' # side-step https://github.com/k3s-io/k3s/issues/4321
-    end
-  end
-  if vm.box.to_s.include?("microos")
-    vm.provision 'k3s-reload', type: 'reload', run: 'once'
-  end
-end
-
-Vagrant.configure("2") do |config|
-  config.vagrant.plugins = ["vagrant-k3s", "vagrant-reload"]
-  # Default provider is libvirt, virtualbox is only provided as a backup
-  config.vm.provider "libvirt" do |v|
-    v.cpus = NODE_CPUS
-    v.memory = NODE_MEMORY
-    # We replicate the default prefix, but add a timestamp to enable parallel runs and cleanup of old VMs
-    v.default_prefix = File.basename(Dir.getwd) + "_" + Time.now.to_i.to_s + "_"
-  end
-  config.vm.provider "virtualbox" do |v|
-    v.cpus = NODE_CPUS
-    v.memory = NODE_MEMORY
-  end
-  
-  if NODE_ROLES.kind_of?(String)
-    NODE_ROLES = NODE_ROLES.split(" ", -1)
-  end
-  if NODE_BOXES.kind_of?(String)
-    NODE_BOXES = NODE_BOXES.split(" ", -1)
-  end
-
-  NODE_ROLES.each_with_index do |role, i|
-    role_num = role.split("-", -1).pop.to_i
-    config.vm.define role do |node|
-      provision(node.vm, role, role_num, i)
-    end
-  end
-end