add arm tests and upgrade tests (#5526)

tests/terraform/Dockerfile.build

@@ -19,7 +19,8 @@ RUN apk update && \
 WORKDIR $GOPATH/src/github.com/k3s-io/k3s
 
 COPY . .
-RUN go install github.com/gruntwork-io/terratest/modules/terraform
-RUN go install -u github.com/onsi/gomega
-RUN go install -u github.com/onsi/ginkgo/v2
-RUN go install -u golang.org/x/crypto/...
+RUN go get github.com/gruntwork-io/terratest/modules/terraform
+RUN go get -u github.com/onsi/gomega
+RUN go get -u github.com/onsi/ginkgo/v2
+RUN go get -u golang.org/x/crypto/...
+RUN go get -u github.com/Thatooine/go-test-html-report

tests/terraform/arm_resource_files/ingress.yaml

-apiVersion: extensions/v1beta1
+apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
- name: ingress
+  name: test-ingress
 spec:
- rules:
- - host: foo1.bar.com
-   http:
-     paths:
-     - path: /name.html
-       backend:
-         serviceName: nginx-ingress-svc
-         servicePort: 80
+  rules:
+    - host: foo1.bar.com
+      http:
+        paths:
+          - backend:
+              service:
+                name: nginx-ingress-svc
+                port:
+                  number: 80
+            path: /
+            pathType: ImplementationSpecific
 ---
 apiVersion: v1
 kind: Service
 metadata:
- name: nginx-ingress-svc
- labels:
-   k8s-app: nginx-app-ingress
+  name: nginx-ingress-svc
+  labels:
+    k8s-app: nginx-app-ingress
 spec:
- ports:
- - port: 80
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-ingress
+  ports:
+    - port: 80
+      targetPort: 80
+      protocol: TCP
+      name: http
+  selector:
+    k8s-app: nginx-app-ingress
 ---
 apiVersion: v1
 kind: ReplicationController
 metadata:
- name: test-ingress
+  name: test-ingress
 spec:
- replicas: 2
- selector:
-   k8s-app: nginx-app-ingress
- template:
-   metadata:
-     labels:
-       k8s-app: nginx-app-ingress
-   spec:
-     terminationGracePeriodSeconds: 60
-     containers:
-     - name: testcontainer
-       image: shylajarancher19/shylajaarm64:v1.0
-       ports:
-       - containerPort: 80
+  replicas: 2
+  selector:
+    k8s-app: nginx-app-ingress
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-app-ingress
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+        - name: testcontainer
+          image: shylajarancher19/shylajaarm64:v1.0
+          ports:
+            - containerPort: 80

tests/terraform/createcluster.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"flag"
 	"fmt"
+
 	"path/filepath"
 	"testing"
 
@@ -17,6 +18,7 @@ var clusterType = flag.String("cluster_type", "etcd", "a string")
 var resourceName = flag.String("resource_name", "etcd", "a string")
 var sshuser = flag.String("sshuser", "ubuntu", "a string")
 var sshkey = flag.String("sshkey", "", "a string")
+var failed = false
 
 var (
 	kubeConfigFile string
@@ -24,8 +26,7 @@ var (
 	workerIPs      string
 )
 
-func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string, error) {
-
+func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool, arch string) (string, string, string, error) {
 	tDir := "./modules/k3scluster"
 	vDir := "/config/" + nodeOs + clusterType + ".tfvars"
 

tests/terraform/createcluster_test.go

@@ -14,19 +14,16 @@ import (
 func Test_E2EClusterCreateValidation(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
+
 	RunSpecs(t, "Create Cluster Test Suite")
 }
 
 var _ = Describe("Test:", func() {
 	Context("Build Cluster:", func() {
 		It("Starts up with no issues", func() {
-			kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy)
+			kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
 			Expect(err).NotTo(HaveOccurred())
 			defer GinkgoRecover()
-			if *destroy {
-				fmt.Printf("\nCluster is being Deleted\n")
-				return
-			}
 			fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
 			fmt.Printf("\nIPs:\n")
 			fmt.Println("Server Node IPS:", masterIPs)
@@ -62,7 +59,7 @@ var _ = Describe("Test:", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -81,13 +78,13 @@ var _ = Describe("Test:", func() {
 					res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(res).Should(ContainSubstring("test-clusterip"))
-
+					fmt.Println(res)
 				}, "420s", "10s").Should(Succeed())
 			}
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 			nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
 			cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
@@ -109,12 +106,13 @@ var _ = Describe("Test:", func() {
 					g.Expect(err).NotTo(HaveOccurred())
 					fmt.Println(res)
 					g.Expect(res).Should(ContainSubstring("test-nodeport"))
+					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 			nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
 			cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
@@ -136,12 +134,13 @@ var _ = Describe("Test:", func() {
 					g.Expect(err).NotTo(HaveOccurred())
 					fmt.Println(res)
 					g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
+					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("ingress.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -162,12 +161,13 @@ var _ = Describe("Test:", func() {
 					res, err := RunCommand(cmd)
 					g.Expect(err).NotTo(HaveOccurred())
 					g.Expect(res).Should(ContainSubstring("test-ingress"))
+					fmt.Println(res)
 				}, "240s", "5s").Should(Succeed())
 			}
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			nodes, _ := ParseNodes(kubeConfigFile, false)
@@ -184,7 +184,7 @@ var _ = Describe("Test:", func() {
 		})
 
 		It("Verifies Local Path Provisioner storage ", func() {
-			_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -200,7 +200,6 @@ var _ = Describe("Test:", func() {
 				cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
 				res, err := RunCommand(cmd)
 				Expect(err).NotTo(HaveOccurred())
-				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("volume-test"))
 				g.Expect(res).Should(ContainSubstring("Running"))
 			}, "420s", "2s").Should(Succeed())
@@ -215,7 +214,7 @@ var _ = Describe("Test:", func() {
 			Expect(err).NotTo(HaveOccurred())
 			fmt.Println(res)
 
-			_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -244,7 +243,7 @@ var _ = Describe("Test:", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, *arch)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -260,15 +259,11 @@ var _ = Describe("Test:", func() {
 				res, _ := RunCommand(cmd)
 				fmt.Println(res)
 				g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
-
+				fmt.Println(res)
 			}, "420s", "2s").Should(Succeed())
 		})
 
 		It("Validate Rebooting nodes", func() {
-			if *destroy {
-				return
-			}
-			defer GinkgoRecover()
 			nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
 			for _, ip := range nodeExternalIP {
 				fmt.Println("\nRebooting node: ", ip)
@@ -303,16 +298,23 @@ var _ = Describe("Test:", func() {
 	})
 })
 
-var failed = false
 var _ = AfterEach(func() {
 	failed = failed || CurrentGinkgoTestDescription().Failed
 })
 
+var _ = BeforeEach(func() {
+	failed = failed || CurrentGinkgoTestDescription().Failed
+	if *destroy {
+		fmt.Printf("\nCluster is being Deleted\n")
+		Skip("Cluster is being Deleted")
+	}
+
+})
 var _ = AfterSuite(func() {
 	if failed {
 		fmt.Println("FAILED!")
 	} else {
-		kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true)
+		kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true, *arch)
 		if err != nil {
 			fmt.Println("Error Destroying Cluster", err)
 		}

tests/terraform/jenkinsfile

@@ -34,16 +34,16 @@ pipeline {
                         vpc_id="${env.VPC_ID}"
                         subnets="${env.SUBNETS}"
                         qa_space="${env.QA_SPACE}"
-                        ec2_instance_class="${env.EC2_INSTANCE_CLASS}"
+                        ec2_instance_class="${env.AWS_INSTANCE_TYPE}"
                         access_key="/config/$AWS_SSH_KEY_NAME"
                         no_of_worker_nodes="${env.NO_OF_WORKER_NODES}"
                         key_name="jenkins-rke-validation"
-                        server_flags="${env.SERVER_FLAGS}"
-                        worker_flags="${env.WORKER_FLAGS}"
+                        server_flags="${env.K3S_SERVER_FLAGS}"
+                        worker_flags="${env.K3S_WORKER_FLAGS}"
                         k3s_version="${env.K3S_VERSION}"
                         availability_zone="${env.AVAILABILITY_ZONE}"
                         sg_id="${env.SG_ID}"
-                        install_mode="${env.INSTALL_MODE}"
+                        install_mode="${env.K3S_INSTALL_MODE}"
                         resource_name="${env.RESOURCE_NAME}"
                         no_of_server_nodes="${env.NO_OF_SERVER_NODES}"
                         username="${env.RHEL_USERNAME}"
@@ -51,8 +51,8 @@ pipeline {
                         db_username="${env.DB_USERNAME}"
                         db_password="${env.DB_PASSWORD}"
                         node_os="${env.NODE_OS}"
-                        environment="${env.ENVIRONMENT}"
-                        engine_mode="${env.ENGINE_MODE}"
+                        environment="${env.DB_ENVIRONMENT}"
+                        engine_mode="${env.DB_ENGINE_MODE}"
                         external_db="${env.EXTERNAL_DB}"
                         external_db_version="${env.EXTERNAL_DB_VERSION}"
                         instance_class="${env.DB_INSTANCE_CLASS}"

tests/terraform/testutils.go

@@ -17,6 +17,7 @@ type Node struct {
 	Name       string
 	Status     string
 	Roles      string
+	Version    string
 	InternalIP string
 	ExternalIP string
 }
@@ -92,7 +93,6 @@ func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
 // RunCmdOnNode executes a command from within the given node
 func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
 	Server := ServerIP + ":22"
-	fmt.Println(Server, SSHUser, SSHKey)
 	conn := ConfigureSSH(Server, SSHUser, SSHKey)
 	res, err := runsshCommand(cmd, conn)
 	res = strings.TrimSpace(res)
@@ -117,10 +117,10 @@ func CountOfStringInSlice(str string, pods []Pod) int {
 	return count
 }
 
-func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
+func DeployWorkload(workload, kubeconfig string, arch string) (string, error) {
 	resourceDir := "./amd64_resource_files"
-	if arch {
-		resourceDir = "./arm64_resource_files"
+	if arch == "arm64" {
+		resourceDir = "./arm_resource_files"
 	}
 	files, err := ioutil.ReadDir(resourceDir)
 	if err != nil {
@@ -186,6 +186,7 @@ func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
 				Name:       fields[0],
 				Status:     fields[1],
 				Roles:      fields[2],
+				Version:    fields[4],
 				InternalIP: fields[5],
 				ExternalIP: fields[6],
 			}