Add scripts to run e2e test using ansible (#5134)

tests/e2e/Dockerfile.build

-FROM golang:alpine
-
-ARG TERRAFORM_VERSION=0.12.10
-ENV TERRAFORM_VERSION=$TERRAFORM_VERSION
-
-
-RUN apk update && \
-    apk upgrade --update-cache --available && \
-    apk add curl git jq bash openssh unzip gcc g++ make ca-certificates && \
-    curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
-    chmod +x ./kubectl && \
-    mv ./kubectl /usr/local/bin
-RUN mkdir tmp && \
-    curl "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" -o tmp/terraform.zip && \
-    unzip tmp/terraform.zip -d /usr/local/bin && \
-    chmod +x /usr/local/bin/terraform && \
-    rm -rf tmp
-
-WORKDIR $GOPATH/src/github.com/k3s-io/k3s-io/k3s/tests/e2e
-
-COPY . .
-

tests/e2e/amd64_resource_files/clusterip.yaml

@@ -24,7 +24,7 @@ metadata:
   labels:
     k8s-app: nginx-app-clusterip
   name: nginx-clusterip-svc
-  namespace: default 
+  namespace: default
 spec:
   type: ClusterIP
   ports:

tests/e2e/arm_resource_files/clusterip.yaml → tests/e2e/cis_amd64_resource_files/clusterip.yaml

@@ -14,9 +14,9 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
@@ -24,10 +24,10 @@ metadata:
   labels:
     k8s-app: nginx-app-clusterip
   name: nginx-clusterip-svc
-  namespace: default 
+  namespace: default
 spec:
   type: ClusterIP
   ports:
-    - port: 80
+    - port: 8080
   selector:
     k8s-app: nginx-app-clusterip

tests/e2e/cis_amd64_resource_files/dnsutils.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  name: dnsutils
+  namespace: default
+spec:
+  containers:
+  - name: dnsutils
+    image: gcr.io/kubernetes-e2e-test-images/dnsutils:1.3
+    command:
+      - sleep
+      - "3600"
+    imagePullPolicy: IfNotPresent
+  restartPolicy: Always

tests/e2e/arm_resource_files/ingress.yaml → tests/e2e/cis_amd64_resource_files/ingress.yaml

-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
- name: ingress
-spec:
- rules:
- - host: foo1.bar.com
-   http:
-     paths:
-     - path: /name.html
-       backend:
-         serviceName: nginx-ingress-svc
-         servicePort: 80
----
 apiVersion: v1
 kind: Service
 metadata:
@@ -20,29 +6,23 @@ metadata:
    k8s-app: nginx-app-ingress
 spec:
  ports:
- - port: 80
-   targetPort: 80
+ - port: 8080
+   targetPort: 8080
    protocol: TCP
    name: http
  selector:
    k8s-app: nginx-app-ingress
 ---
-apiVersion: v1
-kind: ReplicationController
+apiVersion: extensions/v1beta1
+kind: Ingress
 metadata:
- name: test-ingress
+  name: test-ingress
 spec:
- replicas: 2
- selector:
-   k8s-app: nginx-app-ingress
- template:
-   metadata:
-     labels:
-       k8s-app: nginx-app-ingress
-   spec:
-     terminationGracePeriodSeconds: 60
-     containers:
-     - name: testcontainer
-       image: shylajarancher19/shylajaarm64:v1.0
-       ports:
-       - containerPort: 80
+  rules:
+  - host: foo1.bar.com
+    http:
+      paths:
+      - backend:
+          serviceName: nginx-ingress-svc
+          servicePort: 8080
+        pathType: ImplementationSpecific

tests/e2e/arm_resource_files/loadbalancer.yaml → tests/e2e/cis_amd64_resource_files/loadbalancer.yaml

----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -15,22 +14,23 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
 metadata:
- name: nginx-loadbalancer-svc
- labels:
-   k8s-app: nginx-app-loadbalancer
+  labels:
+    k8s-app: nginx-app-loadbalancer
+  name: nginx-loadbalancer-svc
+  namespace: default
 spec:
- type: LoadBalancer
- ports:
- - port: 81
-   targetPort: 80
-   protocol: TCP
-   name: http
- selector:
-   k8s-app: nginx-app-loadbalancer
+  type: LoadBalancer
+  ports:
+  - port: 81
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    k8s-app: nginx-app-loadbalancer

tests/e2e/arm_resource_files/nodeport.yaml → tests/e2e/cis_amd64_resource_files/nodeport.yaml

@@ -14,9 +14,9 @@ spec:
     spec:
       containers:
       - name: nginx
-        image: shylajarancher19/shylajaarm64:v1.0
+        image: ranchertest/mytestcontainer:unprivileged
         ports:
-        - containerPort: 80
+        - containerPort: 8080
 ---
 apiVersion: v1
 kind: Service
@@ -28,7 +28,7 @@ metadata:
 spec:
   type: NodePort
   ports:
-    - port: 80
+    - port: 8080
       nodePort: 30096
       name: http
   selector:

tests/e2e/cis_amd64_resource_files/secrets.yaml

+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret1
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "hello"
+    val: "world"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret2
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "good"
+    val: "day"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret3
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "top-secret"
+    val: "information"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret4
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "lock"
+    val: "key"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: e2e-secret5
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "last"
+    val: "call"
\ No newline at end of file

tests/e2e/dualstack/dualstack_test.go

@@ -16,6 +16,7 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 0, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s1 or nil for latest commit from master
@@ -129,7 +130,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 	})
 
 	It("Verifies ClusterIP Service", func() {
-		_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred())
 		Eventually(func() (string, error) {
 			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
@@ -157,7 +158,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 		}
 	})
 	It("Verifies Ingress", func() {
-		_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 		cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\""
 		hostName, err := e2e.RunCommand(cmd)
@@ -177,7 +178,7 @@ var _ = Describe("Verify DualStack Configuration", func() {
 	})
 
 	It("Verifies NodePort Service", func() {
-		_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, false)
+		_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, *hardened)
 		Expect(err).NotTo(HaveOccurred())
 		cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
 		nodeport, err := e2e.RunCommand(cmd)

tests/e2e/e2e_test_playbook.yaml

+---
+- name: Run Cluster validation tests
+  gather_facts: false
+  hosts: node
+  vars:
+    user: "{{ user }}"
+    key: "{{ key }}"
+    ip: "{{ ip }}"
+    dir: scripts
+  tasks:
+    - name : Print var
+      debug:
+        msg: |
+          user: {{ user }}
+          key: {{ key }}
+          ip: {{ ip }}
+          db: {{ db }}
+          nodeOS: {{ nodeOS }}
+          serverCount: {{ serverCount }}
+          agentCount: {{ agentCount }}
+
+    - name: build container image
+      command: chdir=k3s/tests/e2e /usr/local/bin/docker build -t k3s_nightly_build -f {{ dir }}/Dockerfile .
+
+    - name: Delete docker container
+      command: "/usr/local/bin/docker rm createcluster -f"
+      ignore_errors: True
+
+    - name: create docker container
+      command: "/usr/local/bin/docker run -d -it -v ~/config:/config --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0  --name createcluster --privileged k3s_nightly_build"
+
+    - name: execute command in docker
+      shell: |
+        /usr/local/bin/docker exec -it createcluster /bin/bash ./run_tests.sh {{ key }} {{ user }} {{ ip }} {{ db }} {{ nodeOS }} {{ serverCount }} {{ agentCount }}

tests/e2e/scripts/Dockerfile

+FROM ubuntu:20.04
+ARG EXTERNAL_ENCODED_VPN
+ARG VPN_ENCODED_LOGIN
+
+RUN apt-get update && \
+    apt-get install -y curl bridge-utils iputils-ping openvpn openssh-client && \
+    mkdir -p /dev/net && \
+    mknod /dev/net/tun c 10 200 && \
+    chmod 600 /dev/net/tun
+
+RUN if [[ -z "$EXTERNAL_ENCODED_VPN" ]] ; then echo "no vpn provided" ;  \
+    else echo -n $EXTERNAL_ENCODED_VPN | base64 -di > external.ovpn && \
+    if [[ -z "$VPN_ENCODED_LOGIN" ]]; then echo "no passcode provided" ; \
+    else echo -n $VPN_ENCODED_LOGIN | base64 -di > authfile && \
+    sed -i 's/auth-user-pass/auth-user-pass authfile/g' external.ovpn; fi ; fi
+
+WORKDIR .
+COPY scripts/run_tests.sh .
+COPY scripts/init.sh .

tests/e2e/scripts/Jenkinsfile

+pipeline {
+    agent any
+    environment {
+        AWS_SSH_PEM_KEY = credentials('AWS_SSH_PEM_KEY')
+    }
+        
+    stages {
+        stage('Git Checkout') {
+            steps {
+                    git branch: 'master', url: 'https://github.com/k3s-io/k3s.git'
+                    script {
+                        dir("${WORKSPACE}/tests/e2e") {
+                        if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) {
+                                    def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64())
+                                    writeFile file: AWS_SSH_KEY_NAME, text: decoded
+                        }
+                    }
+                }
+            }
+        }
+        stage('Build Cluster'){
+            steps {
+                script {
+                    try {
+                        sh """
+                            cd ${WORKSPACE}/tests/e2e/
+                            /usr/bin/docker build -f ./scripts/Dockerfile --build-arg EXTERNAL_ENCODED_VPN="$EXTERNAL_ENCODED_VPN" \
+                            --build-arg VPN_ENCODED_LOGIN="$VPN_ENCODED_LOGIN" -t k3s_create_cluster .
+                            /usr/bin/docker run -d -it --cap-add=NET_ADMIN --device /dev/net/tun --sysctl net.ipv6.conf.all.disable_ipv6=0 \
+                            --name createcluster_${BUILD_NUMBER} --privileged k3s_create_cluster
+                            /usr/bin/docker cp ${AWS_SSH_KEY_NAME} createcluster_${BUILD_NUMBER}:/
+                            /usr/bin/docker exec createcluster_${BUILD_NUMBER} ./run_tests.sh ${AWS_SSH_KEY_NAME} ${USER} ${NODEIP} \
+                            ${NODEOS} ${SERVERCOUNT} ${AGENTCOUNT} ${DB} ${HARDENED}
+                        """
+                    } catch(error) {
+                            echo "First build failed, let's retry if accepted"
+                            retry(2) {
+                                build 'k3s_nightly_automation'
+                        }
+                    }
+                }
+            }
+        }
+	stage('Test Report') {
+            steps {
+                sh """
+                    /usr/bin/docker rm -f createcluster_${BUILD_NUMBER}
+                    /usr/bin/docker rmi -f k3s_create_cluster
+                """
+            }
+        }
+    }
+}

tests/e2e/scripts/hosts

+[node]
+localhost ansible_python_interpreter=/usr/local/opt/python@3.9/bin/python3.9
\ No newline at end of file

tests/e2e/scripts/init.sh

+#!/bin/sh
+# Script to install pre-requisites on vsphere node to run e2e test
+
+export PATH=$PATH:/usr/local/go/bin/:/usr/local/bin
+echo 'Installing Kubectl'
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+sudo mv kubectl /usr/local/bin/ && \
+chmod a+x /usr/local/bin/kubectl
+
+echo 'Installing jq'
+sudo apt-get -y install jq
+
+echo 'Installing Go'
+curl -L https://dl.google.com/go/go1.16.10.linux-amd64.tar.gz | tar xz
+sudo mv go /usr/local
+/usr/local/go/bin/go version
+go version
+
+echo 'Installing Virtualbox'
+sudo apt-get -y update && sudo apt-get -y upgrade
+sudo apt-get -y install virtualbox
+vboxmanage --version
+
+# Virtualbox >= 6.1.28 require `/etc/vbox/network.conf
+ls /etc/vbox 2>/dev/null || sudo mkdir -p /etc/vbox
+echo "* 10.0.0.0/8 192.168.0.0/16">>/etc/vbox/networks.conf
+
+echo 'Installing vagrant'
+sudo apt-get -y install -f unzip
+curl -O https://releases.hashicorp.com/vagrant/2.2.19/vagrant_2.2.19_linux_amd64.zip
+unzip vagrant_2.2.19_linux_amd64.zip
+sudo cp vagrant /usr/local/bin/
+vagrant --version
+sudo apt-get -y install libarchive-tools
+vagrant plugin install vagrant-k3s
+vagrant plugin install vagrant-reload
+
+echo 'Cloning repo'
+ls k3s 2>/dev/null || git clone https://github.com/k3s-io/k3s.git

tests/e2e/scripts/run_tests.sh

+#!/bin/bash
+servercount=${5:-3}
+agentcount=${6:-2}
+db=${7:-"etcd"}
+k3s_version=${k3s_version}
+k3s_channel=${k3s_channel:-"commit"}
+hardened=${8:-""}
+
+E2E_EXTERNAL_DB=$db && export E2E_EXTERNAL_DB
+
+eval openvpn --daemon --config external.ovpn &>/dev/null &
+sleep 10
+
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 'cd k3s && git pull --rebase origin master'
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/onsi/ginkgo/v2'
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/onsi/gomega'
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 '/usr/local/go/bin/go get github.com/k3s-io/k3s/tests/e2e'
+
+echo 'RUNNING CLUSTER VALIDATION TEST'
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/validatecluster && vagrant destroy -f'
+ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && E2E_HARDENED=$hardened /usr/local/go/bin/go test -v validatecluster/validatecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount)) -timeout=1h"
+
+echo 'RUNNING SECRETS ENCRYPTION TEST'
+#ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/secretsencryption && vagrant destroy -f'
+#ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v secretsencryption/secretsencryption_test.go -nodeOS="$4" -serverCount=$((servercount)) -timeout=1h"
+
+E2E_RELEASE_VERSION=$k3s_version && export E2E_RELEASE_VERSION
+E2E_RELEASE_CHANNEL=$k3s_channel && export E2E_RELEASE_CHANNEL
+
+echo 'RUNNING CLUSTER UPGRADE TEST'
+#ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 'cd k3s/tests/e2e/upgradecluster && vagrant destroy -f'
+#ssh -i "$1"  -o 'StrictHostKeyChecking no' $2@$3 "cd k3s/tests/e2e && /usr/local/go/bin/go test -v upgradecluster/upgradecluster_test.go -nodeOS="$4" -serverCount=$((servercount)) -agentCount=$((agentcount))  -timeout=1h"

tests/e2e/secretsencryption/Vagrantfile

 ENV['VAGRANT_NO_PARALLEL'] = 'no'
-NODE_ROLES = (ENV['NODE_ROLES'] ||
+NODE_ROLES = (ENV['E2E_NODE_ROLES'] ||
   ["server-0", "server-1", "server-2"])
-NODE_BOXES = (ENV['NODE_BOXES'] ||
+NODE_BOXES = (ENV['E2E_NODE_BOXES'] ||
   ['generic/ubuntu2004', 'generic/ubuntu2004', 'generic/ubuntu2004'])
-GITHUB_BRANCH = (ENV['GITHUB_BRANCH'] || "master")
-RELEASE_VERSION = (ENV['RELEASE_VERSION'] || "")
-NODE_CPUS = (ENV['NODE_CPUS'] || 2).to_i
-NODE_MEMORY = (ENV['NODE_MEMORY'] || 1024).to_i
+GITHUB_BRANCH = (ENV['E2E_GITHUB_BRANCH'] || "master")
+RELEASE_VERSION = (ENV['E2E_RELEASE_VERSION'] || "")
+NODE_CPUS = (ENV['E2E_NODE_CPUS'] || 2).to_i
+NODE_MEMORY = (ENV['E2E_NODE_MEMORY'] || 1024).to_i
 # Virtualbox >= 6.1.28 require `/etc/vbox/network.conf` for expanded private networks 
 NETWORK_PREFIX = "10.10.10"
 install_type = ""

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/k3s-io/k3s/tests/e2e"
 	. "github.com/onsi/ginkgo/v2"
@@ -16,14 +15,15 @@ import (
 // Valid nodeOS: generic/ubuntu2004, opensuse/Leap-15.3.x86_64, dweomer/microos.amd64
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
 
-func Test_E2EClusterValidation(t *testing.T) {
+func Test_E2ESecretsEncryption(t *testing.T) {
 	RegisterFailHandler(Fail)
 	flag.Parse()
-	RunSpecs(t, "Create Cluster Test Suite")
+	RunSpecs(t, "Secrets Encryption Test Suite")
 }
 
 var (
@@ -71,7 +71,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		})
 
 		It("Deploys several secrets", func() {
-			_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
 		})
 
@@ -127,16 +127,19 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 					}
 				}
 			}, "420s", "5s").Should(Succeed())
+			_, _ = e2e.ParseNodes(kubeConfigFile, true)
 		})
 
 		It("Verifies encryption prepare stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
@@ -145,31 +148,34 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 			res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred(), res)
 			for i, nodeName := range serverNodeNames {
-				cmd := "sudo k3s secrets-encrypt status"
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred(), res)
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
-				if i == 0 {
-					Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
-				} else {
-					Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
-				}
+				Eventually(func(g Gomega) {
+					cmd := "sudo k3s secrets-encrypt status"
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: prepare"))
+					}
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
 		It("Restarts K3s servers", func() {
 			Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-			time.Sleep(20 * time.Second)
 		})
 
 		It("Verifies encryption rotate stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: rotate"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 
@@ -181,7 +187,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 			cmd = "sudo k3s secrets-encrypt status"
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-			}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 
 			for _, nodeName := range serverNodeNames[1:] {
 				res, err := e2e.RunCmdOnNode(cmd, nodeName)
@@ -193,17 +199,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 
 		It("Restarts K3s Servers", func() {
 			Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-			time.Sleep(20 * time.Second)
 		})
 
 		It("Verifies Encryption Reencrypt Stage", func() {
 			cmd := "sudo k3s secrets-encrypt status"
 			for _, nodeName := range serverNodeNames {
-				res, err := e2e.RunCmdOnNode(cmd, nodeName)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-				Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
-				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
 			}
 		})
 	})
@@ -220,28 +227,31 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		cmd = "sudo k3s secrets-encrypt status"
 		Eventually(func() (string, error) {
 			return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-		}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+		}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 
 		for i, nodeName := range serverNodeNames {
-			res, err := e2e.RunCmdOnNode(cmd, nodeName)
-			Expect(err).NotTo(HaveOccurred(), res)
-			if i == 0 {
-				Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
-			} else {
-				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
-			}
+			Eventually(func(g Gomega) {
+				res, err := e2e.RunCmdOnNode(cmd, nodeName)
+				g.Expect(err).NotTo(HaveOccurred(), res)
+				if i == 0 {
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+				} else {
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+				}
+			}, "420s", "2s").Should(Succeed())
 		}
 	})
 
 	It("Restarts K3s servers", func() {
 		Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-		time.Sleep(20 * time.Second)
 	})
 
 	It("Verifies encryption disabled on all nodes", func() {
 		cmd := "sudo k3s secrets-encrypt status"
 		for _, nodeName := range serverNodeNames {
-			Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
+			Eventually(func(g Gomega) {
+				g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled"))
+			}, "420s", "2s").Should(Succeed())
 		}
 	})
 
@@ -257,19 +267,21 @@ var _ = Describe("Verify Secrets Encryption Rotation", func() {
 		cmd = "sudo k3s secrets-encrypt status"
 		Eventually(func() (string, error) {
 			return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
-		}, "30s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+		}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
 	})
 
 	It("Restarts K3s servers", func() {
 		Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed())
-		time.Sleep(20 * time.Second)
 	})
 
 	It("Verifies encryption enabled on all nodes", func() {
 		cmd := "sudo k3s secrets-encrypt status"
 		for _, nodeName := range serverNodeNames {
-			Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
+			Eventually(func(g Gomega) {
+				g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
+			}, "420s", "2s").Should(Succeed())
 		}
+
 	})
 
 })

tests/e2e/splitserver/splitserver_test.go

@@ -18,6 +18,7 @@ var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var etcdCount = flag.Int("etcdCount", 1, "number of server nodes only deploying etcd")
 var controlPlaneCount = flag.Int("controlPlaneCount", 1, "number of server nodes acting as control plane")
 var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.1+k3s2 or nil for latest commit from master
@@ -111,7 +112,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
 			cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
@@ -129,7 +130,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -151,7 +152,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -174,7 +175,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range cpNodeNames {
@@ -187,7 +188,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -202,7 +203,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile

tests/e2e/testutils.go

@@ -69,10 +69,10 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 	return serverNodeNames, agentNodeNames, nil
 }
 
-func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
+func DeployWorkload(workload, kubeconfig string, hardened bool) (string, error) {
 	resourceDir := "../amd64_resource_files"
-	if arch {
-		resourceDir = "../arm64_resource_files"
+	if hardened {
+		resourceDir = "../cis_amd64_resource_files"
 	}
 	files, err := ioutil.ReadDir(resourceDir)
 	if err != nil {

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -16,6 +16,7 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_RELEASE_VERSION=v1.23.3+k3s1
@@ -75,7 +76,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
@@ -94,7 +95,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -117,7 +118,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 			for _, nodeName := range serverNodeNames {
 				ip, _ := e2e.FetchNodeExternalIP(nodeName)
@@ -138,7 +139,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -151,7 +152,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			nodes, _ := e2e.ParseNodes(kubeConfigFile, false) //nodes :=
@@ -168,7 +169,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			Eventually(func() (string, error) {
@@ -183,7 +184,7 @@ var _ = Describe("Verify Upgrade", func() {
 		})
 
 		It("Verifies Local Path Provisioner storage ", func() {
-			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 			Eventually(func(g Gomega) {
 				cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
@@ -214,7 +215,7 @@ var _ = Describe("Verify Upgrade", func() {
 			Expect(err).NotTo(HaveOccurred())
 			fmt.Println(res)
 
-			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func() (string, error) {

tests/e2e/validatecluster/validatecluster_test.go

@@ -16,6 +16,7 @@ import (
 var nodeOS = flag.String("nodeOS", "generic/ubuntu2004", "VM operating system")
 var serverCount = flag.Int("serverCount", 3, "number of server nodes")
 var agentCount = flag.Int("agentCount", 2, "number of agent nodes")
+var hardened = flag.Bool("hardened", false, "true or false")
 
 // Environment Variables Info:
 // E2E_EXTERNAL_DB: mysql or postgres, nil for embedded etcd
@@ -74,7 +75,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies ClusterIP Service", func() {
-			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("clusterip.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -98,7 +99,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies NodePort Service", func() {
-			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("nodeport.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -126,7 +127,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies LoadBalancer Service", func() {
-			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -155,7 +156,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Ingress", func() {
-			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("ingress.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
 
 			for _, nodeName := range serverNodeNames {
@@ -173,7 +174,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Daemonset", func() {
-			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("daemonset.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
 
 			nodes, _ := e2e.ParseNodes(kubeConfigFile, false)
@@ -190,7 +191,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies dns access", func() {
-			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("dnsutils.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -210,7 +211,7 @@ var _ = Describe("Verify Create", func() {
 		})
 
 		It("Verifies Local Path Provisioner storage ", func() {
-			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err := e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {
@@ -241,7 +242,7 @@ var _ = Describe("Verify Create", func() {
 			Expect(err).NotTo(HaveOccurred())
 			fmt.Println(res)
 
-			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
+			_, err = e2e.DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
 
 			Eventually(func(g Gomega) {