Skip to content

K
k3s
Unverified Commit 97c69546 authored by ShylajaDevadiga's avatar ShylajaDevadiga Committed by GitHub
Browse files
Options

add arm tests and upgrade tests (#5526)

Signed-off-by: 's avatarShylaja Devadiga <shylaja@rancher.com>
parent a693071c
Show whitespace changes
Inline Side-by-side
Showing with 567 additions and 44 deletions
tests/terraform/Dockerfile.build
View file @ 97c69546
......@@ -19,7 +19,8 @@ RUN apk update && \
WORKDIR $GOPATH/src/github.com/k3s-io/k3s
COPY . .
RUN go install github.com/gruntwork-io/terratest/modules/terraform
RUN go install -u github.com/onsi/gomega
RUN go install -u github.com/onsi/ginkgo/v2
RUN go install -u golang.org/x/crypto/...
RUN go get github.com/gruntwork-io/terratest/modules/terraform
RUN go get -u github.com/onsi/gomega
RUN go get -u github.com/onsi/ginkgo/v2
RUN go get -u golang.org/x/crypto/...
RUN go get -u github.com/Thatooine/go-test-html-report
tests/terraform/arm_resource_files/ingress.yaml
View file @ 97c69546
apiVersion: extensions/v1beta1
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress
name: test-ingress
spec:
rules:
- host: foo1.bar.com
http:
paths:
- path: /name.html
backend:
serviceName: nginx-ingress-svc
servicePort: 80
- backend:
service:
name: nginx-ingress-svc
port:
number: 80
path: /
pathType: ImplementationSpecific
---
apiVersion: v1
kind: Service
......
tests/terraform/createcluster.go
View file @ 97c69546
......@@ -3,6 +3,7 @@ package e2e
import (
"flag"
"fmt"
"path/filepath"
"testing"
......@@ -17,6 +18,7 @@ var clusterType = flag.String("cluster_type", "etcd", "a string")
var resourceName = flag.String("resource_name", "etcd", "a string")
var sshuser = flag.String("sshuser", "ubuntu", "a string")
var sshkey = flag.String("sshkey", "", "a string")
var failed = false
var (
kubeConfigFile string
......@@ -24,8 +26,7 @@ var (
workerIPs string
)
func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool) (string, string, string, error) {
func BuildCluster(nodeOs, clusterType, externalDb, resourceName string, t *testing.T, destroy bool, arch string) (string, string, string, error) {
tDir := "./modules/k3scluster"
vDir := "/config/" + nodeOs + clusterType + ".tfvars"
......
tests/terraform/createcluster_test.go
View file @ 97c69546
......@@ -14,19 +14,16 @@ import (
func Test_E2EClusterCreateValidation(t *testing.T) {
RegisterFailHandler(Fail)
flag.Parse()
RunSpecs(t, "Create Cluster Test Suite")
}
var _ = Describe("Test:", func() {
Context("Build Cluster:", func() {
It("Starts up with no issues", func() {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy)
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
Expect(err).NotTo(HaveOccurred())
defer GinkgoRecover()
if *destroy {
fmt.Printf("\nCluster is being Deleted\n")
return
}
fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
fmt.Printf("\nIPs:\n")
fmt.Println("Server Node IPS:", masterIPs)
......@@ -62,7 +59,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies ClusterIP Service", func() {
_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, false)
_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
Eventually(func(g Gomega) {
......@@ -81,13 +78,13 @@ var _ = Describe("Test:", func() {
res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-clusterip"))
fmt.Println(res)
}, "420s", "10s").Should(Succeed())
}
})
It("Verifies NodePort Service", func() {
_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, false)
_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
......@@ -109,12 +106,13 @@ var _ = Describe("Test:", func() {
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-nodeport"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies LoadBalancer Service", func() {
_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, false)
_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
......@@ -136,12 +134,13 @@ var _ = Describe("Test:", func() {
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Ingress", func() {
_, err := DeployWorkload("ingress.yaml", kubeConfigFile, false)
_, err := DeployWorkload("ingress.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
Eventually(func(g Gomega) {
......@@ -162,12 +161,13 @@ var _ = Describe("Test:", func() {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
fmt.Println(res)
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Daemonset", func() {
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, false)
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
nodes, _ := ParseNodes(kubeConfigFile, false)
......@@ -184,7 +184,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies Local Path Provisioner storage ", func() {
_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
......@@ -200,7 +200,6 @@ var _ = Describe("Test:", func() {
cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
......@@ -215,7 +214,7 @@ var _ = Describe("Test:", func() {
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, false)
_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
......@@ -244,7 +243,7 @@ var _ = Describe("Test:", func() {
})
It("Verifies dns access", func() {
_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, false)
_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
Eventually(func(g Gomega) {
......@@ -260,15 +259,11 @@ var _ = Describe("Test:", func() {
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
fmt.Println(res)
}, "420s", "2s").Should(Succeed())
})
It("Validate Rebooting nodes", func() {
if *destroy {
return
}
defer GinkgoRecover()
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
for _, ip := range nodeExternalIP {
fmt.Println("\nRebooting node: ", ip)
......@@ -303,16 +298,23 @@ var _ = Describe("Test:", func() {
})
})
var failed = false
var _ = AfterEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
})
var _ = BeforeEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
if *destroy {
fmt.Printf("\nCluster is being Deleted\n")
Skip("Cluster is being Deleted")
}
})
var _ = AfterSuite(func() {
if failed {
fmt.Println("FAILED!")
} else {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true)
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, true, *arch)
if err != nil {
fmt.Println("Error Destroying Cluster", err)
}
......
tests/terraform/jenkinsfile
View file @ 97c69546
......@@ -34,16 +34,16 @@ pipeline {
vpc_id="${env.VPC_ID}"
subnets="${env.SUBNETS}"
qa_space="${env.QA_SPACE}"
ec2_instance_class="${env.EC2_INSTANCE_CLASS}"
ec2_instance_class="${env.AWS_INSTANCE_TYPE}"
access_key="/config/$AWS_SSH_KEY_NAME"
no_of_worker_nodes="${env.NO_OF_WORKER_NODES}"
key_name="jenkins-rke-validation"
server_flags="${env.SERVER_FLAGS}"
worker_flags="${env.WORKER_FLAGS}"
server_flags="${env.K3S_SERVER_FLAGS}"
worker_flags="${env.K3S_WORKER_FLAGS}"
k3s_version="${env.K3S_VERSION}"
availability_zone="${env.AVAILABILITY_ZONE}"
sg_id="${env.SG_ID}"
install_mode="${env.INSTALL_MODE}"
install_mode="${env.K3S_INSTALL_MODE}"
resource_name="${env.RESOURCE_NAME}"
no_of_server_nodes="${env.NO_OF_SERVER_NODES}"
username="${env.RHEL_USERNAME}"
......@@ -51,8 +51,8 @@ pipeline {
db_username="${env.DB_USERNAME}"
db_password="${env.DB_PASSWORD}"
node_os="${env.NODE_OS}"
environment="${env.ENVIRONMENT}"
engine_mode="${env.ENGINE_MODE}"
environment="${env.DB_ENVIRONMENT}"
engine_mode="${env.DB_ENGINE_MODE}"
external_db="${env.EXTERNAL_DB}"
external_db_version="${env.EXTERNAL_DB_VERSION}"
instance_class="${env.DB_INSTANCE_CLASS}"
......
tests/terraform/testutils.go
View file @ 97c69546
......@@ -17,6 +17,7 @@ type Node struct {
Name string
Status string
Roles string
Version string
InternalIP string
ExternalIP string
}
......@@ -92,7 +93,6 @@ func runsshCommand(cmd string, conn *ssh.Client) (string, error) {
// RunCmdOnNode executes a command from within the given node
func RunCmdOnNode(cmd string, ServerIP string, SSHUser string, SSHKey string) (string, error) {
Server := ServerIP + ":22"
fmt.Println(Server, SSHUser, SSHKey)
conn := ConfigureSSH(Server, SSHUser, SSHKey)
res, err := runsshCommand(cmd, conn)
res = strings.TrimSpace(res)
......@@ -117,10 +117,10 @@ func CountOfStringInSlice(str string, pods []Pod) int {
return count
}
func DeployWorkload(workload, kubeconfig string, arch bool) (string, error) {
func DeployWorkload(workload, kubeconfig string, arch string) (string, error) {
resourceDir := "./amd64_resource_files"
if arch {
resourceDir = "./arm64_resource_files"
if arch == "arm64" {
resourceDir = "./arm_resource_files"
}
files, err := ioutil.ReadDir(resourceDir)
if err != nil {
......@@ -186,6 +186,7 @@ func ParseNodes(kubeConfig string, print bool) ([]Node, error) {
Name: fields[0],
Status: fields[1],
Roles: fields[2],
Version: fields[4],
InternalIP: fields[5],
ExternalIP: fields[6],
}
......
tests/terraform/upgradecluster_test.go 0 → 100644
View file @ 97c69546
package e2e
import (
"flag"
"fmt"
"strings"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var upgradeVersion = flag.String("upgrade_version", "", "a string")
func Test_E2EClusterUpgradeValidation(t *testing.T) {
RegisterFailHandler(Fail)
flag.Parse()
RunSpecs(t, "Upgrade Cluster Test Suite")
}
var _ = Describe("Test:", func() {
Context("Build Cluster:", func() {
It("Starts up with no issues", func() {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
Expect(err).NotTo(HaveOccurred())
defer GinkgoRecover()
fmt.Println("\nCLUSTER CONFIG:\nOS", *nodeOs, "BACKEND", *clusterType, *externalDb)
fmt.Printf("\nIPs:\n")
fmt.Println("Server Node IPS:", masterIPs)
fmt.Println("Agent Node IPS:", workerIPs)
fmt.Println(kubeConfigFile)
Expect(kubeConfigFile).Should(ContainSubstring(*resourceName))
})
It("Checks Node and Pod Status", func() {
fmt.Printf("\nFetching node status\n")
Eventually(func(g Gomega) {
nodes, err := ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "420s", "5s").Should(Succeed())
_, _ = ParseNodes(kubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n")
Eventually(func(g Gomega) {
pods, err := ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "420s", "5s").Should(Succeed())
_, _ = ParsePods(kubeConfigFile, true)
})
It("Verifies ClusterIP Service", func() {
_, err := DeployWorkload("clusterip.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should((ContainSubstring("test-clusterip")))
}, "420s", "5s").Should(Succeed())
clusterip, _ := FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
cmd := "curl -L --insecure http://" + clusterip + "/name.html"
fmt.Println(cmd)
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-clusterip"))
}, "420s", "10s").Should(Succeed())
}
})
It("Verifies NodePort Service", func() {
_, err := DeployWorkload("nodeport.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-nodeport"))
}, "240s", "5s").Should(Succeed())
cmd = "curl -L --insecure http://" + ip + ":" + nodeport + "/name.html"
fmt.Println(cmd)
Eventually(func(g Gomega) {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-nodeport"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies LoadBalancer Service", func() {
_, err := DeployWorkload("loadbalancer.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Loadbalancer manifest not deployed")
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
port, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
}, "240s", "5s").Should(Succeed())
Eventually(func(g Gomega) {
cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
fmt.Println(cmd)
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Ingress", func() {
_, err := DeployWorkload("ingress.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-ingress --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
}, "240s", "5s").Should(Succeed())
ingressIps, err := FetchIngressIP(kubeConfigFile)
Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned")
for _, ip := range ingressIps {
cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html"
fmt.Println(cmd)
Eventually(func(g Gomega) {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Daemonset", func() {
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
nodes, _ := ParseNodes(kubeConfigFile, false)
pods, _ := ParsePods(kubeConfigFile, false)
Eventually(func(g Gomega) {
count := CountOfStringInSlice("test-daemonset", pods)
fmt.Println("POD COUNT")
fmt.Println(count)
fmt.Println("NODE COUNT")
fmt.Println(len(nodes))
g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count")
}, "420s", "10s").Should(Succeed())
})
It("Verifies Local Path Provisioner storage ", func() {
_, err := DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("local-path-pvc"))
g.Expect(res).Should(ContainSubstring("Bound"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
_, err = RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
fmt.Println("Data stored in pvc: local-path-test")
cmd = "kubectl delete pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
_, err = DeployWorkload("local-path-provisioner.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("pod/local-path-provisioner"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd = "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
res, err = RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println("Data after re-creation", res)
g.Expect(res).Should(ContainSubstring("local-path-test"))
}, "180s", "2s").Should(Succeed())
})
It("Verifies dns access", func() {
_, err := DeployWorkload("dnsutils.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("dnsutils"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default"
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
}, "420s", "2s").Should(Succeed())
})
It("\nVerify Cluster is upgraded", func() {
if *destroy {
//fmt.Printf("\nCluster is Deleted\n")
return
}
MIPs := strings.Split(masterIPs, ",")
fmt.Println(MIPs)
for _, ip := range MIPs {
fmt.Println(ip)
cmd := "sudo sed -i \"s/|/| INSTALL_K3S_VERSION=" + *upgradeVersion + "/g\" /tmp/master_cmd"
fmt.Println(cmd)
Eventually(func(g Gomega) {
result, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
fmt.Println(result)
g.Expect(err).NotTo(HaveOccurred())
}, "420s", "2s").Should(Succeed())
cmd = "sudo chmod u+x /tmp/master_cmd && sudo /tmp/master_cmd"
fmt.Println(cmd)
Eventually(func(g Gomega) {
_, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
}, "420s", "2s").Should(Succeed())
}
WIPs := strings.Split(workerIPs, ",")
for i := 0; i < len(WIPs) && len(WIPs[0]) > 1; i++ {
ip := WIPs[i]
strings.TrimSpace(WIPs[i])
cmd := "sudo sed -i \"s/|/| INSTALL_K3S_VERSION=" + *upgradeVersion + "/g\" /tmp/agent_cmd"
fmt.Println(cmd)
fmt.Println(*sshuser)
fmt.Println(*sshkey)
Eventually(func(g Gomega) {
_, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
}, "420s", "2s").Should(Succeed())
cmd = "sudo chmod u+x /tmp/agent_cmd && sudo /tmp/agent_cmd"
fmt.Println(cmd)
Eventually(func(g Gomega) {
_, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
}, "420s", "2s").Should(Succeed())
}
})
It("Checks Node and Pod Status after upgrade", func() {
fmt.Printf("\nFetching node status after upgrade\n")
Eventually(func(g Gomega) {
nodes, err := ParseNodes(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
g.Expect(node.Version).Should(ContainSubstring(*upgradeVersion))
}
}, "420s", "5s").Should(Succeed())
_, _ = ParseNodes(kubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n")
Eventually(func(g Gomega) {
pods, err := ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "420s", "5s").Should(Succeed())
_, _ = ParsePods(kubeConfigFile, true)
})
It("Verifies ClusterIP Service", func() {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should((ContainSubstring("test-clusterip")))
}, "420s", "5s").Should(Succeed())
clusterip, _ := FetchClusterIP(kubeConfigFile, "nginx-clusterip-svc")
cmd := "curl -L --insecure http://" + clusterip + "/name.html"
fmt.Println(cmd)
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
res, err := RunCmdOnNode(cmd, ip, *sshuser, *sshkey)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-clusterip"))
}, "420s", "10s").Should(Succeed())
}
})
It("Verifies NodePort Service", func() {
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-nodeport"))
}, "240s", "5s").Should(Succeed())
cmd = "curl -L --insecure http://" + ip + ":" + nodeport + "/name.html"
fmt.Println(cmd)
Eventually(func(g Gomega) {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-nodeport"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies LoadBalancer Service", func() {
nodeExternalIP := FetchNodeExternalIP(kubeConfigFile)
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
port, err := RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
for _, ip := range nodeExternalIP {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
}, "240s", "5s").Should(Succeed())
Eventually(func(g Gomega) {
cmd = "curl -L --insecure http://" + ip + ":" + port + "/name.html"
fmt.Println(cmd)
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Ingress", func() {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-ingress --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
}, "240s", "5s").Should(Succeed())
ingressIps, err := FetchIngressIP(kubeConfigFile)
Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned")
for _, ip := range ingressIps {
cmd := "curl --header host:foo1.bar.com" + " http://" + ip + "/name.html"
fmt.Println(cmd)
Eventually(func(g Gomega) {
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-ingress"))
}, "240s", "5s").Should(Succeed())
}
})
It("Verifies Daemonset", func() {
_, err := DeployWorkload("daemonset.yaml", kubeConfigFile, *arch)
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
nodes, _ := ParseNodes(kubeConfigFile, false)
pods, _ := ParsePods(kubeConfigFile, false)
Eventually(func(g Gomega) {
count := CountOfStringInSlice("test-daemonset", pods)
fmt.Println("POD COUNT")
fmt.Println(count)
fmt.Println("NODE COUNT")
fmt.Println(len(nodes))
g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pod count does not match node count")
}, "420s", "10s").Should(Succeed())
})
It("Validating Local Path Provisioner storage after upgrade", func() {
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + kubeConfigFile
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("pod/local-path-provisioner"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl get pod volume-test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl exec volume-test cat /data/test --kubeconfig=" + kubeConfigFile
res, err := RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred())
fmt.Println("Data after upgrade", res)
g.Expect(res).Should(ContainSubstring("local-path-test"))
}, "180s", "2s").Should(Succeed())
})
It("Verifies dns access", func() {
Eventually(func(g Gomega) {
cmd := "kubectl get pods dnsutils --kubeconfig=" + kubeConfigFile
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("dnsutils"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl --kubeconfig=" + kubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default"
res, _ := RunCommand(cmd)
fmt.Println(res)
g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local"))
}, "420s", "2s").Should(Succeed())
})
})
})
var _ = AfterEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
})
var _ = BeforeEach(func() {
failed = failed || CurrentGinkgoTestDescription().Failed
if *destroy {
fmt.Printf("\nCluster is being Deleted\n")
Skip("Cluster is being Deleted")
}
})
var _ = AfterSuite(func() {
if failed {
fmt.Println("FAILED!")
} else {
kubeConfigFile, masterIPs, workerIPs, err = BuildCluster(*nodeOs, *clusterType, *externalDb, *resourceName, &testing.T{}, *destroy, *arch)
if err != nil {
fmt.Println("Error Destroying Cluster", err)
}
}
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment