Skip to content

K
k3s
Commit 74d18149 authored by Derek Nola's avatar Derek Nola
Browse files
Options

Align docker and e2e variable spelling

Signed-off-by: 's avatarDerek Nola <derek.nola@suse.com>
parent b01b892e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 193 additions and 193 deletions
tests/e2e/autoimport/autoimport_test.go
View file @ 74d18149
......@@ -53,18 +53,18 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Create a folder in agent/images", func() {
......@@ -154,7 +154,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -187,7 +187,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -218,6 +218,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/btrfs/btrfs_test.go
View file @ 74d18149
......@@ -46,18 +46,18 @@ var _ = Describe("Verify that btrfs based servers work", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Checks that btrfs snapshots exist", func() {
cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs"
......@@ -82,6 +82,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/dualstack/dualstack_test.go
View file @ 74d18149
......@@ -50,25 +50,25 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
It("Checks Node Status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(tc.KubeConfigFile, true)
_, err := e2e.ParseNodes(tc.KubeconfigFile, true)
Expect(err).NotTo(HaveOccurred())
})
It("Checks pod status", func() {
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Verifies that each node has IPv4 and IPv6", func() {
nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("10.10.10"))
......@@ -76,7 +76,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
}
})
It("Verifies that each pod has IPv4 and IPv6", func() {
podIPs, err := e2e.GetPodIPs(tc.KubeConfigFile)
podIPs, err := e2e.GetPodIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, pod := range podIPs {
Expect(pod.IPv4).Should(Or(ContainSubstring("10.10.10"), ContainSubstring("10.42.")), pod.Name)
......@@ -88,18 +88,18 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
_, err := tc.DeployWorkload("dualstack_clusterip.yaml")
Expect(err).NotTo(HaveOccurred())
Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
return e2e.RunCommand(cmd)
}, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod"))
// Checks both IPv4 and IPv6
clusterips, err := e2e.FetchClusterIP(tc.KubeConfigFile, "ds-clusterip-svc", true)
clusterips, err := e2e.FetchClusterIP(tc.KubeconfigFile, "ds-clusterip-svc", true)
Expect(err).NotTo(HaveOccurred())
for _, ip := range strings.Split(clusterips, ",") {
if strings.Contains(ip, "::") {
ip = "[" + ip + "]"
}
pods, err := tests.ParsePods(tc.KubeConfigFile)
pods, err := tests.ParsePods(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") {
......@@ -118,7 +118,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
cmd := "kubectl get ingress ds-ingress -o jsonpath=\"{.spec.rules[*].host}\""
hostName, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
for _, node := range nodeIPs {
cmd := fmt.Sprintf("curl --header host:%s -m 5 -s -f http://%s/name.html", hostName, node.IPv4)
......@@ -138,7 +138,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
cmd := "kubectl get service ds-nodeport-svc --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs {
cmd = "curl -m 5 -s -f http://" + node.IPv4 + ":" + nodeport + "/name.html"
......@@ -189,6 +189,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/embeddedmirror/embeddedmirror_test.go
View file @ 74d18149
......@@ -54,7 +54,7 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -63,8 +63,8 @@ var _ = Describe("Verify Create", Ordered, func() {
By("Fetching pod status")
Eventually(func() error {
e2e.DumpPods(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "10s").Should(Succeed())
})
It("Should create and validate deployment with embedded registry mirror using image tag", func() {
......@@ -138,6 +138,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/externalip/externalip_test.go
View file @ 74d18149
......@@ -75,40 +75,40 @@ var _ = Describe("Verify External-IP config", Ordered, func() {
It("Checks Node Status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(tc.KubeConfigFile, true)
_, err := e2e.ParseNodes(tc.KubeconfigFile, true)
Expect(err).NotTo(HaveOccurred())
})
It("Checks pod status", func() {
By("Fetching pod status")
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "10s").Should(Succeed())
})
})
Context("Deploy workloads to check cluster connectivity of the nodes", func() {
It("Verifies that each node has vagrant IP", func() {
nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("10.10."))
}
})
It("Verifies that each pod has vagrant IP or clusterCIDR IP", func() {
podIPs, err := e2e.GetPodIPs(tc.KubeConfigFile)
podIPs, err := e2e.GetPodIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, pod := range podIPs {
Expect(pod.IPv4).Should(Or(ContainSubstring("10.10."), ContainSubstring("10.42.")), pod.Name)
}
})
It("Verifies that flannel added the correct annotation for the external-ip", func() {
nodeIPs, err := getExternalIPs(tc.KubeConfigFile)
nodeIPs, err := getExternalIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, annotation := range nodeIPs {
Expect(annotation).Should(ContainSubstring("10.100.100."))
......@@ -120,11 +120,11 @@ var _ = Describe("Verify External-IP config", Ordered, func() {
// Wait for the pod_client to have an IP
Eventually(func() string {
ips, _ := getClientIPs(tc.KubeConfigFile)
ips, _ := getClientIPs(tc.KubeconfigFile)
return ips[0].IPv4
}, "40s", "5s").Should(ContainSubstring("10.42"), "failed getClientIPs")
clientIPs, err := getClientIPs(tc.KubeConfigFile)
clientIPs, err := getClientIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, ip := range clientIPs {
cmd := "kubectl exec svc/client-curl -- curl -m 5 -s -f http://" + ip.IPv4 + "/name.html"
......@@ -157,6 +157,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/privateregistry/privateregistry_test.go
View file @ 74d18149
......@@ -56,17 +56,17 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
By("Fetching pod status")
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "10s").Should(Succeed())
})
......@@ -113,7 +113,7 @@ var _ = Describe("Verify Create", Ordered, func() {
var pod corev1.Pod
Eventually(func(g Gomega) {
pods, err := tests.ParsePods(tc.KubeConfigFile)
pods, err := tests.ParsePods(tc.KubeconfigFile)
for _, p := range pods {
if strings.Contains(p.Name, "my-webpage") {
pod = p
......@@ -148,6 +148,6 @@ var _ = AfterSuite(func() {
Expect(err).NotTo(HaveOccurred(), r2)
Expect(err).NotTo(HaveOccurred())
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/rootless/rootless_test.go
View file @ 74d18149
......@@ -94,26 +94,26 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS: " + *nodeOS)
By(tc.Status())
kubeConfigFile, err := GenRootlessKubeConfigFile(tc.Servers[0].String())
kubeConfigFile, err := GenRootlessKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
tc.KubeConfigFile = kubeConfigFile
tc.KubeconfigFile = kubeConfigFile
})
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "360s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(tc.KubeConfigFile, false)
_, _ = e2e.ParseNodes(tc.KubeconfigFile, false)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Returns pod metrics", func() {
......@@ -163,6 +163,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/rootless/rootless_utils.go
View file @ 74d18149
......@@ -22,7 +22,7 @@ func RunCmdOnRootlessNode(cmd string, nodename string) (string, error) {
return out, nil
}
func GenRootlessKubeConfigFile(serverName string) (string, error) {
func GenRootlessKubeconfigFile(serverName string) (string, error) {
kubeConfig, err := RunCmdOnRootlessNode("cat /home/vagrant/.kube/k3s.yaml", serverName)
if err != nil {
return "", err
......
tests/e2e/rotateca/rotateca_test.go
View file @ 74d18149
......@@ -51,18 +51,18 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.ParseNodes(tc.KubeConfigFile, true)
e2e.ParseNodes(tc.KubeconfigFile, true)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Generates New CA Certificates", func() {
......@@ -93,7 +93,7 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
It("Checks node and pod status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -101,9 +101,9 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
}, "420s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "420s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
})
})
......@@ -121,6 +121,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/s3/s3_test.go
View file @ 74d18149
......@@ -53,18 +53,18 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.ParseNodes(tc.KubeConfigFile, true)
e2e.ParseNodes(tc.KubeconfigFile, true)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("ensures s3 mock is working", func() {
......@@ -167,6 +167,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/secretsencryption/secretsencryption_test.go
View file @ 74d18149
......@@ -55,18 +55,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
e2e.ParseNodes(tc.KubeConfigFile, true)
e2e.ParseNodes(tc.KubeconfigFile, true)
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Deploys several secrets", func() {
......@@ -215,6 +215,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/secretsencryption_old/secretsencryption_test.go
View file @ 74d18149
......@@ -52,7 +52,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -60,9 +60,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}, "620s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Deploys several secrets", func() {
......@@ -104,7 +104,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
It("Checks node and pod status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -112,9 +112,9 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}, "420s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Verifies encryption prepare stage", func() {
......@@ -289,6 +289,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/splitserver/splitserver_test.go
View file @ 74d18149
......@@ -141,9 +141,9 @@ var _ = Describe("Verify Create", Ordered, func() {
fmt.Println("Etcd Server Nodes:", etcdNodes)
fmt.Println("Control Plane Server Nodes:", cpNodes)
fmt.Println("Agent Nodes:", agentNodes)
kubeConfigFile, err := e2e.GenKubeConfigFile(cpNodes[0].String())
kubeConfigFile, err := e2e.GenKubeconfigFile(cpNodes[0].String())
tc = &e2e.TestConfig{
KubeConfigFile: kubeConfigFile,
KubeconfigFile: kubeConfigFile,
Hardened: *hardened,
}
Expect(err).NotTo(HaveOccurred())
......@@ -152,7 +152,7 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -160,21 +160,21 @@ var _ = Describe("Verify Create", Ordered, func() {
}, "620s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Verifies ClusterIP Service", func() {
_, err := tc.DeployWorkload("clusterip.yaml")
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed")
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("test-clusterip"), "failed cmd: "+cmd)
clusterip, _ := e2e.FetchClusterIP(tc.KubeConfigFile, "nginx-clusterip-svc", false)
clusterip, _ := e2e.FetchClusterIP(tc.KubeconfigFile, "nginx-clusterip-svc", false)
cmd = "curl -m 5 -s -f http://" + clusterip + "/name.html"
for _, node := range cpNodes {
Eventually(func() (string, error) {
......@@ -189,11 +189,11 @@ var _ = Describe("Verify Create", Ordered, func() {
for _, node := range cpNodes {
nodeExternalIP, _ := node.FetchNodeExternalIP()
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + tc.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + tc.KubeconfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
......@@ -212,11 +212,11 @@ var _ = Describe("Verify Create", Ordered, func() {
for _, node := range cpNodes {
ip, _ := node.FetchNodeExternalIP()
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + tc.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + tc.KubeconfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
port, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd = "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("test-loadbalancer"), "failed cmd: "+cmd)
......@@ -246,7 +246,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
Eventually(func(g Gomega) {
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeConfigFile)
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(cpNodes).To(HaveLen(count), "Daemonset pod count does not match cp node count")
}, "240s", "10s").Should(Succeed())
......@@ -256,12 +256,12 @@ var _ = Describe("Verify Create", Ordered, func() {
_, err := tc.DeployWorkload("dnsutils.yaml")
Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
cmd := "kubectl get pods dnsutils --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods dnsutils --kubeconfig=" + tc.KubeconfigFile
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "420s", "2s").Should(ContainSubstring("dnsutils"), "failed cmd: "+cmd)
cmd = "kubectl --kubeconfig=" + tc.KubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
cmd = "kubectl --kubeconfig=" + tc.KubeconfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
Eventually(func() (string, error) {
return e2e.RunCommand(cmd)
}, "420s", "2s").Should(ContainSubstring("kubernetes.default.svc.cluster.local"), "failed cmd: "+cmd)
......@@ -284,6 +284,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/startup/startup_test.go
View file @ 74d18149
......@@ -111,26 +111,26 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("Checks node and pod status", func() {
By("Fetching node status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}, tc.KubeConfigFile)
return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}, tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Returns pod metrics", func() {
......@@ -180,26 +180,26 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("Checks node and pod status", func() {
By("Fetching node status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.CheckDefaultDeployments(tc.KubeConfigFile)
return tests.CheckDefaultDeployments(tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Returns kubelet configuration", func() {
......@@ -223,14 +223,14 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("Checks node and pod status", func() {
By("Fetching node status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -238,12 +238,12 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.CheckDefaultDeployments(tc.KubeConfigFile)
return tests.CheckDefaultDeployments(tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Kills the cluster", func() {
err := KillK3sCluster(append(tc.Servers, tc.Agents...))
......@@ -259,14 +259,14 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("Checks node and pod status", func() {
By("Fetching node status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -274,12 +274,12 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.CheckDefaultDeployments(tc.KubeConfigFile)
return tests.CheckDefaultDeployments(tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Kills the cluster", func() {
err := KillK3sCluster(append(tc.Servers, tc.Agents...))
......@@ -295,14 +295,14 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("Checks node and pod status", func() {
By("Fetching node status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -310,12 +310,12 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
Eventually(func() error {
return tests.CheckDefaultDeployments(tc.KubeConfigFile)
return tests.CheckDefaultDeployments(tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Returns pod metrics", func() {
......@@ -367,7 +367,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
By("CLUSTER CONFIG")
By("OS:" + *nodeOS)
By(tc.Status())
tc.KubeConfigFile, err = e2e.GenKubeConfigFile(tc.Servers[0].String())
tc.KubeconfigFile, err = e2e.GenKubeconfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred())
})
It("has loaded the test container image", func() {
......@@ -414,6 +414,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/svcpoliciesandfirewall/svcpoliciesandfirewall_test.go
View file @ 74d18149
......@@ -59,21 +59,21 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
It("Checks Node Status", func() {
Eventually(func(g Gomega) {
var err error
nodes, err = e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err = e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "300s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(tc.KubeConfigFile, true)
_, err := e2e.ParseNodes(tc.KubeconfigFile, true)
Expect(err).NotTo(HaveOccurred())
})
It("Checks Pod Status", func() {
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "300s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
})
Context("Deploy external traffic workloads to test external traffic policies", func() {
......@@ -89,7 +89,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
// Check where the server pod is running
var serverNodeName string
Eventually(func() (string, error) {
pods, err := tests.ParsePods(tc.KubeConfigFile)
pods, err := tests.ParsePods(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred(), "failed to parse pods")
for _, pod := range pods {
if strings.Contains(pod.Name, "test-loadbalancer-ext") {
......@@ -111,11 +111,11 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
lbSvc := "nginx-loadbalancer-svc"
lbSvcExt := "nginx-loadbalancer-svc-ext"
Eventually(func() ([]string, error) {
return e2e.FetchExternalIPs(tc.KubeConfigFile, lbSvc)
return e2e.FetchExternalIPs(tc.KubeconfigFile, lbSvc)
}, "25s", "5s").Should(HaveLen(2), "external IP count not equal to 2")
Eventually(func(g Gomega) {
externalIPs, _ := e2e.FetchExternalIPs(tc.KubeConfigFile, lbSvcExt)
externalIPs, _ := e2e.FetchExternalIPs(tc.KubeconfigFile, lbSvcExt)
g.Expect(externalIPs).To(HaveLen(1), "more than 1 exernalIP found")
g.Expect(externalIPs[0]).To(Equal(serverNodeIP), "external IP does not match servernodeIP")
}, "25s", "5s").Should(Succeed())
......@@ -125,9 +125,9 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
// It also verifies that the service with external traffic policy=cluster can be accessed and the source IP is MASQ
It("Verify connectivity in external traffic policy=local", func() {
lbSvc := "nginx-loadbalancer-svc"
lbSvcExternalIPs, _ := e2e.FetchExternalIPs(tc.KubeConfigFile, lbSvc)
lbSvcExternalIPs, _ := e2e.FetchExternalIPs(tc.KubeconfigFile, lbSvc)
lbSvcExt := "nginx-loadbalancer-svc-ext"
lbSvcExtExternalIPs, _ := e2e.FetchExternalIPs(tc.KubeConfigFile, lbSvcExt)
lbSvcExtExternalIPs, _ := e2e.FetchExternalIPs(tc.KubeconfigFile, lbSvcExt)
// Verify connectivity to the external IP of the lbsvc service and the IP should be the flannel interface IP because of MASQ
for _, externalIP := range lbSvcExternalIPs {
......@@ -166,13 +166,13 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
// Check that service exists
Eventually(func() (string, error) {
clusterIP, _ := e2e.FetchClusterIP(tc.KubeConfigFile, "nginx-loadbalancer-svc-int", false)
clusterIP, _ := e2e.FetchClusterIP(tc.KubeconfigFile, "nginx-loadbalancer-svc-int", false)
return clusterIP, nil
}, "25s", "5s").Should(ContainSubstring("10.43"))
// Check that client pods are running
Eventually(func() string {
pods, err := tests.ParsePods(tc.KubeConfigFile)
pods, err := tests.ParsePods(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "client-deployment") {
......@@ -188,7 +188,7 @@ var _ = Describe("Verify Services Traffic policies and firewall config", Ordered
It("Verify connectivity in internal traffic policy=local", func() {
var clientPod1, clientPod1Node, clientPod1IP, clientPod2, clientPod2Node, clientPod2IP, serverNodeName string
Eventually(func(g Gomega) {
pods, err := tests.ParsePods(tc.KubeConfigFile)
pods, err := tests.ParsePods(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred(), "failed to parse pods")
for _, pod := range pods {
if strings.Contains(pod.Name, "test-loadbalancer-int") {
......@@ -306,7 +306,7 @@ selector:
Expect(err).NotTo(HaveOccurred(), out)
Eventually(func() (string, error) {
clusterIP, _ := e2e.FetchClusterIP(tc.KubeConfigFile, "nginx-loadbalancer-svc-ext-firewall", false)
clusterIP, _ := e2e.FetchClusterIP(tc.KubeconfigFile, "nginx-loadbalancer-svc-ext-firewall", false)
return clusterIP, nil
}, "25s", "5s").Should(ContainSubstring("10.43"))
})
......@@ -356,6 +356,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/tailscale/tailscale_test.go
View file @ 74d18149
......@@ -48,18 +48,18 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() {
// Server node needs to be ready before we continue
It("Checks Node Status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "300s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(tc.KubeConfigFile, true)
_, err := e2e.ParseNodes(tc.KubeconfigFile, true)
Expect(err).NotTo(HaveOccurred())
})
It("Change agent's config", func() {
nodeIPs, _ := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, _ := e2e.GetNodeIPs(tc.KubeconfigFile)
cmd := fmt.Sprintf("sudo sed -i 's/TAILSCALEIP/%s/g' /etc/rancher/k3s/config.yaml", nodeIPs[0].IPv4)
for _, agent := range tc.Agents {
_, err := agent.RunCmdOnNode(cmd)
......@@ -74,19 +74,19 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() {
It("Checks Node Status", func() {
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).To(Equal(*agentCount + *serverCount))
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "300s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(tc.KubeConfigFile, true)
_, err := e2e.ParseNodes(tc.KubeconfigFile, true)
Expect(err).NotTo(HaveOccurred())
})
It("Verifies that server and agent have a tailscale IP as nodeIP", func() {
nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
nodeIPs, err := e2e.GetNodeIPs(tc.KubeconfigFile)
Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("100."))
......@@ -119,6 +119,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/testutils.go
View file @ 74d18149
......@@ -36,7 +36,7 @@ func VagrantSlice(v []VagrantNode) []string {
type TestConfig struct {
Hardened bool
KubeConfigFile string
KubeconfigFile string
Servers []VagrantNode
Agents []VagrantNode
}
......@@ -48,7 +48,7 @@ func (tc *TestConfig) Status() string {
if tc.Hardened {
hardened = "Hardened: true\n"
}
return fmt.Sprintf("%sKubeconfig: %s\nServers Nodes: %s\nAgents Nodes: %s\n)", hardened, tc.KubeConfigFile, sN, aN)
return fmt.Sprintf("%sKubeconfig: %s\nServers Nodes: %s\nAgents Nodes: %s\n)", hardened, tc.KubeconfigFile, sN, aN)
}
type Node struct {
......@@ -165,14 +165,14 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) (*TestConfig, err
return nil, err
}
if !strings.Contains(res, "inactive") && strings.Contains(res, "active") {
kubeConfigFile, err = GenKubeConfigFile(serverNodes[0].String())
kubeConfigFile, err = GenKubeconfigFile(serverNodes[0].String())
if err != nil {
return nil, err
}
}
tc := &TestConfig{
KubeConfigFile: kubeConfigFile,
KubeconfigFile: kubeConfigFile,
Servers: serverNodes,
Agents: agentNodes,
}
......@@ -262,14 +262,14 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) (*TestConfig
var err error
res, _ := serverNodes[0].RunCmdOnNode("systemctl is-active k3s")
if !strings.Contains(res, "inactive") && strings.Contains(res, "active") {
kubeConfigFile, err = GenKubeConfigFile(serverNodes[0].String())
kubeConfigFile, err = GenKubeconfigFile(serverNodes[0].String())
if err != nil {
return nil, err
}
}
tc := &TestConfig{
KubeConfigFile: kubeConfigFile,
KubeconfigFile: kubeConfigFile,
Servers: serverNodes,
Agents: agentNodes,
}
......@@ -291,7 +291,7 @@ func (tc TestConfig) DeployWorkload(workload string) (string, error) {
for _, f := range files {
filename := filepath.Join(resourceDir, f.Name())
if strings.TrimSpace(f.Name()) == workload {
cmd := "kubectl apply -f " + filename + " --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl apply -f " + filename + " --kubeconfig=" + tc.KubeconfigFile
return RunCommand(cmd)
}
}
......@@ -367,16 +367,16 @@ func (v VagrantNode) FetchNodeExternalIP() (string, error) {
return nodeip, nil
}
// GenKubeConfigFile extracts the kubeconfig from the given node and modifies it for use outside the VM.
func GenKubeConfigFile(nodeName string) (string, error) {
kubeConfigFile := fmt.Sprintf("kubeconfig-%s", nodeName)
cmd := fmt.Sprintf("vagrant scp %s:/etc/rancher/k3s/k3s.yaml ./%s", nodeName, kubeConfigFile)
// GenKubeconfigFile extracts the kubeconfig from the given node and modifies it for use outside the VM.
func GenKubeconfigFile(nodeName string) (string, error) {
kubeconfigFile := fmt.Sprintf("kubeconfig-%s", nodeName)
cmd := fmt.Sprintf("vagrant scp %s:/etc/rancher/k3s/k3s.yaml ./%s", nodeName, kubeconfigFile)
_, err := RunCommand(cmd)
if err != nil {
return "", err
}
kubeConfig, err := os.ReadFile(kubeConfigFile)
kubeConfig, err := os.ReadFile(kubeconfigFile)
if err != nil {
return "", err
}
......@@ -389,14 +389,14 @@ func GenKubeConfigFile(nodeName string) (string, error) {
return "", err
}
modifiedKubeConfig = strings.Replace(modifiedKubeConfig, "127.0.0.1", nodeIP, 1)
if err := os.WriteFile(kubeConfigFile, []byte(modifiedKubeConfig), 0644); err != nil {
if err := os.WriteFile(kubeconfigFile, []byte(modifiedKubeConfig), 0644); err != nil {
return "", err
}
if err := os.Setenv("E2E_KUBECONFIG", kubeConfigFile); err != nil {
if err := os.Setenv("E2E_KUBECONFIG", kubeconfigFile); err != nil {
return "", err
}
return kubeConfigFile, nil
return kubeconfigFile, nil
}
func GenReport(specReport ginkgo.SpecReport) {
......
tests/e2e/token/token_test.go
View file @ 74d18149
......@@ -56,7 +56,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -64,9 +64,9 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
}, "420s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "360s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
var permToken string
......@@ -87,7 +87,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 1))
for _, node := range nodes {
......@@ -127,7 +127,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes {
......@@ -153,7 +153,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(err).NotTo(HaveOccurred())
}
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes {
......@@ -169,7 +169,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes {
......@@ -193,6 +193,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/validatecluster/validatecluster_test.go
View file @ 74d18149
......@@ -59,19 +59,19 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
}, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(tc.KubeConfigFile, true)
_, _ = e2e.ParseNodes(tc.KubeconfigFile, true)
fmt.Printf("\nFetching Pods status\n")
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
e2e.DumpPods(tc.KubeConfigFile)
e2e.DumpPods(tc.KubeconfigFile)
})
It("Verifies ClusterIP Service", func() {
......@@ -79,13 +79,13 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed: "+res)
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should((ContainSubstring("test-clusterip")), "failed cmd: %q result: %s", cmd, res)
}, "240s", "5s").Should(Succeed())
clusterip, _ := e2e.FetchClusterIP(tc.KubeConfigFile, "nginx-clusterip-svc", false)
clusterip, _ := e2e.FetchClusterIP(tc.KubeconfigFile, "nginx-clusterip-svc", false)
cmd := "curl -m 5 -s -f http://" + clusterip + "/name.html"
for _, node := range tc.Servers {
Eventually(func(g Gomega) {
......@@ -102,12 +102,12 @@ var _ = Describe("Verify Create", Ordered, func() {
for _, node := range tc.Servers {
nodeExternalIP, _ := node.FetchNodeExternalIP()
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + tc.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
cmd := "kubectl get service nginx-nodeport-svc --kubeconfig=" + tc.KubeconfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("test-nodeport"), "nodeport pod was not created")
......@@ -130,12 +130,12 @@ var _ = Describe("Verify Create", Ordered, func() {
for _, node := range tc.Servers {
ip, _ := node.FetchNodeExternalIP()
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + tc.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
cmd := "kubectl get service nginx-loadbalancer-svc --kubeconfig=" + tc.KubeconfigFile + " --output jsonpath=\"{.spec.ports[0].port}\""
port, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-loadbalancer --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
g.Expect(res).Should(ContainSubstring("test-loadbalancer"))
......@@ -171,10 +171,10 @@ var _ = Describe("Verify Create", Ordered, func() {
_, err := tc.DeployWorkload("daemonset.yaml")
Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed")
nodes, _ := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, _ := e2e.ParseNodes(tc.KubeconfigFile, false)
Eventually(func(g Gomega) {
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeConfigFile)
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(nodes).To(HaveLen(count), "Daemonset pod count does not match node count")
}, "240s", "10s").Should(Succeed())
......@@ -185,14 +185,14 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods dnsutils --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods dnsutils --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
g.Expect(res).Should(ContainSubstring("dnsutils"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl --kubeconfig=" + tc.KubeConfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
cmd := "kubectl --kubeconfig=" + tc.KubeconfigFile + " exec -i -t dnsutils -- nslookup kubernetes.default"
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
......@@ -205,7 +205,7 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed: "+res)
Eventually(func(g Gomega) {
cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pvc local-path-pvc --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
g.Expect(res).Should(ContainSubstring("local-path-pvc"))
......@@ -213,18 +213,18 @@ var _ = Describe("Verify Create", Ordered, func() {
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl get pod volume-test --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pod volume-test --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
g.Expect(res).Should(ContainSubstring("volume-test"))
g.Expect(res).Should(ContainSubstring("Running"))
}, "420s", "2s").Should(Succeed())
cmd := "kubectl --kubeconfig=" + tc.KubeConfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
cmd := "kubectl --kubeconfig=" + tc.KubeconfigFile + " exec volume-test -- sh -c 'echo local-path-test > /data/test'"
res, err = e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
cmd = "kubectl delete pod volume-test --kubeconfig=" + tc.KubeConfigFile
cmd = "kubectl delete pod volume-test --kubeconfig=" + tc.KubeconfigFile
res, err = e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
......@@ -232,13 +232,13 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "local-path-provisioner manifest not deployed")
Eventually(func(g Gomega) {
cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l app=local-path-provisioner --field-selector=status.phase=Running -n kube-system --kubeconfig=" + tc.KubeconfigFile
res, _ := e2e.RunCommand(cmd)
g.Expect(res).Should(ContainSubstring("local-path-provisioner"))
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl get pod volume-test --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pod volume-test --kubeconfig=" + tc.KubeconfigFile
res, err := e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
......@@ -247,7 +247,7 @@ var _ = Describe("Verify Create", Ordered, func() {
}, "420s", "2s").Should(Succeed())
Eventually(func(g Gomega) {
cmd := "kubectl exec volume-test --kubeconfig=" + tc.KubeConfigFile + " -- cat /data/test"
cmd := "kubectl exec volume-test --kubeconfig=" + tc.KubeconfigFile + " -- cat /data/test"
res, err = e2e.RunCommand(cmd)
g.Expect(err).NotTo(HaveOccurred(), "failed cmd: %q result: %s", cmd, res)
fmt.Println("Data after re-creation", res)
......@@ -262,12 +262,12 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(errRestart).NotTo(HaveOccurred(), "Restart Nodes not happened correctly")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
}
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeConfigFile)
count, err := e2e.GetDaemonsetReady("test-daemonset", tc.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should((Equal(count)), "Daemonset pods that are ready does not match node count")
}, "620s", "5s").Should(Succeed())
......@@ -303,7 +303,7 @@ var _ = Describe("Verify Create", Ordered, func() {
}, "620s", "5s").Should(Succeed())
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "5s").Should(Succeed())
})
......@@ -356,6 +356,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
tests/e2e/wasm/wasm_test.go
View file @ 74d18149
......@@ -48,7 +48,7 @@ var _ = Describe("Verify K3s can run Wasm workloads", Ordered, func() {
It("Checks node and pod status", func() {
By("Fetching Nodes status")
Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
nodes, err := e2e.ParseNodes(tc.KubeconfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready"))
......@@ -57,10 +57,10 @@ var _ = Describe("Verify K3s can run Wasm workloads", Ordered, func() {
By("Fetching pod status")
Eventually(func() error {
return tests.AllPodsUp(tc.KubeConfigFile)
return tests.AllPodsUp(tc.KubeconfigFile)
}, "620s", "10s").Should(Succeed())
Eventually(func() error {
return tests.CheckDefaultDeployments(tc.KubeConfigFile)
return tests.CheckDefaultDeployments(tc.KubeconfigFile)
}, "300s", "10s").Should(Succeed())
})
......@@ -84,14 +84,14 @@ var _ = Describe("Verify K3s can run Wasm workloads", Ordered, func() {
It("Wait for slight Pod to be up and running", func() {
Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l app=wasm-slight --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l app=wasm-slight --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("pod/wasm-slight"))
})
It("Wait for spin Pod to be up and running", func() {
Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l app=wasm-spin --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
cmd := "kubectl get pods -o=name -l app=wasm-spin --field-selector=status.phase=Running --kubeconfig=" + tc.KubeconfigFile
return e2e.RunCommand(cmd)
}, "120s", "5s").Should(ContainSubstring("pod/wasm-spin"))
})
......@@ -100,7 +100,7 @@ var _ = Describe("Verify K3s can run Wasm workloads", Ordered, func() {
var ingressIPs []string
var err error
Eventually(func(g Gomega) {
ingressIPs, err = e2e.FetchIngressIP(tc.KubeConfigFile)
ingressIPs, err = e2e.FetchIngressIP(tc.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(ingressIPs).To(HaveLen(1))
}, "120s", "5s").Should(Succeed())
......@@ -132,6 +132,6 @@ var _ = AfterSuite(func() {
}
if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
Expect(os.Remove(tc.KubeconfigFile)).To(Succeed())
}
})
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment