Commit 286793d1 authored by Derek Nola's avatar Derek Nola

Consolidate test utility functions into top level package (#11711)

* [e2e] Convert RunCmdOnNode to method * Consolidate e2e variables into TestConfig struct * Consolidate docker and integration test helper functions * E2E: Directly count daemonsets, not their pods * Add missing Context levels for E2E tests * Migrate e2e.ParsePods to new tests client package * Run the go test compile test on their respective architectures Signed-off-by: 's avatarDerek Nola <derek.nola@suse.com>
parent 2b7476a3
...@@ -108,10 +108,10 @@ jobs: ...@@ -108,10 +108,10 @@ jobs:
build-go-tests: build-go-tests:
name: "Build Go Tests" name: "Build Go Tests"
runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
arch: [amd64, arm64] arch: [amd64, arm64]
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
outputs: outputs:
channel: ${{ steps.channel_step.outputs.channel }} channel: ${{ steps.channel_step.outputs.channel }}
steps: steps:
...@@ -122,7 +122,7 @@ jobs: ...@@ -122,7 +122,7 @@ jobs:
- name: Build Go Tests - name: Build Go Tests
run: | run: |
mkdir -p ./dist/artifacts mkdir -p ./dist/artifacts
GOOS=linux GOARCH=${{ matrix.arch }} go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/... go test -c -ldflags="-w -s" -o ./dist/artifacts ./tests/docker/...
- name: Upload Go Tests - name: Upload Go Tests
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
......
package tests
import (
"context"
"fmt"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/utils/set"
)
// This file consolidates functions that are used across multiple testing frameworks.
// Most of it relates to interacting with the Kubernetes API and checking the status of resources.
// CheckDefaultDeployments checks if the standard array of K3s deployments are ready, otherwise returns an error
func CheckDefaultDeployments(kubeconfigFile string) error {
return CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, kubeconfigFile)
}
// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error
func CheckDeployments(deployments []string, kubeconfigFile string) error {
deploymentSet := make(map[string]bool)
for _, d := range deployments {
deploymentSet[d] = false
}
client, err := K8sClient(kubeconfigFile)
if err != nil {
return err
}
deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return err
}
for _, deployment := range deploymentList.Items {
if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas {
deploymentSet[deployment.Name] = true
}
}
for d, found := range deploymentSet {
if !found {
return fmt.Errorf("failed to deploy %s", d)
}
}
return nil
}
func ParseNodes(kubeconfigFile string) ([]corev1.Node, error) {
clientSet, err := K8sClient(kubeconfigFile)
if err != nil {
return nil, err
}
nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return nodes.Items, nil
}
func ParsePods(kubeconfigFile string) ([]corev1.Pod, error) {
clientSet, err := K8sClient(kubeconfigFile)
if err != nil {
return nil, err
}
pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return pods.Items, nil
}
// AllPodsUp checks if pods on the cluster are Running or Succeeded, otherwise returns an error
func AllPodsUp(kubeconfigFile string) error {
clientSet, err := K8sClient(kubeconfigFile)
if err != nil {
return err
}
pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return err
}
for _, pod := range pods.Items {
// Check if the pod is running
if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded {
return fmt.Errorf("pod %s is %s", pod.Name, pod.Status.Phase)
}
}
return nil
}
// PodReady checks if a pod is ready by querying its status
func PodReady(podName, namespace, kubeconfigFile string) (bool, error) {
clientSet, err := K8sClient(kubeconfigFile)
if err != nil {
return false, err
}
pod, err := clientSet.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
return false, fmt.Errorf("failed to get pod: %v", err)
}
// Check if the pod is running
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerStatus.Name == podName && containerStatus.Ready {
return true, nil
}
}
return false, nil
}
// Checks if provided nodes are ready, otherwise returns an error
func NodesReady(kubeconfigFile string, nodeNames []string) error {
nodes, err := ParseNodes(kubeconfigFile)
if err != nil {
return err
}
nodesToCheck := set.New(nodeNames...)
readyNodes := make(set.Set[string], 0)
for _, node := range nodes {
for _, condition := range node.Status.Conditions {
if condition.Type == corev1.NodeReady && condition.Status != corev1.ConditionTrue {
return fmt.Errorf("node %s is not ready", node.Name)
}
readyNodes.Insert(node.Name)
}
}
// Check if all nodes are ready
if !nodesToCheck.Equal(readyNodes) {
return fmt.Errorf("expected nodes %v, found %v", nodesToCheck, readyNodes)
}
return nil
}
func K8sClient(kubeconfigFile string) (*kubernetes.Clientset, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile)
if err != nil {
return nil, err
}
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return clientSet, nil
}
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -31,10 +32,10 @@ var _ = Describe("Basic Tests", Ordered, func() { ...@@ -31,10 +32,10 @@ var _ = Describe("Basic Tests", Ordered, func() {
Expect(config.ProvisionServers(1)).To(Succeed()) Expect(config.ProvisionServers(1)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "40s", "5s").Should(Succeed()) }, "40s", "5s").Should(Succeed())
}) })
}) })
...@@ -46,7 +47,7 @@ var _ = Describe("Basic Tests", Ordered, func() { ...@@ -46,7 +47,7 @@ var _ = Describe("Basic Tests", Ordered, func() {
}) })
It("should validate local storage volume", func() { It("should validate local storage volume", func() {
Eventually(func() (bool, error) { Eventually(func() (bool, error) {
return tester.PodReady("volume-test", "kube-system", config.KubeconfigFile) return tests.PodReady("volume-test", "kube-system", config.KubeconfigFile)
}, "20s", "5s").Should(BeTrue()) }, "20s", "5s").Should(BeTrue())
}) })
}) })
......
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -28,7 +29,7 @@ var _ = Describe("Boostrap Token Tests", Ordered, func() { ...@@ -28,7 +29,7 @@ var _ = Describe("Boostrap Token Tests", Ordered, func() {
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(config.ProvisionServers(1)).To(Succeed()) Expect(config.ProvisionServers(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
}) })
...@@ -46,10 +47,10 @@ var _ = Describe("Boostrap Token Tests", Ordered, func() { ...@@ -46,10 +47,10 @@ var _ = Describe("Boostrap Token Tests", Ordered, func() {
config.Token = newSecret config.Token = newSecret
Expect(config.ProvisionAgents(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := tester.ParseNodes(config.KubeconfigFile) nodes, err := tests.ParseNodes(config.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(nodes).To(HaveLen(2)) g.Expect(nodes).To(HaveLen(2))
g.Expect(tester.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed()) g.Expect(tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed())
}, "40s", "5s").Should(Succeed()) }, "40s", "5s").Should(Succeed())
}) })
}) })
......
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -62,7 +63,7 @@ var _ = Describe("CA Certs Tests", Ordered, func() { ...@@ -62,7 +63,7 @@ var _ = Describe("CA Certs Tests", Ordered, func() {
Expect(config.ProvisionServers(1)).To(Succeed()) Expect(config.ProvisionServers(1)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
}) })
......
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"os" "os"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -30,10 +31,10 @@ var _ = Describe("Etcd Tests", Ordered, func() { ...@@ -30,10 +31,10 @@ var _ = Describe("Etcd Tests", Ordered, func() {
It("should provision servers", func() { It("should provision servers", func() {
Expect(config.ProvisionServers(3)).To(Succeed()) Expect(config.ProvisionServers(3)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
It("should destroy the cluster", func() { It("should destroy the cluster", func() {
...@@ -56,10 +57,10 @@ var _ = Describe("Etcd Tests", Ordered, func() { ...@@ -56,10 +57,10 @@ var _ = Describe("Etcd Tests", Ordered, func() {
Expect(config.ProvisionServers(5)).To(Succeed()) Expect(config.ProvisionServers(5)).To(Succeed())
Expect(config.ProvisionAgents(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "90s", "5s").Should(Succeed()) }, "90s", "5s").Should(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "90s", "5s").Should(Succeed()) }, "90s", "5s").Should(Succeed())
}) })
}) })
......
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -30,10 +31,10 @@ var _ = Describe("LazyPull Tests", Ordered, func() { ...@@ -30,10 +31,10 @@ var _ = Describe("LazyPull Tests", Ordered, func() {
config.ServerYaml = "snapshotter: stargz" config.ServerYaml = "snapshotter: stargz"
Expect(config.ProvisionServers(1)).To(Succeed()) Expect(config.ProvisionServers(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "40s", "5s").Should(Succeed()) }, "40s", "5s").Should(Succeed())
}) })
}) })
...@@ -45,7 +46,7 @@ var _ = Describe("LazyPull Tests", Ordered, func() { ...@@ -45,7 +46,7 @@ var _ = Describe("LazyPull Tests", Ordered, func() {
}) })
It("should have the pod come up", func() { It("should have the pod come up", func() {
Eventually(func() (bool, error) { Eventually(func() (bool, error) {
return tester.PodReady("stargz-snapshot-test", "default", config.KubeconfigFile) return tests.PodReady("stargz-snapshot-test", "default", config.KubeconfigFile)
}, "30s", "5s").Should(BeTrue()) }, "30s", "5s").Should(BeTrue())
}) })
var topLayer string var topLayer string
......
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"testing" "testing"
"github.com/blang/semver/v4" "github.com/blang/semver/v4"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -62,7 +63,7 @@ var _ = Describe("Skew Tests", Ordered, func() { ...@@ -62,7 +63,7 @@ var _ = Describe("Skew Tests", Ordered, func() {
config.K3sImage = "rancher/k3s:" + lastMinorVersion config.K3sImage = "rancher/k3s:" + lastMinorVersion
Expect(config.ProvisionAgents(1)).To(Succeed()) Expect(config.ProvisionAgents(1)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
It("should match respective versions", func() { It("should match respective versions", func() {
...@@ -85,7 +86,7 @@ var _ = Describe("Skew Tests", Ordered, func() { ...@@ -85,7 +86,7 @@ var _ = Describe("Skew Tests", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest") Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest")
Eventually(func() (bool, error) { Eventually(func() (bool, error) {
return tester.PodReady("volume-test", "kube-system", config.KubeconfigFile) return tests.PodReady("volume-test", "kube-system", config.KubeconfigFile)
}, "20s", "5s").Should(BeTrue()) }, "20s", "5s").Should(BeTrue())
}) })
It("should destroy the cluster", func() { It("should destroy the cluster", func() {
...@@ -103,11 +104,11 @@ var _ = Describe("Skew Tests", Ordered, func() { ...@@ -103,11 +104,11 @@ var _ = Describe("Skew Tests", Ordered, func() {
config.K3sImage = *k3sImage config.K3sImage = *k3sImage
Expect(config.ProvisionServers(3)).To(Succeed()) Expect(config.ProvisionServers(3)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
g.Expect(tester.ParseNodes(config.KubeconfigFile)).To(HaveLen(3)) g.Expect(tests.ParseNodes(config.KubeconfigFile)).To(HaveLen(3))
g.Expect(tester.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed()) g.Expect(tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())).To(Succeed())
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
It("should match respective versions", func() { It("should match respective versions", func() {
......
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -35,10 +36,10 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() { ...@@ -35,10 +36,10 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
Expect(config.ProvisionServers(*serverCount)).To(Succeed()) Expect(config.ProvisionServers(*serverCount)).To(Succeed())
Expect(config.ProvisionAgents(*agentCount)).To(Succeed()) Expect(config.ProvisionAgents(*agentCount)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.CheckDefaultDeployments(config.KubeconfigFile) return tests.CheckDefaultDeployments(config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "40s", "5s").Should(Succeed()) }, "40s", "5s").Should(Succeed())
}) })
}) })
...@@ -135,12 +136,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() { ...@@ -135,12 +136,12 @@ var _ = Describe("Verify snapshots and cluster restores work", Ordered, func() {
It("Checks that all nodes and pods are ready", func() { It("Checks that all nodes and pods are ready", func() {
By("Fetching node status") By("Fetching node status")
Eventually(func() error { Eventually(func() error {
return tester.NodesReady(config.KubeconfigFile, config.GetNodeNames()) return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
By("Fetching Pods status") By("Fetching Pods status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
pods, err := tester.ParsePods(config.KubeconfigFile) pods, err := tests.ParsePods(config.KubeconfigFile)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods { for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") { if strings.Contains(pod.Name, "helm-install") {
......
package docker package docker
import ( import (
"context"
"fmt" "fmt"
"math/rand" "math/rand"
"net" "net"
...@@ -15,11 +14,6 @@ import ( ...@@ -15,11 +14,6 @@ import (
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/utils/set"
) )
type TestConfig struct { type TestConfig struct {
...@@ -491,121 +485,3 @@ func (config TestConfig) DeployWorkload(workload string) (string, error) { ...@@ -491,121 +485,3 @@ func (config TestConfig) DeployWorkload(workload string) (string, error) {
} }
return "", nil return "", nil
} }
// TODO the below functions are duplicated in the integration test utils. Consider combining into commmon package
// CheckDefaultDeployments checks if the default deployments: coredns, local-path-provisioner, metrics-server, traefik
// for K3s are ready, otherwise returns an error
func CheckDefaultDeployments(kubeconfigFile string) error {
return DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, kubeconfigFile)
}
// DeploymentsReady checks if the provided list of deployments are ready, otherwise returns an error
func DeploymentsReady(deployments []string, kubeconfigFile string) error {
deploymentSet := make(map[string]bool)
for _, d := range deployments {
deploymentSet[d] = false
}
client, err := k8sClient(kubeconfigFile)
if err != nil {
return err
}
deploymentList, err := client.AppsV1().Deployments("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return err
}
for _, deployment := range deploymentList.Items {
if _, ok := deploymentSet[deployment.Name]; ok && deployment.Status.ReadyReplicas == deployment.Status.Replicas {
deploymentSet[deployment.Name] = true
}
}
for d, found := range deploymentSet {
if !found {
return fmt.Errorf("failed to deploy %s", d)
}
}
return nil
}
func ParseNodes(kubeconfigFile string) ([]corev1.Node, error) {
clientSet, err := k8sClient(kubeconfigFile)
if err != nil {
return nil, err
}
nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return nodes.Items, nil
}
func ParsePods(kubeconfigFile string) ([]corev1.Pod, error) {
clientSet, err := k8sClient(kubeconfigFile)
if err != nil {
return nil, err
}
pods, err := clientSet.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return pods.Items, nil
}
// PodReady checks if a pod is ready by querying its status
func PodReady(podName, namespace, kubeconfigFile string) (bool, error) {
clientSet, err := k8sClient(kubeconfigFile)
if err != nil {
return false, err
}
pod, err := clientSet.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
if err != nil {
return false, fmt.Errorf("failed to get pod: %v", err)
}
// Check if the pod is running
for _, containerStatus := range pod.Status.ContainerStatuses {
if containerStatus.Name == podName && containerStatus.Ready {
return true, nil
}
}
return false, nil
}
// Checks if provided nodes are ready, otherwise returns an error
func NodesReady(kubeconfigFile string, nodeNames []string) error {
nodes, err := ParseNodes(kubeconfigFile)
if err != nil {
return err
}
nodesToCheck := set.New(nodeNames...)
readyNodes := make(set.Set[string], 0)
for _, node := range nodes {
for _, condition := range node.Status.Conditions {
if condition.Type == corev1.NodeReady && condition.Status != corev1.ConditionTrue {
return fmt.Errorf("node %s is not ready", node.Name)
}
readyNodes.Insert(node.Name)
}
}
// Check if all nodes are ready
if !nodesToCheck.Equal(readyNodes) {
return fmt.Errorf("expected nodes %v, found %v", nodesToCheck, readyNodes)
}
return nil
}
func k8sClient(kubeconfigFile string) (*kubernetes.Clientset, error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeconfigFile)
if err != nil {
return nil, err
}
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, err
}
return clientSet, nil
}
...@@ -9,6 +9,7 @@ import ( ...@@ -9,6 +9,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
tester "github.com/k3s-io/k3s/tests/docker" tester "github.com/k3s-io/k3s/tests/docker"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -69,7 +70,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() { ...@@ -69,7 +70,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
Expect(config.ProvisionServers(numServers)).To(Succeed()) Expect(config.ProvisionServers(numServers)).To(Succeed())
Expect(config.ProvisionAgents(numAgents)).To(Succeed()) Expect(config.ProvisionAgents(numAgents)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
It("should confirm latest version", func() { It("should confirm latest version", func() {
...@@ -84,7 +85,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() { ...@@ -84,7 +85,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest") Expect(err).NotTo(HaveOccurred(), "failed to apply volume test manifest")
Eventually(func() (bool, error) { Eventually(func() (bool, error) {
return tester.PodReady("volume-test", "kube-system", config.KubeconfigFile) return tests.PodReady("volume-test", "kube-system", config.KubeconfigFile)
}, "20s", "5s").Should(BeTrue()) }, "20s", "5s").Should(BeTrue())
}) })
It("should upgrade to current commit build", func() { It("should upgrade to current commit build", func() {
...@@ -111,7 +112,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() { ...@@ -111,7 +112,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
Expect(config.ProvisionAgents(numAgents)).To(Succeed()) Expect(config.ProvisionAgents(numAgents)).To(Succeed())
Eventually(func() error { Eventually(func() error {
return tester.DeploymentsReady([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
}) })
It("should confirm commit version", func() { It("should confirm commit version", func() {
...@@ -131,7 +132,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() { ...@@ -131,7 +132,7 @@ var _ = Describe("Upgrade Tests", Ordered, func() {
}) })
It("should confirm test pod is still Running", func() { It("should confirm test pod is still Running", func() {
Eventually(func() (bool, error) { Eventually(func() (bool, error) {
return tester.PodReady("volume-test", "kube-system", config.KubeconfigFile) return tests.PodReady("volume-test", "kube-system", config.KubeconfigFile)
}, "20s", "5s").Should(BeTrue()) }, "20s", "5s").Should(BeTrue())
}) })
......
...@@ -2,11 +2,10 @@ package rotateca ...@@ -2,11 +2,10 @@ package rotateca
import ( import (
"flag" "flag"
"fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -25,10 +24,7 @@ func Test_E2EBtrfsSnapshot(t *testing.T) { ...@@ -25,10 +24,7 @@ func Test_E2EBtrfsSnapshot(t *testing.T) {
RunSpecs(t, "Btrfs Snapshot Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Btrfs Snapshot Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -38,45 +34,34 @@ var _ = Describe("Verify that btrfs based servers work", Ordered, func() { ...@@ -38,45 +34,34 @@ var _ = Describe("Verify that btrfs based servers work", Ordered, func() {
var err error var err error
// OS and server are hardcoded because only openSUSE Leap 15.5 natively supports Btrfs // OS and server are hardcoded because only openSUSE Leap 15.5 natively supports Btrfs
if *local { if *local {
serverNodeNames, _, err = e2e.CreateLocalCluster("opensuse/Leap-15.6.x86_64", 1, 0) tc, err = e2e.CreateLocalCluster("opensuse/Leap-15.6.x86_64", 1, 0)
} else { } else {
serverNodeNames, _, err = e2e.CreateCluster("opensuse/Leap-15.6.x86_64", 1, 0) tc, err = e2e.CreateCluster("opensuse/Leap-15.6.x86_64", 1, 0)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks node and pod status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
fmt.Printf("\nFetching pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
}) })
It("Checks that btrfs snapshots exist", func() { It("Checks that btrfs snapshots exist", func() {
cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs" cmd := "btrfs subvolume list /var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.btrfs"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/\\d+")) Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/active/\\d+"))
Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/\\d+")) Expect(res).To(MatchRegexp("agent/containerd/io.containerd.snapshotter.v1.btrfs/snapshots/\\d+"))
...@@ -91,10 +76,10 @@ var _ = AfterEach(func() { ...@@ -91,10 +76,10 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(serverNodeNames)).To(Succeed()) Expect(e2e.SaveJournalLogs(tc.Servers)).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -27,62 +28,47 @@ func Test_E2EDualStack(t *testing.T) { ...@@ -27,62 +28,47 @@ func Test_E2EDualStack(t *testing.T) {
RunSpecs(t, "DualStack Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "DualStack Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
var _ = Describe("Verify DualStack Configuration", Ordered, func() { var _ = Describe("Verify DualStack Configuration", Ordered, func() {
Context("Cluster Deploys with both IPv6 and IPv4 networks", func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") tc.Hardened = *hardened
fmt.Println("OS:", *nodeOS) By("CLUSTER CONFIG")
fmt.Println("Server Nodes:", serverNodeNames) By("OS: " + *nodeOS)
fmt.Println("Agent Nodes:", agentNodeNames) By(tc.Status())
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node Status", func() { It("Checks Node Status", func() {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(kubeConfigFile, true) _, err := e2e.ParseNodes(tc.KubeConfigFile, true)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Pod Status", func() { It("Checks pod status", func() {
Eventually(func(g Gomega) { Eventually(func() error {
pods, err := e2e.ParsePods(kubeConfigFile, false) return tests.AllPodsUp(tc.KubeConfigFile)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, err := e2e.ParsePods(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred())
}) })
It("Verifies that each node has IPv4 and IPv6", func() { It("Verifies that each node has IPv4 and IPv6", func() {
nodeIPs, err := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs { for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("10.10.10")) Expect(node.IPv4).Should(ContainSubstring("10.10.10"))
...@@ -90,7 +76,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() { ...@@ -90,7 +76,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
} }
}) })
It("Verifies that each pod has IPv4 and IPv6", func() { It("Verifies that each pod has IPv4 and IPv6", func() {
podIPs, err := e2e.GetPodIPs(kubeConfigFile) podIPs, err := e2e.GetPodIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, pod := range podIPs { for _, pod := range podIPs {
Expect(pod.IPv4).Should(Or(ContainSubstring("10.10.10"), ContainSubstring("10.42.")), pod.Name) Expect(pod.IPv4).Should(Or(ContainSubstring("10.10.10"), ContainSubstring("10.42.")), pod.Name)
...@@ -99,21 +85,21 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() { ...@@ -99,21 +85,21 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
}) })
It("Verifies ClusterIP Service", func() { It("Verifies ClusterIP Service", func() {
_, err := e2e.DeployWorkload("dualstack_clusterip.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("dualstack_clusterip.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Eventually(func() (string, error) { Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile cmd := "kubectl get pods -o=name -l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod")) }, "120s", "5s").Should(ContainSubstring("ds-clusterip-pod"))
// Checks both IPv4 and IPv6 // Checks both IPv4 and IPv6
clusterips, err := e2e.FetchClusterIP(kubeConfigFile, "ds-clusterip-svc", true) clusterips, err := e2e.FetchClusterIP(tc.KubeConfigFile, "ds-clusterip-svc", true)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, ip := range strings.Split(clusterips, ",") { for _, ip := range strings.Split(clusterips, ",") {
if strings.Contains(ip, "::") { if strings.Contains(ip, "::") {
ip = "[" + ip + "]" ip = "[" + ip + "]"
} }
pods, err := e2e.ParsePods(kubeConfigFile, false) pods, err := tests.ParsePods(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, pod := range pods { for _, pod := range pods {
if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") { if !strings.HasPrefix(pod.Name, "ds-clusterip-pod") {
...@@ -121,18 +107,18 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() { ...@@ -121,18 +107,18 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
} }
cmd := fmt.Sprintf("curl -L --insecure http://%s", ip) cmd := fmt.Sprintf("curl -L --insecure http://%s", ip)
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) return tc.Servers[0].RunCmdOnNode(cmd)
}, "60s", "5s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd) }, "60s", "5s").Should(ContainSubstring("Welcome to nginx!"), "failed cmd: "+cmd)
} }
} }
}) })
It("Verifies Ingress", func() { It("Verifies Ingress", func() {
_, err := e2e.DeployWorkload("dualstack_ingress.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("dualstack_ingress.yaml")
Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed")
cmd := "kubectl get ingress ds-ingress --kubeconfig=" + kubeConfigFile + " -o jsonpath=\"{.spec.rules[*].host}\"" cmd := "kubectl get ingress ds-ingress -o jsonpath=\"{.spec.rules[*].host}\""
hostName, err := e2e.RunCommand(cmd) hostName, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
nodeIPs, err := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
for _, node := range nodeIPs { for _, node := range nodeIPs {
cmd := fmt.Sprintf("curl --header host:%s http://%s/name.html", hostName, node.IPv4) cmd := fmt.Sprintf("curl --header host:%s http://%s/name.html", hostName, node.IPv4)
...@@ -147,12 +133,12 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() { ...@@ -147,12 +133,12 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
}) })
It("Verifies NodePort Service", func() { It("Verifies NodePort Service", func() {
_, err := e2e.DeployWorkload("dualstack_nodeport.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("dualstack_nodeport.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd := "kubectl get service ds-nodeport-svc --kubeconfig=" + kubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" cmd := "kubectl get service ds-nodeport-svc --output jsonpath=\"{.spec.ports[0].nodePort}\""
nodeport, err := e2e.RunCommand(cmd) nodeport, err := e2e.RunCommand(cmd)
Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed cmd: "+cmd)
nodeIPs, err := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs { for _, node := range nodeIPs {
cmd = "curl -L --insecure http://" + node.IPv4 + ":" + nodeport + "/name.html" cmd = "curl -L --insecure http://" + node.IPv4 + ":" + nodeport + "/name.html"
...@@ -166,27 +152,28 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() { ...@@ -166,27 +152,28 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
} }
}) })
It("Verifies podSelector Network Policy", func() { It("Verifies podSelector Network Policy", func() {
_, err := e2e.DeployWorkload("pod_client.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("pod_client.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd := "kubectl exec svc/client-curl --kubeconfig=" + kubeConfigFile + " -- curl -m7 ds-clusterip-svc/name.html" cmd := "kubectl exec svc/client-curl -- curl -m7 ds-clusterip-svc/name.html"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "20s", "3s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd) }, "20s", "3s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd)
_, err = e2e.DeployWorkload("netpol-fail.yaml", kubeConfigFile, *hardened) _, err = tc.DeployWorkload("netpol-fail.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd = "kubectl exec svc/client-curl --kubeconfig=" + kubeConfigFile + " -- curl -m7 ds-clusterip-svc/name.html" cmd = "kubectl exec svc/client-curl -- curl -m7 ds-clusterip-svc/name.html"
Eventually(func() error { Eventually(func() error {
_, err = e2e.RunCommand(cmd) _, err = e2e.RunCommand(cmd)
Expect(err).To(HaveOccurred()) Expect(err).To(HaveOccurred())
return err return err
}, "20s", "3s") }, "20s", "3s")
_, err = e2e.DeployWorkload("netpol-work.yaml", kubeConfigFile, *hardened) _, err = tc.DeployWorkload("netpol-work.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd = "kubectl exec svc/client-curl --kubeconfig=" + kubeConfigFile + " -- curl -m7 ds-clusterip-svc/name.html" cmd = "kubectl exec svc/client-curl -- curl -m7 ds-clusterip-svc/name.html"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "20s", "3s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd) }, "20s", "3s").Should(ContainSubstring("ds-clusterip-pod"), "failed cmd: "+cmd)
}) })
})
}) })
var failed bool var failed bool
...@@ -196,12 +183,12 @@ var _ = AfterEach(func() { ...@@ -196,12 +183,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -4,9 +4,9 @@ import ( ...@@ -4,9 +4,9 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -32,11 +32,7 @@ func Test_E2EPrivateRegistry(t *testing.T) { ...@@ -32,11 +32,7 @@ func Test_E2EPrivateRegistry(t *testing.T) {
RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -45,42 +41,31 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -45,42 +41,31 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node and Pod Status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n") By("Fetching pod status")
Eventually(func(g Gomega) { Eventually(func() error {
pods, err := e2e.ParsePods(kubeConfigFile, false) e2e.DumpPods(tc.KubeConfigFile)
g.Expect(err).NotTo(HaveOccurred()) return tests.AllPodsUp(tc.KubeConfigFile)
for _, pod := range pods { }, "620s", "10s").Should(Succeed())
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true)
}) })
It("Should create and validate deployment with embedded registry mirror using image tag", func() { It("Should create and validate deployment with embedded registry mirror using image tag", func() {
res, err := e2e.RunCommand("kubectl create deployment my-webpage-1 --image=docker.io/library/nginx:1.25.3") res, err := e2e.RunCommand("kubectl create deployment my-webpage-1 --image=docker.io/library/nginx:1.25.3")
...@@ -128,7 +113,7 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -128,7 +113,7 @@ var _ = Describe("Verify Create", Ordered, func() {
}) })
It("Should expose embedded registry metrics", func() { It("Should expose embedded registry metrics", func() {
grepCmd := fmt.Sprintf("kubectl get --raw /api/v1/nodes/%s/proxy/metrics | grep -F 'spegel_advertised_images{registry=\"docker.io\"}'", serverNodeNames[0]) grepCmd := fmt.Sprintf("kubectl get --raw /api/v1/nodes/%s/proxy/metrics | grep -F 'spegel_advertised_images{registry=\"docker.io\"}'", tc.Servers[0])
res, err := e2e.RunCommand(grepCmd) res, err := e2e.RunCommand(grepCmd)
fmt.Println(res) fmt.Println(res)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
...@@ -147,12 +132,12 @@ var _ = AfterEach(func() { ...@@ -147,12 +132,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.SaveJournalLogs(append(tc.Servers, tc.Agents...))).To(Succeed())
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -7,11 +7,11 @@ package externalip ...@@ -7,11 +7,11 @@ package externalip
import ( import (
"flag" "flag"
"fmt"
"os" "os"
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -21,7 +21,6 @@ import ( ...@@ -21,7 +21,6 @@ import (
var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system") var nodeOS = flag.String("nodeOS", "bento/ubuntu-24.04", "VM operating system")
var serverCount = flag.Int("serverCount", 1, "number of server nodes") var serverCount = flag.Int("serverCount", 1, "number of server nodes")
var agentCount = flag.Int("agentCount", 1, "number of agent nodes") var agentCount = flag.Int("agentCount", 1, "number of agent nodes")
var hardened = flag.Bool("hardened", false, "true or false")
var ci = flag.Bool("ci", false, "running on CI") var ci = flag.Bool("ci", false, "running on CI")
var local = flag.Bool("local", false, "deploy a locally built K3s binary") var local = flag.Bool("local", false, "deploy a locally built K3s binary")
...@@ -55,108 +54,94 @@ func Test_E2EExternalIP(t *testing.T) { ...@@ -55,108 +54,94 @@ func Test_E2EExternalIP(t *testing.T) {
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
var _ = Describe("Verify External-IP config", Ordered, func() { var _ = Describe("Verify External-IP config", Ordered, func() {
Context("Cluster comes up with External-IP configuration", func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node Status", func() { It("Checks Node Status", func() {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(kubeConfigFile, true) _, err := e2e.ParseNodes(tc.KubeConfigFile, true)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Pod Status", func() { It("Checks pod status", func() {
Eventually(func(g Gomega) { By("Fetching pod status")
pods, err := e2e.ParsePods(kubeConfigFile, false) Eventually(func() error {
g.Expect(err).NotTo(HaveOccurred()) return tests.AllPodsUp(tc.KubeConfigFile)
for _, pod := range pods { }, "620s", "10s").Should(Succeed())
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed())
_, err := e2e.ParsePods(kubeConfigFile, true)
Expect(err).NotTo(HaveOccurred())
}) })
})
Context("Deploy workloads to check cluster connectivity of the nodes", func() {
It("Verifies that each node has vagrant IP", func() { It("Verifies that each node has vagrant IP", func() {
nodeIPs, err := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs { for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("10.10.")) Expect(node.IPv4).Should(ContainSubstring("10.10."))
} }
}) })
It("Verifies that each pod has vagrant IP or clusterCIDR IP", func() { It("Verifies that each pod has vagrant IP or clusterCIDR IP", func() {
podIPs, err := e2e.GetPodIPs(kubeConfigFile) podIPs, err := e2e.GetPodIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, pod := range podIPs { for _, pod := range podIPs {
Expect(pod.IPv4).Should(Or(ContainSubstring("10.10."), ContainSubstring("10.42.")), pod.Name) Expect(pod.IPv4).Should(Or(ContainSubstring("10.10."), ContainSubstring("10.42.")), pod.Name)
} }
}) })
It("Verifies that flannel added the correct annotation for the external-ip", func() { It("Verifies that flannel added the correct annotation for the external-ip", func() {
nodeIPs, err := getExternalIPs(kubeConfigFile) nodeIPs, err := getExternalIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, annotation := range nodeIPs { for _, annotation := range nodeIPs {
Expect(annotation).Should(ContainSubstring("10.100.100.")) Expect(annotation).Should(ContainSubstring("10.100.100."))
} }
}) })
It("Verifies internode connectivity over the tunnel", func() { It("Verifies internode connectivity over the tunnel", func() {
_, err := e2e.DeployWorkload("pod_client.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("pod_client.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Wait for the pod_client to have an IP // Wait for the pod_client to have an IP
Eventually(func() string { Eventually(func() string {
ips, _ := getClientIPs(kubeConfigFile) ips, _ := getClientIPs(tc.KubeConfigFile)
return ips[0].IPv4 return ips[0].IPv4
}, "40s", "5s").Should(ContainSubstring("10.42"), "failed getClientIPs") }, "40s", "5s").Should(ContainSubstring("10.42"), "failed getClientIPs")
clientIPs, err := getClientIPs(kubeConfigFile) clientIPs, err := getClientIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, ip := range clientIPs { for _, ip := range clientIPs {
cmd := "kubectl exec svc/client-curl --kubeconfig=" + kubeConfigFile + " -- curl -m7 " + ip.IPv4 + "/name.html" cmd := "kubectl exec svc/client-curl -- curl -m7 " + ip.IPv4 + "/name.html"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "20s", "3s").Should(ContainSubstring("client-deployment"), "failed cmd: "+cmd) }, "20s", "3s").Should(ContainSubstring("client-deployment"), "failed cmd: "+cmd)
} }
}) })
It("Verifies loadBalancer service's IP is the node-external-ip", func() { It("Verifies loadBalancer service's IP is the node-external-ip", func() {
_, err := e2e.DeployWorkload("loadbalancer.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("loadbalancer.yaml")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd := "kubectl --kubeconfig=" + kubeConfigFile + " get svc -l k8s-app=nginx-app-loadbalancer -o=jsonpath='{range .items[*]}{.metadata.name}{.status.loadBalancer.ingress[*].ip}{end}'" cmd := "kubectl get svc -l k8s-app=nginx-app-loadbalancer -o=jsonpath='{range .items[*]}{.metadata.name}{.status.loadBalancer.ingress[*].ip}{end}'"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "20s", "3s").Should(ContainSubstring("10.100.100"), "failed cmd: "+cmd) }, "20s", "3s").Should(ContainSubstring("10.100.100"), "failed cmd: "+cmd)
}) })
})
}) })
var failed bool var failed bool
...@@ -166,12 +151,12 @@ var _ = AfterEach(func() { ...@@ -166,12 +151,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.SaveJournalLogs(append(tc.Servers, tc.Agents...))).To(Succeed())
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -7,9 +7,11 @@ import ( ...@@ -7,9 +7,11 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
) )
// Valid nodeOS: // Valid nodeOS:
...@@ -32,11 +34,7 @@ func Test_E2EPrivateRegistry(t *testing.T) { ...@@ -32,11 +34,7 @@ func Test_E2EPrivateRegistry(t *testing.T) {
RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -45,52 +43,41 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -45,52 +43,41 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node and Pod Status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
fmt.Printf("\nFetching Pods status\n") By("Fetching pod status")
Eventually(func(g Gomega) { Eventually(func() error {
pods, err := e2e.ParsePods(kubeConfigFile, false) return tests.AllPodsUp(tc.KubeConfigFile)
g.Expect(err).NotTo(HaveOccurred()) }, "620s", "10s").Should(Succeed())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true)
}) })
It("Create new private registry", func() { It("Create new private registry", func() {
registry, err := e2e.RunCmdOnNode("docker run --init -d -p 5000:5000 --restart=always --name registry registry:2 ", serverNodeNames[0]) registry, err := tc.Servers[0].RunCmdOnNode("docker run --init -d -p 5000:5000 --restart=always --name registry registry:2 ")
fmt.Println(registry) fmt.Println(registry)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("ensures registry is working", func() { It("ensures registry is working", func() {
a, err := e2e.RunCmdOnNode("docker ps -a | grep registry\n", serverNodeNames[0]) a, err := tc.Servers[0].RunCmdOnNode("docker ps -a | grep registry\n")
fmt.Println(a) fmt.Println(a)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
...@@ -100,44 +87,44 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -100,44 +87,44 @@ var _ = Describe("Verify Create", Ordered, func() {
// NODEIP:5000 as a mirror. // NODEIP:5000 as a mirror.
It("Should pull and image from dockerhub and send it to private registry", func() { It("Should pull and image from dockerhub and send it to private registry", func() {
cmd := "docker pull docker.io/library/nginx:1.27.3" cmd := "docker pull docker.io/library/nginx:1.27.3"
_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), "failed: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
nodeIP, err := e2e.FetchNodeExternalIP(serverNodeNames[0]) nodeIP, err := tc.Servers[0].FetchNodeExternalIP()
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
cmd = "docker tag docker.io/library/nginx:1.27.3 " + nodeIP + ":5000/docker-io-library/nginx:1.27.3" cmd = "docker tag docker.io/library/nginx:1.27.3 " + nodeIP + ":5000/docker-io-library/nginx:1.27.3"
_, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err = tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), "failed: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
cmd = "docker push " + nodeIP + ":5000/docker-io-library/nginx:1.27.3" cmd = "docker push " + nodeIP + ":5000/docker-io-library/nginx:1.27.3"
_, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err = tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), "failed: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
cmd = "docker image remove docker.io/library/nginx:1.27.3 " + nodeIP + ":5000/docker-io-library/nginx:1.27.3" cmd = "docker image remove docker.io/library/nginx:1.27.3 " + nodeIP + ":5000/docker-io-library/nginx:1.27.3"
_, err = e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err = tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), "failed: "+cmd) Expect(err).NotTo(HaveOccurred(), "failed: "+cmd)
}) })
It("Should create and validate deployment with private registry on", func() { It("Should create and validate deployment with private registry on", func() {
res, err := e2e.RunCmdOnNode("kubectl create deployment my-webpage --image=my-registry.local/library/nginx:1.27.3", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("kubectl create deployment my-webpage --image=my-registry.local/library/nginx:1.27.3")
fmt.Println(res) fmt.Println(res)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
var pod e2e.Pod var pod corev1.Pod
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
pods, err := e2e.ParsePods(kubeConfigFile, false) pods, err := tests.ParsePods(tc.KubeConfigFile)
for _, p := range pods { for _, p := range pods {
if strings.Contains(p.Name, "my-webpage") { if strings.Contains(p.Name, "my-webpage") {
pod = p pod = p
} }
} }
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(pod.Status).Should(Equal("Running")) g.Expect(string(pod.Status.Phase)).Should(Equal("Running"))
}, "60s", "5s").Should(Succeed()) }, "60s", "5s").Should(Succeed())
cmd := "curl " + pod.IP cmd := "curl " + pod.Status.PodIP
Expect(e2e.RunCmdOnNode(cmd, serverNodeNames[0])).To(ContainSubstring("Welcome to nginx!")) Expect(tc.Servers[0].RunCmdOnNode(cmd)).To(ContainSubstring("Welcome to nginx!"))
}) })
}) })
...@@ -150,17 +137,17 @@ var _ = AfterEach(func() { ...@@ -150,17 +137,17 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.SaveJournalLogs(append(tc.Servers, tc.Agents...))).To(Succeed())
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
r1, err := e2e.RunCmdOnNode("docker rm -f registry", serverNodeNames[0]) r1, err := tc.Servers[0].RunCmdOnNode("docker rm -f registry")
Expect(err).NotTo(HaveOccurred(), r1) Expect(err).NotTo(HaveOccurred(), r1)
r2, err := e2e.RunCmdOnNode("kubectl delete deployment my-webpage", serverNodeNames[0]) r2, err := tc.Servers[0].RunCmdOnNode("kubectl delete deployment my-webpage")
Expect(err).NotTo(HaveOccurred(), r2) Expect(err).NotTo(HaveOccurred(), r2)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -4,9 +4,9 @@ import ( ...@@ -4,9 +4,9 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -29,43 +29,40 @@ func Test_E2ERootlessStartupValidation(t *testing.T) { ...@@ -29,43 +29,40 @@ func Test_E2ERootlessStartupValidation(t *testing.T) {
RunSpecs(t, "Startup Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Startup Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
)
func StartK3sCluster(nodes []string, serverYAML string) error { func StartK3sCluster(nodes []e2e.VagrantNode, serverYAML string) error {
for _, node := range nodes { for _, node := range nodes {
resetCmd := "head -n 3 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml" resetCmd := "head -n 3 /etc/rancher/k3s/config.yaml > /tmp/config.yaml && sudo mv /tmp/config.yaml /etc/rancher/k3s/config.yaml"
yamlCmd := fmt.Sprintf("echo '%s' >> /etc/rancher/k3s/config.yaml", serverYAML) yamlCmd := fmt.Sprintf("echo '%s' >> /etc/rancher/k3s/config.yaml", serverYAML)
startCmd := "systemctl --user restart k3s-rootless" startCmd := "systemctl --user restart k3s-rootless"
if _, err := e2e.RunCmdOnNode(resetCmd, node); err != nil { if _, err := node.RunCmdOnNode(resetCmd); err != nil {
return err return err
} }
if _, err := e2e.RunCmdOnNode(yamlCmd, node); err != nil { if _, err := node.RunCmdOnNode(yamlCmd); err != nil {
return err return err
} }
if _, err := RunCmdOnRootlesNode("systemctl --user daemon-reload", node); err != nil { if _, err := RunCmdOnRootlessNode("systemctl --user daemon-reload", node.String()); err != nil {
return err return err
} }
if _, err := RunCmdOnRootlesNode(startCmd, node); err != nil { if _, err := RunCmdOnRootlessNode(startCmd, node.String()); err != nil {
return err return err
} }
} }
return nil return nil
} }
func KillK3sCluster(nodes []string) error { func KillK3sCluster(nodes []e2e.VagrantNode) error {
for _, node := range nodes { for _, node := range nodes {
if _, err := RunCmdOnRootlesNode(`systemctl --user stop k3s-rootless`, node); err != nil { if _, err := RunCmdOnRootlessNode(`systemctl --user stop k3s-rootless`, node.String()); err != nil {
return err return err
} }
if _, err := RunCmdOnRootlesNode("k3s-killall.sh", node); err != nil { if _, err := RunCmdOnRootlessNode("k3s-killall.sh", node.String()); err != nil {
return err return err
} }
if _, err := RunCmdOnRootlesNode("rm -rf /home/vagrant/.rancher/k3s/server/db", node); err != nil { if _, err := RunCmdOnRootlessNode("rm -rf /home/vagrant/.rancher/k3s/server/db", node.String()); err != nil {
return err return err
} }
} }
...@@ -77,13 +74,13 @@ var _ = ReportAfterEach(e2e.GenReport) ...@@ -77,13 +74,13 @@ var _ = ReportAfterEach(e2e.GenReport)
var _ = BeforeSuite(func() { var _ = BeforeSuite(func() {
var err error var err error
if *local { if *local {
serverNodeNames, _, err = e2e.CreateLocalCluster(*nodeOS, 1, 0) tc, err = e2e.CreateLocalCluster(*nodeOS, 1, 0)
} else { } else {
serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, 1, 0) tc, err = e2e.CreateCluster(*nodeOS, 1, 0)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
//Checks if system is using cgroup v2 //Checks if system is using cgroup v2
_, err = e2e.RunCmdOnNode("cat /sys/fs/cgroup/cgroup.controllers", serverNodeNames[0]) _, err = tc.Servers[0].RunCmdOnNode("cat /sys/fs/cgroup/cgroup.controllers")
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
}) })
...@@ -91,40 +88,32 @@ var _ = BeforeSuite(func() { ...@@ -91,40 +88,32 @@ var _ = BeforeSuite(func() {
var _ = Describe("Various Startup Configurations", Ordered, func() { var _ = Describe("Various Startup Configurations", Ordered, func() {
Context("Verify standard startup :", func() { Context("Verify standard startup :", func() {
It("Starts K3s with no issues", func() { It("Starts K3s with no issues", func() {
err := StartK3sCluster(serverNodeNames, "") err := StartK3sCluster(tc.Servers, "")
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
kubeConfigFile, err = GenRootlessKubeConfigFile(serverNodeNames[0]) kubeConfigFile, err := GenRootlessKubeConfigFile(tc.Servers[0].String())
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
tc.KubeConfigFile = kubeConfigFile
}) })
It("Checks node and pod status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "360s", "5s").Should(Succeed()) }, "360s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, false) _, _ = e2e.ParseNodes(tc.KubeConfigFile, false)
fmt.Printf("\nFetching pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "360s", "5s").Should(Succeed()) }, "360s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, false) e2e.DumpPods(tc.KubeConfigFile)
}) })
It("Returns pod metrics", func() { It("Returns pod metrics", func() {
...@@ -154,7 +143,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() { ...@@ -154,7 +143,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
}) })
It("Kills the cluster", func() { It("Kills the cluster", func() {
err := KillK3sCluster(serverNodeNames) err := KillK3sCluster(tc.Servers)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
}) })
...@@ -168,12 +157,12 @@ var _ = AfterEach(func() { ...@@ -168,12 +157,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames)) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, tc.Servers))
} else { } else {
Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) Expect(e2e.GetCoverageReport(tc.Servers)).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -8,8 +8,8 @@ import ( ...@@ -8,8 +8,8 @@ import (
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
) )
// RunCmdOnRootlesNode executes a command from within the given node as user vagrant // RunCmdOnRootlessNode executes a command from within the given node as user vagrant
func RunCmdOnRootlesNode(cmd string, nodename string) (string, error) { func RunCmdOnRootlessNode(cmd string, nodename string) (string, error) {
injectEnv := "" injectEnv := ""
if _, ok := os.LookupEnv("E2E_GOCOVER"); ok && strings.HasPrefix(cmd, "k3s") { if _, ok := os.LookupEnv("E2E_GOCOVER"); ok && strings.HasPrefix(cmd, "k3s") {
injectEnv = "GOCOVERDIR=/tmp/k3scov " injectEnv = "GOCOVERDIR=/tmp/k3scov "
...@@ -23,11 +23,12 @@ func RunCmdOnRootlesNode(cmd string, nodename string) (string, error) { ...@@ -23,11 +23,12 @@ func RunCmdOnRootlesNode(cmd string, nodename string) (string, error) {
} }
func GenRootlessKubeConfigFile(serverName string) (string, error) { func GenRootlessKubeConfigFile(serverName string) (string, error) {
kubeConfig, err := RunCmdOnRootlesNode("cat /home/vagrant/.kube/k3s.yaml", serverName) kubeConfig, err := RunCmdOnRootlessNode("cat /home/vagrant/.kube/k3s.yaml", serverName)
if err != nil { if err != nil {
return "", err return "", err
} }
nodeIP, err := e2e.FetchNodeExternalIP(serverName) vNode := e2e.VagrantNode(serverName)
nodeIP, err := vNode.FetchNodeExternalIP()
if err != nil { if err != nil {
return "", err return "", err
} }
......
...@@ -2,11 +2,10 @@ package rotateca ...@@ -2,11 +2,10 @@ package rotateca
import ( import (
"flag" "flag"
"fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -29,11 +28,7 @@ func Test_E2ECustomCARotation(t *testing.T) { ...@@ -29,11 +28,7 @@ func Test_E2ECustomCARotation(t *testing.T) {
RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Custom Certificate Rotation Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
agentNodeNames []string
serverNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -42,43 +37,32 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { ...@@ -42,43 +37,32 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks node and pod status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.ParseNodes(tc.KubeConfigFile, true)
fmt.Printf("\nFetching pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
}) })
It("Generates New CA Certificates", func() { It("Generates New CA Certificates", func() {
...@@ -88,46 +72,38 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() { ...@@ -88,46 +72,38 @@ var _ = Describe("Verify Custom CA Rotation", Ordered, func() {
"DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh", "DATA_DIR=/opt/rancher/k3s /tmp/generate-custom-ca-certs.sh",
} }
for _, cmd := range cmds { for _, cmd := range cmds {
_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
}) })
It("Rotates CA Certificates", func() { It("Rotates CA Certificates", func() {
cmd := "k3s certificate rotate-ca --path=/opt/rancher/k3s/server" cmd := "k3s certificate rotate-ca --path=/opt/rancher/k3s/server"
_, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Restarts K3s servers", func() { It("Restarts K3s servers", func() {
Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) Expect(e2e.RestartCluster(tc.Servers)).To(Succeed())
}) })
It("Restarts K3s agents", func() { It("Restarts K3s agents", func() {
Expect(e2e.RestartCluster(agentNodeNames)).To(Succeed()) Expect(e2e.RestartCluster(tc.Agents)).To(Succeed())
}) })
It("Checks node and pod status", func() { It("Checks node and pod status", func() {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "420s", "5s").Should(Succeed()) }, "420s", "5s").Should(Succeed())
Eventually(func(g Gomega) { Eventually(func() error {
pods, err := e2e.ParsePods(kubeConfigFile, false) return tests.AllPodsUp(tc.KubeConfigFile)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "420s", "5s").Should(Succeed()) }, "420s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
}) })
}) })
}) })
...@@ -139,12 +115,12 @@ var _ = AfterEach(func() { ...@@ -139,12 +115,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -31,11 +32,7 @@ func Test_E2ES3(t *testing.T) { ...@@ -31,11 +32,7 @@ func Test_E2ES3(t *testing.T) {
RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Create Cluster Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -44,93 +41,79 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -44,93 +41,79 @@ var _ = Describe("Verify Create", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, 1, 0) tc, err = e2e.CreateLocalCluster(*nodeOS, 1, 0)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, 1, 0) tc, err = e2e.CreateCluster(*nodeOS, 1, 0)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node and Pod Status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.ParseNodes(tc.KubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
}) })
It("ensures s3 mock is working", func() { It("ensures s3 mock is working", func() {
res, err := e2e.RunCmdOnNode("docker ps -a | grep mock\n", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("docker ps -a | grep mock\n")
fmt.Println(res) fmt.Println(res)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("save s3 snapshot using CLI", func() { It("save s3 snapshot using CLI", func() {
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot save "+ res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot save " +
"--etcd-s3-insecure=true "+ "--etcd-s3-insecure=true " +
"--etcd-s3-bucket=test-bucket "+ "--etcd-s3-bucket=test-bucket " +
"--etcd-s3-folder=test-folder "+ "--etcd-s3-folder=test-folder " +
"--etcd-s3-endpoint=localhost:9090 "+ "--etcd-s3-endpoint=localhost:9090 " +
"--etcd-s3-skip-ssl-verify=true "+ "--etcd-s3-skip-ssl-verify=true " +
"--etcd-s3-access-key=test ", "--etcd-s3-access-key=test ")
serverNodeNames[0])
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("Snapshot on-demand-server-0")) Expect(res).To(ContainSubstring("Snapshot on-demand-server-0"))
}) })
It("creates s3 config secret", func() { It("creates s3 config secret", func() {
res, err := e2e.RunCmdOnNode("k3s kubectl create secret generic k3s-etcd-s3-config --namespace=kube-system "+ res, err := tc.Servers[0].RunCmdOnNode("k3s kubectl create secret generic k3s-etcd-s3-config --namespace=kube-system " +
"--from-literal=etcd-s3-insecure=true "+ "--from-literal=etcd-s3-insecure=true " +
"--from-literal=etcd-s3-bucket=test-bucket "+ "--from-literal=etcd-s3-bucket=test-bucket " +
"--from-literal=etcd-s3-folder=test-folder "+ "--from-literal=etcd-s3-folder=test-folder " +
"--from-literal=etcd-s3-endpoint=localhost:9090 "+ "--from-literal=etcd-s3-endpoint=localhost:9090 " +
"--from-literal=etcd-s3-skip-ssl-verify=true "+ "--from-literal=etcd-s3-skip-ssl-verify=true " +
"--from-literal=etcd-s3-access-key=test ", "--from-literal=etcd-s3-access-key=test ")
serverNodeNames[0])
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("secret/k3s-etcd-s3-config created")) Expect(res).To(ContainSubstring("secret/k3s-etcd-s3-config created"))
}) })
It("save s3 snapshot using secret", func() { It("save s3 snapshot using secret", func() {
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot save")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("Snapshot on-demand-server-0")) Expect(res).To(ContainSubstring("Snapshot on-demand-server-0"))
}) })
It("lists saved s3 snapshot", func() { It("lists saved s3 snapshot", func() {
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot list", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot list")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("file:///var/lib/rancher/k3s/server/db/snapshots/on-demand-server-0")) Expect(res).To(ContainSubstring("file:///var/lib/rancher/k3s/server/db/snapshots/on-demand-server-0"))
Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0")) Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0"))
}) })
It("save 3 more s3 snapshots", func() { It("save 3 more s3 snapshots", func() {
for _, i := range []string{"1", "2", "3"} { for _, i := range []string{"1", "2", "3"} {
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot save --name special-"+i, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot save --name special-" + i)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("Snapshot special-" + i + "-server-0")) Expect(res).To(ContainSubstring("Snapshot special-" + i + "-server-0"))
} }
}) })
It("lists saved s3 snapshot", func() { It("lists saved s3 snapshot", func() {
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot list", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot list")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0")) Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/on-demand-server-0"))
Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-1-server-0")) Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-1-server-0"))
...@@ -138,25 +121,25 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -138,25 +121,25 @@ var _ = Describe("Verify Create", Ordered, func() {
Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-3-server-0")) Expect(res).To(ContainSubstring("s3://test-bucket/test-folder/special-3-server-0"))
}) })
It("delete first on-demand s3 snapshot", func() { It("delete first on-demand s3 snapshot", func() {
_, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot ls >> ./snapshotname.txt", serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode("sudo k3s etcd-snapshot ls >> ./snapshotname.txt")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
snapshotName, err := e2e.RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt | head -1", serverNodeNames[0]) snapshotName, err := tc.Servers[0].RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt | head -1")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
res, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot delete "+snapshotName, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("sudo k3s etcd-snapshot delete " + snapshotName)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(ContainSubstring("Snapshot " + strings.TrimSpace(snapshotName) + " deleted")) Expect(res).To(ContainSubstring("Snapshot " + strings.TrimSpace(snapshotName) + " deleted"))
}) })
It("prunes s3 snapshots", func() { It("prunes s3 snapshots", func() {
_, err := e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot save")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
time.Sleep(time.Second) time.Sleep(time.Second)
_, err = e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0]) _, err = tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot save")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
time.Sleep(time.Second) time.Sleep(time.Second)
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot prune", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot prune")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// There should now be 4 on-demand snapshots - 2 local, and 2 on s3 // There should now be 4 on-demand snapshots - 2 local, and 2 on s3
res, err = e2e.RunCmdOnNode("k3s etcd-snapshot ls 2>/dev/null | grep on-demand | wc -l", serverNodeNames[0]) res, err = tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot ls 2>/dev/null | grep on-demand | wc -l")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(strings.TrimSpace(res)).To(Equal("4")) Expect(strings.TrimSpace(res)).To(Equal("4"))
}) })
...@@ -164,7 +147,7 @@ var _ = Describe("Verify Create", Ordered, func() { ...@@ -164,7 +147,7 @@ var _ = Describe("Verify Create", Ordered, func() {
// Wait until the retention works with 3 minutes // Wait until the retention works with 3 minutes
fmt.Printf("\nWaiting 3 minutes until retention works\n") fmt.Printf("\nWaiting 3 minutes until retention works\n")
time.Sleep(3 * time.Minute) time.Sleep(3 * time.Minute)
res, err := e2e.RunCmdOnNode("k3s etcd-snapshot ls 2>/dev/null | grep etcd-snapshot | wc -l", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s etcd-snapshot ls 2>/dev/null | grep etcd-snapshot | wc -l")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(strings.TrimSpace(res)).To(Equal("4")) Expect(strings.TrimSpace(res)).To(Equal("4"))
}) })
...@@ -178,12 +161,12 @@ var _ = AfterEach(func() { ...@@ -178,12 +161,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.SaveJournalLogs(append(tc.Servers, tc.Agents...))).To(Succeed())
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -2,11 +2,10 @@ package secretsencryption ...@@ -2,11 +2,10 @@ package secretsencryption
import ( import (
"flag" "flag"
"fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -32,10 +31,7 @@ func Test_E2ESecretsEncryption(t *testing.T) { ...@@ -32,10 +31,7 @@ func Test_E2ESecretsEncryption(t *testing.T) {
RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -44,53 +40,44 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -44,53 +40,44 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, _, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, 0) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, 0)
} else { } else {
serverNodeNames, _, err = e2e.CreateCluster(*nodeOS, *serverCount, 0) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, 0)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") tc.Hardened = *hardened
fmt.Println("OS:", *nodeOS) By("CLUSTER CONFIG")
fmt.Println("Server Nodes:", serverNodeNames) By("OS: " + *nodeOS)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0]) By(tc.Status())
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Checks node and pod status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true) e2e.ParseNodes(tc.KubeConfigFile, true)
fmt.Printf("\nFetching pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true) e2e.DumpPods(tc.KubeConfigFile)
}) })
It("Deploys several secrets", func() { It("Deploys several secrets", func() {
_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened) _, err := tc.DeployWorkload("secrets.yaml")
Expect(err).NotTo(HaveOccurred(), "Secrets not deployed") Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
}) })
It("Verifies encryption start stage", func() { It("Verifies encryption start stage", func() {
cmd := "k3s secrets-encrypt status" cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames { for _, node := range tc.Servers {
res, err := e2e.RunCmdOnNode(cmd, nodeName) res, err := node.RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
Expect(res).Should(ContainSubstring("Current Rotation Stage: start")) Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
...@@ -100,17 +87,12 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -100,17 +87,12 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
It("Rotates the Secrets-Encryption Keys", func() { It("Rotates the Secrets-Encryption Keys", func() {
cmd := "k3s secrets-encrypt rotate-keys" cmd := "k3s secrets-encrypt rotate-keys"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode(cmd)
// Before we fail on error, get the server logs which actually contain the error Expect(err).NotTo(HaveOccurred(), res)
var slogs string for i, node := range tc.Servers {
if err != nil {
slogs, _ = e2e.RunCmdOnNode("journalctl -u k3s -n 10", serverNodeNames[0])
}
Expect(err).NotTo(HaveOccurred(), res+slogs)
for i, nodeName := range serverNodeNames {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
cmd := "k3s secrets-encrypt status" cmd := "k3s secrets-encrypt status"
res, err := e2e.RunCmdOnNode(cmd, nodeName) res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred(), res) g.Expect(err).NotTo(HaveOccurred(), res)
g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match")) g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
if i == 0 { if i == 0 {
...@@ -123,14 +105,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -123,14 +105,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}) })
It("Restarts K3s servers", func() { It("Restarts K3s servers", func() {
Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed(), e2e.GetVagrantLog(nil)) Expect(e2e.RestartCluster(tc.Servers)).To(Succeed(), e2e.GetVagrantLog(nil))
}) })
It("Verifies reencryption_finished stage", func() { It("Verifies reencryption_finished stage", func() {
cmd := "k3s secrets-encrypt status" cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames { for _, node := range tc.Servers {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName) res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
...@@ -144,17 +126,17 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -144,17 +126,17 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
Context("Disabling Secrets-Encryption", func() { Context("Disabling Secrets-Encryption", func() {
It("Disables encryption", func() { It("Disables encryption", func() {
cmd := "k3s secrets-encrypt disable" cmd := "k3s secrets-encrypt disable"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), res) Expect(err).NotTo(HaveOccurred(), res)
cmd = "k3s secrets-encrypt status" cmd = "k3s secrets-encrypt status"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) return tc.Servers[0].RunCmdOnNode(cmd)
}, "240s", "10s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) }, "240s", "10s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
for i, nodeName := range serverNodeNames { for i, node := range tc.Servers {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName) res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred(), res) g.Expect(err).NotTo(HaveOccurred(), res)
if i == 0 { if i == 0 {
g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled")) g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
...@@ -166,14 +148,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -166,14 +148,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}) })
It("Restarts K3s servers", func() { It("Restarts K3s servers", func() {
Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) Expect(e2e.RestartCluster(tc.Servers)).To(Succeed())
}) })
It("Verifies encryption disabled on all nodes", func() { It("Verifies encryption disabled on all nodes", func() {
cmd := "k3s secrets-encrypt status" cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames { for _, node := range tc.Servers {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Disabled")) g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Disabled"))
}, "420s", "2s").Should(Succeed()) }, "420s", "2s").Should(Succeed())
} }
}) })
...@@ -183,17 +165,17 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -183,17 +165,17 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
Context("Enabling Secrets-Encryption", func() { Context("Enabling Secrets-Encryption", func() {
It("Enables encryption", func() { It("Enables encryption", func() {
cmd := "k3s secrets-encrypt enable" cmd := "k3s secrets-encrypt enable"
res, err := e2e.RunCmdOnNode(cmd, serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred(), res) Expect(err).NotTo(HaveOccurred(), res)
cmd = "k3s secrets-encrypt status" cmd = "k3s secrets-encrypt status"
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCmdOnNode(cmd, serverNodeNames[0]) return tc.Servers[0].RunCmdOnNode(cmd)
}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished")) }, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
for i, nodeName := range serverNodeNames { for i, node := range tc.Servers {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
res, err := e2e.RunCmdOnNode(cmd, nodeName) res, err := node.RunCmdOnNode(cmd)
g.Expect(err).NotTo(HaveOccurred(), res) g.Expect(err).NotTo(HaveOccurred(), res)
if i == 0 { if i == 0 {
g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled")) g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
...@@ -205,14 +187,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() { ...@@ -205,14 +187,14 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
}) })
It("Restarts K3s servers", func() { It("Restarts K3s servers", func() {
Expect(e2e.RestartCluster(serverNodeNames)).To(Succeed()) Expect(e2e.RestartCluster(tc.Servers)).To(Succeed())
}) })
It("Verifies encryption enabled on all nodes", func() { It("Verifies encryption enabled on all nodes", func() {
cmd := "k3s secrets-encrypt status" cmd := "k3s secrets-encrypt status"
for _, nodeName := range serverNodeNames { for _, node := range tc.Servers {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled")) g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Enabled"))
}, "420s", "2s").Should(Succeed()) }, "420s", "2s").Should(Succeed())
} }
}) })
...@@ -227,12 +209,12 @@ var _ = AfterEach(func() { ...@@ -227,12 +209,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames)) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, tc.Servers))
} else { } else {
Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) Expect(e2e.GetCoverageReport(tc.Servers)).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -25,11 +25,7 @@ func Test_E2ETailscale(t *testing.T) { ...@@ -25,11 +25,7 @@ func Test_E2ETailscale(t *testing.T) {
RunSpecs(t, "Tailscale Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Tailscale Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -38,61 +34,59 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() { ...@@ -38,61 +34,59 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
// Server node needs to be ready before we continue // Server node needs to be ready before we continue
It("Checks Node Status", func() { It("Checks Node Status", func() {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "300s", "5s").Should(Succeed()) }, "300s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(kubeConfigFile, true) _, err := e2e.ParseNodes(tc.KubeConfigFile, true)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Change agent's config", func() { It("Change agent's config", func() {
nodeIPs, _ := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, _ := e2e.GetNodeIPs(tc.KubeConfigFile)
cmd := fmt.Sprintf("sudo sed -i 's/TAILSCALEIP/%s/g' /etc/rancher/k3s/config.yaml", nodeIPs[0].IPv4) cmd := fmt.Sprintf("sudo sed -i 's/TAILSCALEIP/%s/g' /etc/rancher/k3s/config.yaml", nodeIPs[0].IPv4)
for _, agent := range agentNodeNames { for _, agent := range tc.Agents {
_, err := e2e.RunCmdOnNode(cmd, agent) _, err := agent.RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
}) })
It("Restart agents", func() { It("Restart agents", func() {
err := e2e.RestartCluster(agentNodeNames) err := e2e.RestartCluster(tc.Agents)
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
}) })
It("Checks Node Status", func() { It("Checks Node Status", func() {
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).To(Equal(*agentCount + *serverCount)) g.Expect(len(nodes)).To(Equal(*agentCount + *serverCount))
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "300s", "5s").Should(Succeed()) }, "300s", "5s").Should(Succeed())
_, err := e2e.ParseNodes(kubeConfigFile, true) _, err := e2e.ParseNodes(tc.KubeConfigFile, true)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
}) })
It("Verifies that server and agent have a tailscale IP as nodeIP", func() { It("Verifies that server and agent have a tailscale IP as nodeIP", func() {
nodeIPs, err := e2e.GetNodeIPs(kubeConfigFile) nodeIPs, err := e2e.GetNodeIPs(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
for _, node := range nodeIPs { for _, node := range nodeIPs {
Expect(node.IPv4).Should(ContainSubstring("100.")) Expect(node.IPv4).Should(ContainSubstring("100."))
...@@ -102,8 +96,8 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() { ...@@ -102,8 +96,8 @@ var _ = Describe("Verify Tailscale Configuration", Ordered, func() {
It("Verify routing is correct and uses tailscale0 interface for internode traffic", func() { It("Verify routing is correct and uses tailscale0 interface for internode traffic", func() {
// table 52 is the one configured by tailscale // table 52 is the one configured by tailscale
cmd := "ip route show table 52" cmd := "ip route show table 52"
for _, node := range append(serverNodeNames, agentNodeNames...) { for _, node := range append(tc.Servers, tc.Agents...) {
output, err := e2e.RunCmdOnNode(cmd, node) output, err := node.RunCmdOnNode(cmd)
fmt.Println(err) fmt.Println(err)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(output).Should(ContainSubstring("10.42.")) Expect(output).Should(ContainSubstring("10.42."))
...@@ -119,12 +113,12 @@ var _ = AfterEach(func() { ...@@ -119,12 +113,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -4,10 +4,10 @@ import ( ...@@ -4,10 +4,10 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strings"
"testing" "testing"
"time" "time"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -33,11 +33,7 @@ func Test_E2EToken(t *testing.T) { ...@@ -33,11 +33,7 @@ func Test_E2EToken(t *testing.T) {
RunSpecs(t, "SnapshotRestore Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "SnapshotRestore Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
...@@ -46,66 +42,54 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() ...@@ -46,66 +42,54 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
It("Checks Node and Pod Status", func() { It("Checks node and pod status", func() {
fmt.Printf("\nFetching node status\n") By("Fetching Nodes status")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "420s", "5s").Should(Succeed()) }, "420s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n") Eventually(func() error {
Eventually(func(g Gomega) { return tests.AllPodsUp(tc.KubeConfigFile)
pods, err := e2e.ParsePods(kubeConfigFile, false) }, "360s", "5s").Should(Succeed())
g.Expect(err).NotTo(HaveOccurred()) e2e.DumpPods(tc.KubeConfigFile)
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "420s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true)
}) })
var permToken string var permToken string
It("Creates a permanent agent token", func() { It("Creates a permanent agent token", func() {
permToken = "perage.s0xt4u0hl5guoyi6" permToken = "perage.s0xt4u0hl5guoyi6"
_, err := e2e.RunCmdOnNode("k3s token create --ttl=0 "+permToken, serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode("k3s token create --ttl=0 " + permToken)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
res, err := e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s token list")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(MatchRegexp(`perage\s+<forever>\s+<never>`)) Expect(res).To(MatchRegexp(`perage\s+<forever>\s+<never>`))
}) })
It("Joins an agent with the permanent token", func() { It("Joins an agent with the permanent token", func() {
cmd := fmt.Sprintf("echo 'token: %s' | sudo tee -a /etc/rancher/k3s/config.yaml > /dev/null", permToken) cmd := fmt.Sprintf("echo 'token: %s' | sudo tee -a /etc/rancher/k3s/config.yaml > /dev/null", permToken)
_, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0]) _, err := tc.Agents[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = e2e.RunCmdOnNode("systemctl start k3s-agent", agentNodeNames[0]) _, err = tc.Agents[0].RunCmdOnNode("systemctl start k3s-agent")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 1)) g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 1))
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
...@@ -114,38 +98,38 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() ...@@ -114,38 +98,38 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
}) })
Context("Agent joins with temporary token:", func() { Context("Agent joins with temporary token:", func() {
It("Creates a 20s agent token", func() { It("Creates a 20s agent token", func() {
_, err := e2e.RunCmdOnNode("k3s token create --ttl=20s 20sect.jxnpve6vg8dqm895", serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode("k3s token create --ttl=20s 20sect.jxnpve6vg8dqm895")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
res, err := e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s token list")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(MatchRegexp(`20sect\s+[0-9]{2}s`)) Expect(res).To(MatchRegexp(`20sect\s+[0-9]{2}s`))
}) })
It("Cleans up 20s token automatically", func() { It("Cleans up 20s token automatically", func() {
Eventually(func() (string, error) { Eventually(func() (string, error) {
return e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) return tc.Servers[0].RunCmdOnNode("k3s token list")
}, "25s", "5s").ShouldNot(ContainSubstring("20sect")) }, "25s", "5s").ShouldNot(ContainSubstring("20sect"))
}) })
var tempToken string var tempToken string
It("Creates a 10m agent token", func() { It("Creates a 10m agent token", func() {
tempToken = "10mint.ida18trbbk43szwk" tempToken = "10mint.ida18trbbk43szwk"
_, err := e2e.RunCmdOnNode("k3s token create --ttl=10m "+tempToken, serverNodeNames[0]) _, err := tc.Servers[0].RunCmdOnNode("k3s token create --ttl=10m " + tempToken)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
time.Sleep(2 * time.Second) time.Sleep(2 * time.Second)
res, err := e2e.RunCmdOnNode("k3s token list", serverNodeNames[0]) res, err := tc.Servers[0].RunCmdOnNode("k3s token list")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(res).To(MatchRegexp(`10mint\s+[0-9]m`)) Expect(res).To(MatchRegexp(`10mint\s+[0-9]m`))
}) })
It("Joins an agent with the 10m token", func() { It("Joins an agent with the 10m token", func() {
cmd := fmt.Sprintf("echo 'token: %s' | sudo tee -a /etc/rancher/k3s/config.yaml > /dev/null", tempToken) cmd := fmt.Sprintf("echo 'token: %s' | sudo tee -a /etc/rancher/k3s/config.yaml > /dev/null", tempToken)
_, err := e2e.RunCmdOnNode(cmd, agentNodeNames[1]) _, err := tc.Agents[1].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = e2e.RunCmdOnNode("systemctl start k3s-agent", agentNodeNames[1]) _, err = tc.Agents[1].RunCmdOnNode("systemctl start k3s-agent")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
...@@ -155,23 +139,23 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() ...@@ -155,23 +139,23 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
Context("Rotate server bootstrap token", func() { Context("Rotate server bootstrap token", func() {
serverToken := "1234" serverToken := "1234"
It("Creates a new server token", func() { It("Creates a new server token", func() {
Expect(e2e.RunCmdOnNode("k3s token rotate -t vagrant --new-token="+serverToken, serverNodeNames[0])). Expect(tc.Servers[0].RunCmdOnNode("k3s token rotate -t vagrant --new-token=" + serverToken)).
To(ContainSubstring("Token rotated, restart k3s nodes with new token")) To(ContainSubstring("Token rotated, restart k3s nodes with new token"))
}) })
It("Restarts servers with the new token", func() { It("Restarts servers with the new token", func() {
cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
for _, node := range serverNodeNames { for _, node := range tc.Servers {
_, err := e2e.RunCmdOnNode(cmd, node) _, err := node.RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
for _, node := range serverNodeNames { for _, node := range tc.Servers {
_, err := e2e.RunCmdOnNode("systemctl restart k3s", node) _, err := node.RunCmdOnNode("systemctl restart k3s")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
...@@ -179,15 +163,15 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func() ...@@ -179,15 +163,15 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
}) })
It("Rejoins an agent with the new server token", func() { It("Rejoins an agent with the new server token", func() {
cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken) cmd := fmt.Sprintf("sed -i 's/token:.*/token: %s/' /etc/rancher/k3s/config.yaml", serverToken)
_, err := e2e.RunCmdOnNode(cmd, agentNodeNames[0]) _, err := tc.Agents[0].RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
_, err = e2e.RunCmdOnNode("systemctl restart k3s-agent", agentNodeNames[0]) _, err = tc.Agents[0].RunCmdOnNode("systemctl restart k3s-agent")
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
g.Expect(len(nodes)).Should(Equal(len(serverNodeNames) + 2)) g.Expect(len(nodes)).Should(Equal(len(tc.Servers) + 2))
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
...@@ -203,12 +187,12 @@ var _ = AfterEach(func() { ...@@ -203,12 +187,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(tc.Servers, tc.Agents...)))
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -4,9 +4,9 @@ import ( ...@@ -4,9 +4,9 @@ import (
"flag" "flag"
"fmt" "fmt"
"os" "os"
"strings"
"testing" "testing"
"github.com/k3s-io/k3s/tests"
"github.com/k3s-io/k3s/tests/e2e" "github.com/k3s-io/k3s/tests/e2e"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -26,92 +26,75 @@ func Test_E2EWasm(t *testing.T) { ...@@ -26,92 +26,75 @@ func Test_E2EWasm(t *testing.T) {
RunSpecs(t, "Run WebAssenbly Workloads Test Suite", suiteConfig, reporterConfig) RunSpecs(t, "Run WebAssenbly Workloads Test Suite", suiteConfig, reporterConfig)
} }
var ( var tc *e2e.TestConfig
kubeConfigFile string
serverNodeNames []string
agentNodeNames []string
)
var _ = ReportAfterEach(e2e.GenReport) var _ = ReportAfterEach(e2e.GenReport)
var _ = Describe("Verify Can run Wasm workloads", Ordered, func() { var _ = Describe("Verify K3s can run Wasm workloads", Ordered, func() {
Context("Cluster comes up with Wasm configuration", func() {
It("Starts up with no issues", func() { It("Starts up with no issues", func() {
var err error var err error
if *local { if *local {
serverNodeNames, agentNodeNames, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, *agentCount)
} else { } else {
serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount) tc, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
} }
Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err)) Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
fmt.Println("CLUSTER CONFIG") By("CLUSTER CONFIG")
fmt.Println("OS:", *nodeOS) By("OS: " + *nodeOS)
fmt.Println("Server Nodes:", serverNodeNames) By(tc.Status())
fmt.Println("Agent Nodes:", agentNodeNames)
kubeConfigFile, err = e2e.GenKubeConfigFile(serverNodeNames[0])
Expect(err).NotTo(HaveOccurred())
}) })
// Server node needs to be ready before we continue It("Checks node and pod status", func() {
It("Checks Node and Pod Status", func() { By("Fetching Nodes status")
fmt.Printf("\nFetching node status\n")
Eventually(func(g Gomega) { Eventually(func(g Gomega) {
nodes, err := e2e.ParseNodes(kubeConfigFile, false) nodes, err := e2e.ParseNodes(tc.KubeConfigFile, false)
g.Expect(err).NotTo(HaveOccurred()) g.Expect(err).NotTo(HaveOccurred())
for _, node := range nodes { for _, node := range nodes {
g.Expect(node.Status).Should(Equal("Ready")) g.Expect(node.Status).Should(Equal("Ready"))
} }
}, "620s", "5s").Should(Succeed()) }, "620s", "5s").Should(Succeed())
_, _ = e2e.ParseNodes(kubeConfigFile, true)
fmt.Printf("\nFetching Pods status\n") By("Fetching pod status")
Eventually(func(g Gomega) { Eventually(func() error {
pods, err := e2e.ParsePods(kubeConfigFile, false) return tests.AllPodsUp(tc.KubeConfigFile)
g.Expect(err).NotTo(HaveOccurred()) }, "620s", "10s").Should(Succeed())
for _, pod := range pods {
if strings.Contains(pod.Name, "helm-install") {
g.Expect(pod.Status).Should(Equal("Completed"), pod.Name)
} else {
g.Expect(pod.Status).Should(Equal("Running"), pod.Name)
}
}
}, "620s", "5s").Should(Succeed())
_, _ = e2e.ParsePods(kubeConfigFile, true)
}) })
It("Verify wasm-related containerd shims are installed", func() { It("Verify wasm-related containerd shims are installed", func() {
expected_shims := []string{"containerd-shim-spin-v2", "containerd-shim-slight-v1"} expected_shims := []string{"containerd-shim-spin-v2", "containerd-shim-slight-v1"}
for _, node := range append(serverNodeNames, agentNodeNames...) { for _, node := range append(tc.Servers, tc.Agents...) {
for _, shim := range expected_shims { for _, shim := range expected_shims {
cmd := fmt.Sprintf("which %s", shim) cmd := fmt.Sprintf("which %s", shim)
_, err := e2e.RunCmdOnNode(cmd, node) _, err := node.RunCmdOnNode(cmd)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
} }
} }
}) })
})
Context("Verify Wasm workloads can run on the cluster", func() { Context("Verify Wasm workloads can run on the cluster", func() {
It("Deploy Wasm workloads", func() { It("Deploy Wasm workloads", func() {
out, err := e2e.DeployWorkload("wasm-workloads.yaml", kubeConfigFile, false) out, err := tc.DeployWorkload("wasm-workloads.yaml")
Expect(err).NotTo(HaveOccurred(), out) Expect(err).NotTo(HaveOccurred(), out)
}) })
It("Wait for slight Pod to be up and running", func() { It("Wait for slight Pod to be up and running", func() {
Eventually(func() (string, error) { Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l app=wasm-slight --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile cmd := "kubectl get pods -o=name -l app=wasm-slight --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "240s", "5s").Should(ContainSubstring("pod/wasm-slight")) }, "240s", "5s").Should(ContainSubstring("pod/wasm-slight"))
}) })
It("Wait for spin Pod to be up and running", func() { It("Wait for spin Pod to be up and running", func() {
Eventually(func() (string, error) { Eventually(func() (string, error) {
cmd := "kubectl get pods -o=name -l app=wasm-spin --field-selector=status.phase=Running --kubeconfig=" + kubeConfigFile cmd := "kubectl get pods -o=name -l app=wasm-spin --field-selector=status.phase=Running --kubeconfig=" + tc.KubeConfigFile
return e2e.RunCommand(cmd) return e2e.RunCommand(cmd)
}, "120s", "5s").Should(ContainSubstring("pod/wasm-spin")) }, "120s", "5s").Should(ContainSubstring("pod/wasm-spin"))
}) })
It("Interact with Wasm applications", func() { It("Interact with Wasm applications", func() {
ingressIPs, err := e2e.FetchIngressIP(kubeConfigFile) ingressIPs, err := e2e.FetchIngressIP(tc.KubeConfigFile)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(ingressIPs).To(HaveLen(1)) Expect(ingressIPs).To(HaveLen(1))
...@@ -136,12 +119,12 @@ var _ = AfterEach(func() { ...@@ -136,12 +119,12 @@ var _ = AfterEach(func() {
var _ = AfterSuite(func() { var _ = AfterSuite(func() {
if failed { if failed {
Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.SaveJournalLogs(append(tc.Servers, tc.Agents...))).To(Succeed())
} else { } else {
Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) Expect(e2e.GetCoverageReport(append(tc.Servers, tc.Agents...))).To(Succeed())
} }
if !failed || *ci { if !failed || *ci {
Expect(e2e.DestroyCluster()).To(Succeed()) Expect(e2e.DestroyCluster()).To(Succeed())
Expect(os.Remove(kubeConfigFile)).To(Succeed()) Expect(os.Remove(tc.KubeConfigFile)).To(Succeed())
} }
}) })
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -36,7 +37,7 @@ var _ = Describe("ca certificate rotation", Ordered, func() { ...@@ -36,7 +37,7 @@ var _ = Describe("ca certificate rotation", Ordered, func() {
When("a new server is created", func() { When("a new server is created", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
It("get certificate hash", func() { It("get certificate hash", func() {
...@@ -69,7 +70,7 @@ var _ = Describe("ca certificate rotation", Ordered, func() { ...@@ -69,7 +70,7 @@ var _ = Describe("ca certificate rotation", Ordered, func() {
}) })
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "360s", "5s").Should(Succeed()) }, "360s", "5s").Should(Succeed())
}) })
It("get certificate hash", func() { It("get certificate hash", func() {
......
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -35,7 +36,7 @@ var _ = Describe("certificate rotation", Ordered, func() { ...@@ -35,7 +36,7 @@ var _ = Describe("certificate rotation", Ordered, func() {
When("a new server is created", func() { When("a new server is created", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
It("get certificate hash", func() { It("get certificate hash", func() {
...@@ -61,7 +62,7 @@ var _ = Describe("certificate rotation", Ordered, func() { ...@@ -61,7 +62,7 @@ var _ = Describe("certificate rotation", Ordered, func() {
}) })
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "360s", "5s").Should(Succeed()) }, "360s", "5s").Should(Succeed())
}) })
It("checks the certificate status", func() { It("checks the certificate status", func() {
......
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -40,7 +41,7 @@ var _ = Describe("dual stack", Ordered, func() { ...@@ -40,7 +41,7 @@ var _ = Describe("dual stack", Ordered, func() {
When("a ipv4 and ipv6 cidr is present", func() { When("a ipv4 and ipv6 cidr is present", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "10s").Should(Succeed()) }, "180s", "10s").Should(Succeed())
}) })
It("creates pods with two IPs", func() { It("creates pods with two IPs", func() {
......
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -33,7 +34,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { ...@@ -33,7 +34,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
When("a snapshot is restored on existing node", func() { When("a snapshot is restored on existing node", func() {
It("etcd starts up with no problems", func() { It("etcd starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
It("create a workload", func() { It("create a workload", func() {
...@@ -85,7 +86,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() { ...@@ -85,7 +86,7 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
}) })
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "360s", "5s").Should(Succeed()) }, "360s", "5s").Should(Succeed())
}) })
It("make sure workload 1 exists", func() { It("make sure workload 1 exists", func() {
......
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"testing" "testing"
"time" "time"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -40,7 +41,7 @@ var _ = Describe("etcd snapshots", Ordered, func() { ...@@ -40,7 +41,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
When("a new etcd is created", func() { When("a new etcd is created", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "10s").Should(Succeed()) }, "180s", "10s").Should(Succeed())
}) })
It("saves an etcd snapshot", func() { It("saves an etcd snapshot", func() {
...@@ -130,7 +131,7 @@ var _ = Describe("etcd snapshots", Ordered, func() { ...@@ -130,7 +131,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
server, err = testutil.K3sStartServer(localServerArgs...) server, err = testutil.K3sStartServer(localServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
...@@ -156,7 +157,7 @@ var _ = Describe("etcd snapshots", Ordered, func() { ...@@ -156,7 +157,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
server, err = testutil.K3sStartServer(localServerArgs...) server, err = testutil.K3sStartServer(localServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
......
...@@ -5,6 +5,7 @@ import ( ...@@ -5,6 +5,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -41,7 +42,7 @@ var _ = Describe("flannel-ipv6-masq", Ordered, func() { ...@@ -41,7 +42,7 @@ var _ = Describe("flannel-ipv6-masq", Ordered, func() {
When("a ipv4 and ipv6 cidr is present", func() { When("a ipv4 and ipv6 cidr is present", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "10s").Should(Succeed()) }, "180s", "10s").Should(Succeed())
}) })
It("creates pods with two IPs", func() { It("creates pods with two IPs", func() {
......
...@@ -14,8 +14,6 @@ import ( ...@@ -14,8 +14,6 @@ import (
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
var server *testutil.K3sServer var server *testutil.K3sServer
...@@ -44,16 +42,15 @@ var _ = Describe("flannel-backend=none", Ordered, func() { ...@@ -44,16 +42,15 @@ var _ = Describe("flannel-backend=none", Ordered, func() {
It("checks pods status", func() { It("checks pods status", func() {
// Wait for pods to come up before running the real test // Wait for pods to come up before running the real test
Eventually(func() int { Eventually(func() int {
pods, _ := testutil.ParsePods("kube-system", metav1.ListOptions{}) pods, _ := testutil.ParsePodsInNS("kube-system")
return len(pods) return len(pods)
}, "180s", "5s").Should(BeNumerically(">", 0)) }, "180s", "5s").Should(BeNumerically(">", 0))
pods, err := testutil.ParsePodsInNS("kube-system")
pods, err := testutil.ParsePods("kube-system", metav1.ListOptions{})
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
// Pods should remain in Pending state because there is no network plugin // Pods should remain in Pending state because there is no network plugin
Consistently(func () bool { Consistently(func() bool {
for _, pod := range pods { for _, pod := range pods {
if !strings.Contains(string(pod.Status.Phase), "Pending") { if !strings.Contains(string(pod.Status.Phase), "Pending") {
return false return false
......
...@@ -15,20 +15,20 @@ import ( ...@@ -15,20 +15,20 @@ import (
"time" "time"
"github.com/k3s-io/k3s/pkg/flock" "github.com/k3s-io/k3s/pkg/flock"
"github.com/k3s-io/k3s/tests"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/vishvananda/netlink" "github.com/vishvananda/netlink"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/clientcmd"
) )
// Compile-time variable // Compile-time variable
var existingServer = "False" var existingServer = "False"
const lockFile = "/tmp/k3s-test.lock" const lockFile = "/tmp/k3s-test.lock"
const DefaultConfig = "/etc/rancher/k3s/k3s.yaml"
type K3sServer struct { type K3sServer struct {
cmd *exec.Cmd cmd *exec.Cmd
...@@ -128,60 +128,8 @@ func K3sServerArgs() []string { ...@@ -128,60 +128,8 @@ func K3sServerArgs() []string {
return args return args
} }
// K3sDefaultDeployments checks if the default deployments for K3s are ready, otherwise returns an error
func K3sDefaultDeployments() error {
return CheckDeployments(metav1.NamespaceSystem, []string{"coredns", "local-path-provisioner", "metrics-server", "traefik"})
}
// CheckDeployments checks if the provided list of deployments are ready, otherwise returns an error
func CheckDeployments(namespace string, deployments []string) error {
client, err := k8sClient()
if err != nil {
return err
}
for _, deploymentName := range deployments {
deployment, err := client.AppsV1().Deployments(namespace).Get(context.Background(), deploymentName, metav1.GetOptions{})
if err != nil {
return err
}
if deployment.Status.ReadyReplicas != deployment.Status.Replicas || deployment.Status.AvailableReplicas != deployment.Status.Replicas {
return fmt.Errorf("deployment %s not ready: replicas=%d readyReplicas=%d availableReplicas=%d",
deploymentName, deployment.Status.Replicas, deployment.Status.ReadyReplicas, deployment.Status.AvailableReplicas)
}
}
return nil
}
func ParsePods(namespace string, opts metav1.ListOptions) ([]corev1.Pod, error) {
clientSet, err := k8sClient()
if err != nil {
return nil, err
}
pods, err := clientSet.CoreV1().Pods(namespace).List(context.Background(), opts)
if err != nil {
return nil, err
}
return pods.Items, nil
}
func ParseNodes() ([]corev1.Node, error) {
clientSet, err := k8sClient()
if err != nil {
return nil, err
}
nodes, err := clientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
if err != nil {
return nil, err
}
return nodes.Items, nil
}
func GetPod(namespace, name string) (*corev1.Pod, error) { func GetPod(namespace, name string) (*corev1.Pod, error) {
client, err := k8sClient() client, err := tests.K8sClient(DefaultConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -189,7 +137,7 @@ func GetPod(namespace, name string) (*corev1.Pod, error) { ...@@ -189,7 +137,7 @@ func GetPod(namespace, name string) (*corev1.Pod, error) {
} }
func GetPersistentVolumeClaim(namespace, name string) (*corev1.PersistentVolumeClaim, error) { func GetPersistentVolumeClaim(namespace, name string) (*corev1.PersistentVolumeClaim, error) {
client, err := k8sClient() client, err := tests.K8sClient(DefaultConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -197,7 +145,7 @@ func GetPersistentVolumeClaim(namespace, name string) (*corev1.PersistentVolumeC ...@@ -197,7 +145,7 @@ func GetPersistentVolumeClaim(namespace, name string) (*corev1.PersistentVolumeC
} }
func GetPersistentVolume(name string) (*corev1.PersistentVolume, error) { func GetPersistentVolume(name string) (*corev1.PersistentVolume, error) {
client, err := k8sClient() client, err := tests.K8sClient(DefaultConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -355,7 +303,7 @@ func K3sSaveLog(server *K3sServer, dump bool) error { ...@@ -355,7 +303,7 @@ func K3sSaveLog(server *K3sServer, dump bool) error {
} }
func GetEndpointsAddresses() (string, error) { func GetEndpointsAddresses() (string, error) {
client, err := k8sClient() client, err := tests.K8sClient(DefaultConfig)
if err != nil { if err != nil {
return "", err return "", err
} }
...@@ -414,14 +362,15 @@ func unmountFolder(folder string) error { ...@@ -414,14 +362,15 @@ func unmountFolder(folder string) error {
return nil return nil
} }
func k8sClient() (*kubernetes.Clientset, error) { func ParsePodsInNS(namespace string) ([]corev1.Pod, error) {
config, err := clientcmd.BuildConfigFromFlags("", "/etc/rancher/k3s/k3s.yaml") clientSet, err := tests.K8sClient(DefaultConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
clientSet, err := kubernetes.NewForConfig(config) pods, err := clientSet.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{})
if err != nil { if err != nil {
return nil, err return nil, err
} }
return clientSet, nil
return pods.Items, nil
} }
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -126,7 +127,7 @@ var _ = Describe("create a new cluster with kube-* flags", Ordered, func() { ...@@ -126,7 +127,7 @@ var _ = Describe("create a new cluster with kube-* flags", Ordered, func() {
// Pods should not be healthy without kube-proxy // Pods should not be healthy without kube-proxy
Consistently(func() error { Consistently(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "100s", "5s").Should(HaveOccurred()) }, "100s", "5s").Should(HaveOccurred())
}) })
It("should not find kube-proxy starting", func() { It("should not find kube-proxy starting", func() {
...@@ -178,7 +179,7 @@ var _ = Describe("create a new cluster with kube-* flags", Ordered, func() { ...@@ -178,7 +179,7 @@ var _ = Describe("create a new cluster with kube-* flags", Ordered, func() {
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
......
...@@ -7,6 +7,7 @@ import ( ...@@ -7,6 +7,7 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -35,7 +36,7 @@ var _ = Describe("local storage", Ordered, func() { ...@@ -35,7 +36,7 @@ var _ = Describe("local storage", Ordered, func() {
When("a new local storage is created", func() { When("a new local storage is created", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
It("creates a new pvc", func() { It("creates a new pvc", func() {
......
...@@ -6,10 +6,10 @@ import ( ...@@ -6,10 +6,10 @@ import (
"strings" "strings"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
var server *testutil.K3sServer var server *testutil.K3sServer
...@@ -38,7 +38,7 @@ var _ = Describe("longhorn", Ordered, func() { ...@@ -38,7 +38,7 @@ var _ = Describe("longhorn", Ordered, func() {
When("a new cluster is created", func() { When("a new cluster is created", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
}) })
...@@ -57,7 +57,7 @@ var _ = Describe("longhorn", Ordered, func() { ...@@ -57,7 +57,7 @@ var _ = Describe("longhorn", Ordered, func() {
}) })
It("starts the longhorn pods with no problems", func() { It("starts the longhorn pods with no problems", func() {
Eventually(func() error { Eventually(func() error {
pods, err := testutil.ParsePods("longhorn-system", metav1.ListOptions{}) pods, err := testutil.ParsePodsInNS("longhorn-system")
if err != nil { if err != nil {
return err return err
} }
......
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"testing" "testing"
"time" "time"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -35,7 +36,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { ...@@ -35,7 +36,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
When("A server starts with secrets encryption", func() { When("A server starts with secrets encryption", func() {
It("starts up with no problems", func() { It("starts up with no problems", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
It("it creates a encryption key", func() { It("it creates a encryption key", func() {
...@@ -66,7 +67,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { ...@@ -66,7 +67,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
Eventually(func() (string, error) { Eventually(func() (string, error) {
return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir) return testutil.K3sCmd("secrets-encrypt status -d", secretsEncryptionDataDir)
...@@ -91,7 +92,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { ...@@ -91,7 +92,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
Eventually(func() (string, error) { Eventually(func() (string, error) {
...@@ -128,7 +129,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() { ...@@ -128,7 +129,7 @@ var _ = Describe("secrets encryption rotation", Ordered, func() {
secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...) secretsEncryptionServer, err = testutil.K3sStartServer(secretsEncryptionServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
time.Sleep(10 * time.Second) time.Sleep(10 * time.Second)
}) })
......
...@@ -6,6 +6,7 @@ import ( ...@@ -6,6 +6,7 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
tests "github.com/k3s-io/k3s/tests"
testutil "github.com/k3s-io/k3s/tests/integration" testutil "github.com/k3s-io/k3s/tests/integration"
. "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
...@@ -37,7 +38,7 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -37,7 +38,7 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
It("has kine without tls", func() { It("has kine without tls", func() {
...@@ -78,7 +79,7 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -78,7 +79,7 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
It("set kine to use tls", func() { It("set kine to use tls", func() {
...@@ -107,7 +108,7 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -107,7 +108,7 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
It("dies cleanly", func() { It("dies cleanly", func() {
...@@ -124,9 +125,9 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -124,9 +125,9 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods without traefik deployed", func() { It("has the default pods without traefik deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.CheckDeployments("kube-system", []string{"coredns", "local-path-provisioner", "metrics-server"}) return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server"}, testutil.DefaultConfig)
}, "90s", "10s").Should(Succeed()) }, "90s", "10s").Should(Succeed())
nodes, err := testutil.ParseNodes() nodes, err := tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
}) })
...@@ -156,10 +157,10 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -156,10 +157,10 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the node deployed with correct IPs", func() { It("has the node deployed with correct IPs", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "10s").Should(Succeed()) }, "120s", "10s").Should(Succeed())
nodes, err := testutil.ParseNodes() nodes, err := tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{ Expect(nodes[0].Status.Addresses).To(ContainElements([]v1.NodeAddress{
...@@ -201,9 +202,9 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -201,9 +202,9 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
nodes, err := testutil.ParseNodes() nodes, err := tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
}) })
...@@ -229,16 +230,16 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -229,16 +230,16 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
nodes, err := testutil.ParseNodes() nodes, err := tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
}) })
var nodes []v1.Node var nodes []v1.Node
It("has a custom node name with id appended", func() { It("has a custom node name with id appended", func() {
var err error var err error
nodes, err = testutil.ParseNodes() nodes, err = tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
Expect(nodes[0].Name).To(MatchRegexp(`-[0-9a-f]*`)) Expect(nodes[0].Name).To(MatchRegexp(`-[0-9a-f]*`))
...@@ -264,9 +265,9 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -264,9 +265,9 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
nodes, err := testutil.ParseNodes() nodes, err := tests.ParseNodes(testutil.DefaultConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
Expect(nodes).To(HaveLen(1)) Expect(nodes).To(HaveLen(1))
}) })
...@@ -285,7 +286,7 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -285,7 +286,7 @@ var _ = Describe("startup tests", Ordered, func() {
}) })
It("has the default pods deployed", func() { It("has the default pods deployed", func() {
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "120s", "5s").Should(Succeed()) }, "120s", "5s").Should(Succeed())
}) })
It("creates a new pod", func() { It("creates a new pod", func() {
...@@ -301,7 +302,7 @@ var _ = Describe("startup tests", Ordered, func() { ...@@ -301,7 +302,7 @@ var _ = Describe("startup tests", Ordered, func() {
startupServer, err = testutil.K3sStartServer(startupServerArgs...) startupServer, err = testutil.K3sStartServer(startupServerArgs...)
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
Eventually(func() error { Eventually(func() error {
return testutil.K3sDefaultDeployments() return tests.CheckDefaultDeployments(testutil.DefaultConfig)
}, "180s", "5s").Should(Succeed()) }, "180s", "5s").Should(Succeed())
}) })
It("has the dummy pod not restarted", func() { It("has the dummy pod not restarted", func() {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment