Migrate hardened docker test

.github/workflows/e2e.yaml

@@ -160,7 +160,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [basics, bootstraptoken, cacerts, etcd, lazypull, skew, snapshotrestore, token, upgrade]
+        dtest: [basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade]
         arch: [amd64, arm64]
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
     env:
@@ -202,7 +202,7 @@ jobs:
         cd ./tests/docker/${{ matrix.dtest }}
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
-        elif [ ${{ matrix.dtest }} = "snapshotrestore" ]; then
+        elif [ ${{ matrix.dtest }} = "snapshotrestore" ] || [ ${{ matrix.dtest }} = "hardened" ]; then
           ./${{ matrix.dtest }}.test -ci 
         else
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE

tests/docker/hardened/cluster-level-pss.yaml

+apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+- name: PodSecurity
+  configuration:
+    apiVersion: pod-security.admission.config.k8s.io/v1beta1
+    kind: PodSecurityConfiguration
+    defaults:
+      enforce: "privileged"
+      enforce-version: "latest"
+      audit: "baseline"
+      audit-version: "latest"
+      warn: "baseline"
+      warn-version: "latest"
+    exemptions:
+      usernames: []
+      runtimeClasses: []
+      namespaces: [kube-system]

tests/docker/hardened/hardened_test.go

+package main
+
+import (
+	"flag"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var config *docker.TestConfig
+var ci = flag.Bool("ci", false, "running on CI")
+
+func Test_DockerHardened(t *testing.T) {
+	flag.Parse()
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Hardened Docker Test Suite")
+}
+
+var _ = Describe("Hardened Tests", Ordered, func() {
+
+	Context("Setup Cluster", func() {
+		It("should provision servers and agents", func() {
+			var err error
+			config, err = docker.NewTestConfig("rancher/systemd-node")
+			Expect(err).NotTo(HaveOccurred())
+			config.ServerYaml = `
+secrets-encryption: true
+kube-controller-manager-arg:
+  - 'terminated-pod-gc-threshold=10'
+kubelet-arg:
+  - 'streaming-connection-idle-timeout=5m'
+  - 'make-iptables-util-chains=true'
+  - 'event-qps=0'
+  - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
+kube-apiserver-arg:
+  - 'admission-control-config-file=/tmp/cluster-level-pss.yaml'
+  - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
+  - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
+  - 'audit-log-maxage=30'
+  - 'audit-log-maxbackup=10'
+  - 'audit-log-maxsize=100'
+`
+			config.AgentYaml = `
+kubelet-arg:
+  - 'streaming-connection-idle-timeout=5m'
+  - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
+`
+			config.SkipStart = true
+			Expect(config.ProvisionServers(1)).To(Succeed())
+
+			for _, server := range config.Servers {
+				cmd := "docker cp ./cluster-level-pss.yaml " + server.Name + ":/tmp/cluster-level-pss.yaml"
+				Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
+
+				cmd = "mkdir -p /var/lib/rancher/k3s/server/logs"
+				Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
+				auditYaml := "apiVersion: audit.k8s.io/v1\nkind: Policy\nrules:\n- level: Metadata"
+				cmd = fmt.Sprintf("echo -e '%s' > /var/lib/rancher/k3s/server/audit.yaml", auditYaml)
+				Expect(server.RunCmdOnNode(cmd)).Error().NotTo(HaveOccurred())
+				Expect(server.RunCmdOnNode("systemctl start k3s")).Error().NotTo(HaveOccurred())
+			}
+			Expect(config.CopyAndModifyKubeconfig()).To(Succeed())
+			config.SkipStart = false
+			Expect(config.ProvisionAgents(1)).To(Succeed())
+			Eventually(func() error {
+				return tests.CheckDeployments([]string{"coredns", "local-path-provisioner", "metrics-server", "traefik"}, config.KubeconfigFile)
+			}, "60s", "5s").Should(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(config.KubeconfigFile, config.GetNodeNames())
+			}, "40s", "5s").Should(Succeed())
+		})
+	})
+
+	Context("Verify Network Policies", func() {
+		It("applies network policies", func() {
+			_, err := config.DeployWorkload("hardened-ingress.yaml")
+			Expect(err).NotTo(HaveOccurred())
+			Eventually(func() (string, error) {
+				cmd := "kubectl get daemonset -n default example -o jsonpath='{.status.numberReady}' --kubeconfig=" + config.KubeconfigFile
+				return docker.RunCommand(cmd)
+			}, "60s", "5s").Should(Equal("2"))
+			_, err = config.DeployWorkload("hardened-netpool.yaml")
+		})
+		It("checks ingress connections", func() {
+			for _, scheme := range []string{"http", "https"} {
+				for _, server := range config.Servers {
+					cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, server.IP)
+					Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
+				}
+				for _, agent := range config.Agents {
+					cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' %s://%s/", scheme, agent.IP)
+					Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
+				}
+			}
+		})
+		It("confirms we can make a request through the nodeport service", func() {
+			for _, server := range config.Servers {
+				cmd := "kubectl get service/example -o 'jsonpath={.spec.ports[*].nodePort}' --kubeconfig=" + config.KubeconfigFile
+				ports, err := docker.RunCommand(cmd)
+				Expect(err).NotTo(HaveOccurred())
+				for _, port := range strings.Split(ports, " ") {
+					cmd := fmt.Sprintf("curl -vksf -H 'Host: example.com' http://%s:%s", server.IP, port)
+					Expect(docker.RunCommand(cmd)).Error().NotTo(HaveOccurred())
+				}
+			}
+		})
+	})
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if *ci || (config != nil && !failed) {
+		Expect(config.Cleanup()).To(Succeed())
+	}
+})

tests/docker/resources/hardened-ingress.yaml

+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: example
+  namespace: default
+  labels:
+    app.kubernetes.io: example
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: example
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: example
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 405
+        runAsGroup: 100
+      containers:
+      - name: socat
+        image: docker.io/alpine/socat:1.7.4.3-r1
+        args:
+        - "TCP-LISTEN:8080,reuseaddr,fork"
+        - "EXEC:echo -e 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n$(NODE_IP) $(POD_NAMESPACE)/$(POD_NAME)\r\n'"
+        ports:
+        - containerPort: 8080
+          name: http
+        env:
+        - name: NODE_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        readinessProbe:
+          initialDelaySeconds: 2
+          periodSeconds: 10
+          httpGet:
+            path: /
+            port: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: example
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app.kubernetes.io/name: example
+  ports:
+  - name: http
+    protocol: TCP
+    port: 80
+    nodePort: 30096
+    targetPort: http
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: example
+spec:
+  rules:
+  - host: "example.com"
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: example
+            port:
+              name: http
+---
+# Allow access to example backend from traefik ingress
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: ingress-to-backend-example
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: example
+  ingress:
+  - ports:
+    - port: 8080
+      protocol: TCP
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: kube-system
+      podSelector:
+        matchLabels:
+          app.kubernetes.io/name: traefik
+  policyTypes:
+  - Ingress
+---
+# Allow access to example backend from outside the cluster via nodeport service
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: nodeport-to-backend-example
+  namespace: default
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: example
+  ingress:
+  - ports:
+    - port: 8080
+      protocol: TCP
+  - from:
+    - ipBlock:
+        cidr: 0.0.0.0/0
+        except:
+          - 10.42.0.0/16
+          - 10.43.0.0/16
+  policyTypes:
+  - Ingress

tests/docker/resources/hardened-netpol.yaml

+---
+# Allow all traffic within the kube-system namespace; block all other access
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: kube-system
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: kube-system
+  policyTypes:
+  - Ingress
+---
+# Allow all traffic within the default namespace; block all other access
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: default
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: default
+  policyTypes:
+  - Ingress
+---
+# Allow traffic within the kube-public namespace; block all other access
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: intra-namespace
+  namespace: kube-public
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+      - namespaceSelector:
+          matchLabels:
+            name: kube-public
+  policyTypes:
+  - Ingress
+---
+# Allow all access to metrics-server
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-metrics-server
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      k8s-app: metrics-server
+  ingress:
+  - {}
+  policyTypes:
+  - Ingress
+---
+# Allow all access to coredns DNS ports
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: default-network-dns-policy
+  namespace: kube-system
+spec:
+  ingress:
+  - ports:
+    - port: 53
+      protocol: TCP
+    - port: 53
+      protocol: UDP
+  podSelector:
+    matchLabels:
+      k8s-app: kube-dns
+  policyTypes:
+  - Ingress
+---
+# Allow all access to the servicelb traefik HTTP/HTTPS ports
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-svclb-traefik
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      svccontroller.k3s.cattle.io/svcname: traefik
+  ingress:
+  - ports:
+    - port: 80
+      protocol: TCP
+    - port: 443
+      protocol: TCP
+  policyTypes:
+  - Ingress
+---
+# Allow all access to traefik HTTP/HTTPS ports
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-all-traefik
+  namespace: kube-system
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/name: traefik
+  ingress:
+  - ports:
+    - port: 8000
+      protocol: TCP
+    - port: 8443
+      protocol: TCP
+  policyTypes:
+  - Ingress

tests/docker/test-helpers.go

@@ -177,14 +177,13 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 			}
 			// Write the raw YAML directly to the config.yaml on the systemd-node container
 			if config.ServerYaml != "" {
-				cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml)
+				cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.ServerYaml)
 				if out, err := newServer.RunCmdOnNode(cmd); err != nil {
 					return fmt.Errorf("failed to write server yaml: %s: %v", out, err)
 				}
 			}
 
-			// The pipe requires that we use sh -c with "" to run the command
-			cmd = fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
+			cmd = fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='%s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -",
 				dbConnect+" "+joinServer+" "+os.Getenv(fmt.Sprintf("SERVER_%d_ARGS", i)), skipStart)
 			if _, err := newServer.RunCmdOnNode(cmd); err != nil {
 				// Attempt to dump the last few lines of the journalctl logs
@@ -247,7 +246,7 @@ func (config *TestConfig) ProvisionServers(numOfServers int) error {
 	}
 	// Wait for kubeconfig to be available
 	time.Sleep(5 * time.Second)
-	return CopyAndModifyKubeconfig(config)
+	return config.CopyAndModifyKubeconfig()
 }
 
 // setupDatabase will start the configured database if startDB is true,
@@ -333,14 +332,13 @@ func (config *TestConfig) ProvisionAgents(numOfAgents int) error {
 				}
 				// Write the raw YAML directly to the config.yaml on the systemd-node container
 				if config.AgentYaml != "" {
-					cmd = fmt.Sprintf("echo \"%s\" > /etc/rancher/k3s/config.yaml", config.ServerYaml)
+					cmd = fmt.Sprintf("echo '%s' > /etc/rancher/k3s/config.yaml", config.AgentYaml)
 					if out, err := newAgent.RunCmdOnNode(cmd); err != nil {
 						return fmt.Errorf("failed to write server yaml: %s: %v", out, err)
 					}
 				}
 
-				// The pipe requires that we use sh -c with "" to run the command
-				sCmd := fmt.Sprintf("/bin/sh -c \"curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -\"",
+				sCmd := fmt.Sprintf("curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='agent %s' %s INSTALL_K3S_SKIP_DOWNLOAD=true sh -",
 					os.Getenv(agentInstanceArgs), skipStart)
 				if _, err := newAgent.RunCmdOnNode(sCmd); err != nil {
 					// Attempt to dump the last few lines of the journalctl logs
@@ -470,7 +468,7 @@ func (config *TestConfig) Cleanup() error {
 
 // CopyAndModifyKubeconfig copies out kubeconfig from first control-plane server
 // and updates the port to match the external port
-func CopyAndModifyKubeconfig(config *TestConfig) error {
+func (config *TestConfig) CopyAndModifyKubeconfig() error {
 	if len(config.Servers) == 0 {
 		return fmt.Errorf("no servers available to copy kubeconfig")
 	}
@@ -500,7 +498,7 @@ func CopyAndModifyKubeconfig(config *TestConfig) error {
 
 // RunCmdOnNode runs a command on a docker container
 func (node DockerNode) RunCmdOnNode(cmd string) (string, error) {
-	dCmd := fmt.Sprintf("docker exec %s %s", node.Name, cmd)
+	dCmd := fmt.Sprintf("docker exec %s /bin/sh -c \"%s\"", node.Name, cmd)
 	out, err := RunCommand(dCmd)
 	if err != nil {
 		return out, fmt.Errorf("%v: on node %s: %s", err, node.Name, out)

tests/docker/token/token_test.go

@@ -57,8 +57,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 			Expect(res).To(MatchRegexp(`perage\s+<forever>\s+<never>`))
 		})
 		It("Joins an agent with the permanent token", func() {
-			// wrap in bash to handle pipe
-			cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", permToken)
+			cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", permToken)
 			_, err := tc.Agents[0].RunCmdOnNode(cmd)
 			Expect(err).NotTo(HaveOccurred())
 			_, err = tc.Agents[0].RunCmdOnNode("systemctl start k3s-agent")
@@ -95,7 +94,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 			Expect(res).To(MatchRegexp(`10mint\s+[0-9]m`))
 		})
 		It("Joins an agent with the 10m token", func() {
-			cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", tempToken)
+			cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", tempToken)
 			_, err := tc.Agents[1].RunCmdOnNode(cmd)
 			Expect(err).NotTo(HaveOccurred())
 			_, err = tc.Agents[1].RunCmdOnNode("systemctl start k3s-agent")
@@ -117,7 +116,7 @@ var _ = Describe("Use the token CLI to create and join agents", Ordered, func()
 				To(ContainSubstring("Token rotated, restart k3s nodes with new token"))
 		})
 		It("Restarts servers with the new token", func() {
-			cmd := fmt.Sprintf("/bin/bash -c \"echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null\"", serverToken)
+			cmd := fmt.Sprintf("echo 'token: %s' | tee -a /etc/rancher/k3s/config.yaml > /dev/null", serverToken)
 			for _, node := range tc.Servers {
 				_, err := node.RunCmdOnNode(cmd)
 				Expect(err).NotTo(HaveOccurred())