CLI + Backend for Secrets Encryption v3

cmd/encrypt/main.go

@@ -22,6 +22,7 @@ func main() {
 			secretsencrypt.Prepare,
 			secretsencrypt.Rotate,
 			secretsencrypt.Reencrypt,
+			secretsencrypt.RotateKeys,
 		),
 	}
 

cmd/k3s/main.go

@@ -71,6 +71,7 @@ func main() {
 			secretsencryptCommand,
 			secretsencryptCommand,
 			secretsencryptCommand,
+			secretsencryptCommand,
 		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(

cmd/server/main.go

@@ -68,6 +68,7 @@ func main() {
 			secretsencrypt.Prepare,
 			secretsencrypt.Rotate,
 			secretsencrypt.Reencrypt,
+			secretsencrypt.RotateKeys,
 		),
 		cmds.NewCertCommand(
 			cmds.NewCertSubcommands(

pkg/cli/cmds/secrets_encrypt.go

@@ -26,7 +26,7 @@ var (
 	}
 )
 
-func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencrypt func(ctx *cli.Context) error) cli.Command {
+func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencrypt, rotateKeys func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:           SecretsEncryptCommand,
 		Usage:          "Control secrets encryption and keys rotation",
@@ -84,6 +84,13 @@ func NewSecretsEncryptCommands(status, enable, disable, prepare, rotate, reencry
 						Destination: &ServerConfig.EncryptSkip,
 					}),
 			},
+			{
+				Name:           "rotate-keys",
+				Usage:          "Add, rotate and rencryption with a new encryption key",
+				SkipArgReorder: true,
+				Action:         rotateKeys,
+				Flags:          EncryptFlags,
+			},
 		},
 	}
 }

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -156,7 +156,7 @@ func Prepare(app *cli.Context) error {
 		return err
 	}
 	b, err := json.Marshal(server.EncryptionRequest{
-		Stage: pointer.StringPtr(secretsencrypt.EncryptionPrepare),
+		Stage: pointer.String(secretsencrypt.EncryptionPrepare),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
 	if err != nil {
@@ -178,7 +178,7 @@ func Rotate(app *cli.Context) error {
 		return err
 	}
 	b, err := json.Marshal(server.EncryptionRequest{
-		Stage: pointer.StringPtr(secretsencrypt.EncryptionRotate),
+		Stage: pointer.String(secretsencrypt.EncryptionRotate),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
 	if err != nil {
@@ -201,7 +201,7 @@ func Reencrypt(app *cli.Context) error {
 		return err
 	}
 	b, err := json.Marshal(server.EncryptionRequest{
-		Stage: pointer.StringPtr(secretsencrypt.EncryptionReencryptActive),
+		Stage: pointer.String(secretsencrypt.EncryptionReencryptActive),
 		Force: cmds.ServerConfig.EncryptForce,
 		Skip:  cmds.ServerConfig.EncryptSkip,
 	})
@@ -214,3 +214,25 @@ func Reencrypt(app *cli.Context) error {
 	fmt.Println("reencryption started")
 	return nil
 }
+
+func RotateKeys(app *cli.Context) error {
+	var err error
+	if err = cmds.InitLogging(); err != nil {
+		return err
+	}
+	info, err := commandPrep(app, &cmds.ServerConfig)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(server.EncryptionRequest{
+		Stage: pointer.String(secretsencrypt.EncryptionRotateKeys),
+	})
+	if err != nil {
+		return err
+	}
+	if err = info.Put("/v1-"+version.Program+"/encrypt/config", b); err != nil {
+		return wrapServerError(err)
+	}
+	fmt.Println("keys rotated, rencryption started")
+	return nil
+}

pkg/daemons/control/server.go

@@ -214,6 +214,7 @@ func apiServer(ctx context.Context, cfg *config.Control) error {
 	}
 	if cfg.EncryptSecrets {
 		argsMap["encryption-provider-config"] = runtime.EncryptionConfig
+		argsMap["encryption-provider-config-automatic-reload"] = "true"
 	}
 	args := config.GetArgs(argsMap, cfg.ExtraAPIArgs)
 

pkg/secretsencrypt/config.go

@@ -21,6 +21,7 @@ const (
 	EncryptionStart             string = "start"
 	EncryptionPrepare           string = "prepare"
 	EncryptionRotate            string = "rotate"
+	EncryptionRotateKeys        string = "rotate_keys"
 	EncryptionReencryptRequest  string = "reencrypt_request"
 	EncryptionReencryptActive   string = "reencrypt_active"
 	EncryptionReencryptFinished string = "reencrypt_finished"

pkg/server/secrets-encrypt.go

@@ -17,6 +17,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -139,6 +140,9 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool)
 		logrus.Infoln("Secrets encryption already disabled")
 		return nil
 	} else if providers[0].Identity != nil && providers[1].AESCBC != nil && enable {
+		if nodeArgs := getNodeArgs(server); !strings.Contains(nodeArgs, "secrets-encryption") {
+			return fmt.Errorf("secrets encryption cannot be enabled without first starting the server with --secrets-encryption flag")
+		}
 		logrus.Infoln("Enabling secrets encryption")
 		if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, enable); err != nil {
 			return err
@@ -149,7 +153,20 @@ func encryptionEnable(ctx context.Context, server *config.Control, enable bool)
 	} else {
 		return fmt.Errorf("unable to enable/disable secrets encryption, unknown configuration")
 	}
-	return cluster.Save(ctx, server, true)
+	if err := cluster.Save(ctx, server, true); err != nil {
+		return err
+	}
+	server.EncryptSkip = true
+	return setReencryptAnnotation(server)
+}
+
+func getNodeArgs(server *config.Control) string {
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return ""
+	}
+	return node.Annotations[version.Program+".io/node-args"]
 }
 
 func encryptionConfigHandler(ctx context.Context, server *config.Control) http.Handler {
@@ -174,6 +191,8 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H
 				err = encryptionPrepare(ctx, server, encryptReq.Force)
 			case secretsencrypt.EncryptionRotate:
 				err = encryptionRotate(ctx, server, encryptReq.Force)
+			case secretsencrypt.EncryptionRotateKeys:
+				err = encryptionRotateKeys(ctx, server)
 			case secretsencrypt.EncryptionReencryptActive:
 				err = encryptionReencrypt(ctx, server, encryptReq.Force, encryptReq.Skip)
 			default:
@@ -280,6 +299,63 @@ func encryptionReencrypt(ctx context.Context, server *config.Control, force bool
 	return nil
 }
 
+func addAndRotateKeys(server *config.Control) error {
+
+	curKeys, err := secretsencrypt.GetEncryptionKeys(server.Runtime)
+	if err != nil {
+		return err
+	}
+
+	if err := AppendNewEncryptionKey(&curKeys); err != nil {
+		return err
+	}
+	logrus.Infoln("Adding secrets-encryption key: ", curKeys[len(curKeys)-1])
+
+	if err := secretsencrypt.WriteEncryptionConfig(server.Runtime, curKeys, true); err != nil {
+		return err
+	}
+
+	// Right rotate elements
+	rotatedKeys := append(curKeys[len(curKeys)-1:], curKeys[:len(curKeys)-1]...)
+
+	return secretsencrypt.WriteEncryptionConfig(server.Runtime, rotatedKeys, true)
+}
+
+// encryptionRotateKeys is both adds and rotates keys, and sets the annotaiton that triggers the
+// reencryption process. It is the preferred way to rotate keys, starting with v1.28
+func encryptionRotateKeys(ctx context.Context, server *config.Control) error {
+	states := secretsencrypt.EncryptionStart + "-" + secretsencrypt.EncryptionReencryptFinished
+	if err := verifyEncryptionHashAnnotation(server.Runtime, server.Runtime.Core.Core(), states); err != nil {
+		return err
+	}
+
+	if err := addAndRotateKeys(server); err != nil {
+		return err
+	}
+
+	return setReencryptAnnotation(server)
+}
+
+func setReencryptAnnotation(server *config.Control) error {
+	nodeName := os.Getenv("NODE_NAME")
+	node, err := server.Runtime.Core.Core().V1().Node().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	reencryptHash, err := secretsencrypt.GenReencryptHash(server.Runtime, secretsencrypt.EncryptionReencryptRequest)
+	if err != nil {
+		return err
+	}
+	ann := secretsencrypt.EncryptionReencryptRequest + "-" + reencryptHash
+	node.Annotations[secretsencrypt.EncryptionHashAnnotation] = ann
+	if _, err = server.Runtime.Core.Core().V1().Node().Update(node); err != nil {
+		return err
+	}
+	logrus.Debugf("encryption hash annotation set successfully on node: %s\n", node.ObjectMeta.Name)
+	return nil
+}
+
 func AppendNewEncryptionKey(keys *[]apiserverconfigv1.Key) error {
 
 	aescbcKey := make([]byte, aescbcKeySize)

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -3,6 +3,7 @@ package secretsencryption
 import (
 	"flag"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 
@@ -40,7 +41,7 @@ var _ = ReportAfterEach(e2e.GenReport)
 
 var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 	Context("Secrets Keys are rotated:", func() {
-		FIt("Starts up with no issues", func() {
+		It("Starts up with no issues", func() {
 			var err error
 			if *local {
 				serverNodeNames, _, err = e2e.CreateLocalCluster(*nodeOS, *serverCount, 0)
@@ -55,7 +56,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		FIt("Checks node and pod status", func() {
+		It("Checks node and pod status", func() {
 			fmt.Printf("\nFetching node status\n")
 			Eventually(func(g Gomega) {
 				nodes, err := e2e.ParseNodes(kubeConfigFile, false)
@@ -81,7 +82,7 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			_, _ = e2e.ParsePods(kubeConfigFile, true)
 		})
 
-		FIt("Deploys several secrets", func() {
+		It("Deploys several secrets", func() {
 			_, err := e2e.DeployWorkload("secrets.yaml", kubeConfigFile, *hardened)
 			Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
 		})
@@ -184,6 +185,18 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 			Eventually(func() (string, error) {
 				return e2e.RunCmdOnNode(cmd, serverNodeNames[0])
 			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+
+			for i, nodeName := range serverNodeNames {
+				Eventually(func(g Gomega) {
+					res, err := e2e.RunCmdOnNode(cmd, nodeName)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+					}
+				}, "420s", "2s").Should(Succeed())
+			}
 		})
 
 		It("Restarts K3s servers", func() {
@@ -197,7 +210,6 @@ var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
 					g.Expect(e2e.RunCmdOnNode(cmd, nodeName)).Should(ContainSubstring("Encryption Status: Enabled"))
 				}, "420s", "2s").Should(Succeed())
 			}
-
 		})
 	})
 
@@ -212,8 +224,8 @@ var _ = AfterSuite(func() {
 	if failed && !*ci {
 		fmt.Println("FAILED!")
 	} else {
-		// Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
-		// Expect(e2e.DestroyCluster()).To(Succeed())
-		// Expect(os.Remove(kubeConfigFile)).To(Succeed())
+		Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
+		Expect(e2e.DestroyCluster()).To(Succeed())
+		Expect(os.Remove(kubeConfigFile)).To(Succeed())
 	}
 })