Merge pull request #593 from erikwilson/cert-storage

pkg/cli/cmds/server.go

@@ -23,7 +23,7 @@ type Server struct {
 	ExtraSchedulerArgs  cli.StringSlice
 	ExtraControllerArgs cli.StringSlice
 	Rootless            bool
-	CertStorageBackend  string
+	BootstrapType       string
 	StorageBackend      string
 	StorageEndpoint     string
 	StorageCAFile       string
@@ -146,9 +146,9 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.Rootless,
 			},
 			cli.StringFlag{
-				Name:        "cert-storage-backend",
-				Usage:       "(experimental) Specify storage type for storing certificate information",
-				Destination: &ServerConfig.CertStorageBackend,
+				Name:        "bootstrap",
+				Usage:       "(experimental) Specify data bootstrap behavior (one of: none, read, write, or full), etcd3 only",
+				Destination: &ServerConfig.BootstrapType,
 			},
 			cli.StringFlag{
 				Name:        "storage-backend",

pkg/cli/server/server.go

@@ -124,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
-	serverConfig.ControlConfig.CertStorageBackend = cfg.CertStorageBackend
+	serverConfig.ControlConfig.BootstrapType = cfg.BootstrapType
 
 	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
 		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP

pkg/daemons/config/types.go

@@ -81,7 +81,7 @@ type Control struct {
 	KubeConfigMode        string
 	DataDir               string
 	Skips                 []string
-	CertStorageBackend    string
+	BootstrapType         string
 	StorageBackend        string
 	StorageEndpoint       string
 	StorageCAFile         string

pkg/daemons/control/ha.go → pkg/daemons/control/bootstrap.go

@@ -2,7 +2,11 @@ package control
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -11,15 +15,20 @@ import (
 	"encoding/base64"
 
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/clientv3"
 )
 
 const (
 	etcdDialTimeout    = 5 * time.Second
 	k3sRuntimeEtcdPath = "/k3s/runtime"
+	bootstrapTypeNone  = "none"
+	bootstrapTypeRead  = "read"
+	bootstrapTypeWrite = "write"
+	bootstrapTypeFull  = "full"
 )
 
-type serverHA struct {
+type serverBootstrap struct {
 	ServerCAData           string `json:"serverCAData,omitempty"`
 	ServerCAKeyData        string `json:"serverCAKeyData,omitempty"`
 	ClientCAData           string `json:"clientCAData,omitempty"`
@@ -30,15 +39,33 @@ type serverHA struct {
 	RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"`
 }
 
-func setHAData(cfg *config.Control) error {
-	if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" {
+var validBootstrapTypes = map[string]bool{
+	bootstrapTypeRead:  true,
+	bootstrapTypeWrite: true,
+	bootstrapTypeFull:  true,
+}
+
+func fetchBootstrapData(cfg *config.Control) error {
+	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
+		bootstrapTypeFull: true,
+		bootstrapTypeRead: true,
+	}); !valid {
+		if err != nil {
+			logrus.Warnf("Not fetching bootstrap data: %v", err)
+		}
 		return nil
 	}
 
+	tlsConfig, err := genBootstrapTLSConfig(cfg)
+	if err != nil {
+		return err
+	}
+
 	endpoints := strings.Split(cfg.StorageEndpoint, ",")
 	cli, err := clientv3.New(clientv3.Config{
 		Endpoints:   endpoints,
 		DialTimeout: etcdDialTimeout,
+		TLS:         tlsConfig,
 	})
 	if err != nil {
 		return err
@@ -49,32 +76,42 @@ func setHAData(cfg *config.Control) error {
 	if err != nil {
 		return err
 	}
-	if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" {
+	if len(gr.Kvs) == 0 {
 		return nil
 	}
-	certData, err := readRuntimeCertData(cfg.Runtime)
+
+	runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value))
 	if err != nil {
 		return err
 	}
-
-	runtimeBase64 := base64.StdEncoding.EncodeToString(certData)
-	_, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64)
-	if err != nil {
+	serverRuntime := &serverBootstrap{}
+	if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil {
 		return err
 	}
-
-	return nil
+	return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime)
 }
 
-func getHAData(cfg *config.Control) error {
-	serverRuntime := &serverHA{}
-	if cfg.StorageBackend != "etcd3" || cfg.CertStorageBackend != "etcd3" {
+func storeBootstrapData(cfg *config.Control) error {
+	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
+		bootstrapTypeFull:  true,
+		bootstrapTypeWrite: true,
+	}); !valid {
+		if err != nil {
+			logrus.Warnf("Not storing boostrap data: %v", err)
+		}
 		return nil
 	}
+
+	tlsConfig, err := genBootstrapTLSConfig(cfg)
+	if err != nil {
+		return err
+	}
+
 	endpoints := strings.Split(cfg.StorageEndpoint, ",")
 	cli, err := clientv3.New(clientv3.Config{
 		Endpoints:   endpoints,
 		DialTimeout: etcdDialTimeout,
+		TLS:         tlsConfig,
 	})
 	if err != nil {
 		return err
@@ -85,22 +122,70 @@ func getHAData(cfg *config.Control) error {
 	if err != nil {
 		return err
 	}
-	if len(gr.Kvs) == 0 {
+	if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" {
 		return nil
 	}
-
-	runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value))
+	certData, err := readRuntimeBootstrapData(cfg.Runtime)
 	if err != nil {
 		return err
 	}
-	if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil {
+
+	runtimeBase64 := base64.StdEncoding.EncodeToString(certData)
+	_, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64)
+	if err != nil {
 		return err
 	}
-	return writeRuntimeCertData(cfg.Runtime, serverRuntime)
+
+	return nil
+}
+
+func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) {
+	if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone {
+		return false, nil
+	}
+	if !validBootstrapTypes[cfg.BootstrapType] {
+		return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType)
+	}
+	if cfg.StorageBackend != "etcd3" {
+		return false, errors.New("bootstrap only supported with etcd3 as storage backend")
+	}
+	if !accepted[cfg.BootstrapType] {
+		return false, nil
+	}
+	return true, nil
+}
+
+func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) {
+	tlsConfig := &tls.Config{}
+	if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" {
+		certPem, err := ioutil.ReadFile(cfg.StorageCertFile)
+		if err != nil {
+			return nil, err
+		}
+		keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile)
+		if err != nil {
+			return nil, err
+		}
+		tlsCert, err := tls.X509KeyPair(certPem, keyPem)
+		if err != nil {
+			return nil, err
+		}
+		tlsConfig.Certificates = []tls.Certificate{tlsCert}
+	}
+	if cfg.StorageCAFile != "" {
+		caData, err := ioutil.ReadFile(cfg.StorageCAFile)
+		if err != nil {
+			return nil, err
+		}
+		certPool := x509.NewCertPool()
+		certPool.AppendCertsFromPEM(caData)
+		tlsConfig.RootCAs = certPool
+	}
+	return tlsConfig, nil
 }
 
-func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) {
-	serverHACerts := map[string]string{
+func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) {
+	serverBootstrapFiles := map[string]string{
 		runtime.ServerCA:           "",
 		runtime.ServerCAKey:        "",
 		runtime.ClientCA:           "",
@@ -110,27 +195,27 @@ func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) {
 		runtime.RequestHeaderCA:    "",
 		runtime.RequestHeaderCAKey: "",
 	}
-	for k := range serverHACerts {
+	for k := range serverBootstrapFiles {
 		data, err := ioutil.ReadFile(k)
 		if err != nil {
 			return nil, err
 		}
-		serverHACerts[k] = string(data)
+		serverBootstrapFiles[k] = string(data)
 	}
-	serverHACertsData := &serverHA{
-		ServerCAData:           serverHACerts[runtime.ServerCA],
-		ServerCAKeyData:        serverHACerts[runtime.ServerCAKey],
-		ClientCAData:           serverHACerts[runtime.ClientCA],
-		ClientCAKeyData:        serverHACerts[runtime.ClientCAKey],
-		ServiceKeyData:         serverHACerts[runtime.ServiceKey],
-		PasswdFileData:         serverHACerts[runtime.PasswdFile],
-		RequestHeaderCAData:    serverHACerts[runtime.RequestHeaderCA],
-		RequestHeaderCAKeyData: serverHACerts[runtime.RequestHeaderCAKey],
+	serverBootstrapFileData := &serverBootstrap{
+		ServerCAData:           serverBootstrapFiles[runtime.ServerCA],
+		ServerCAKeyData:        serverBootstrapFiles[runtime.ServerCAKey],
+		ClientCAData:           serverBootstrapFiles[runtime.ClientCA],
+		ClientCAKeyData:        serverBootstrapFiles[runtime.ClientCAKey],
+		ServiceKeyData:         serverBootstrapFiles[runtime.ServiceKey],
+		PasswdFileData:         serverBootstrapFiles[runtime.PasswdFile],
+		RequestHeaderCAData:    serverBootstrapFiles[runtime.RequestHeaderCA],
+		RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey],
 	}
-	return json.Marshal(serverHACertsData)
+	return json.Marshal(serverBootstrapFileData)
 }
 
-func writeRuntimeCertData(runtime *config.ControlRuntime, runtimeData *serverHA) error {
+func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error {
 	runtimePathValue := map[string]string{
 		runtime.ServerCA:           runtimeData.ServerCAData,
 		runtime.ServerCAKey:        runtimeData.ServerCAKeyData,

pkg/daemons/control/server.go

@@ -290,7 +290,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
-	if err := getHAData(config); err != nil {
+	if err := fetchBootstrapData(config); err != nil {
 		return err
 	}
 
@@ -306,7 +306,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
 		return err
 	}
 
-	if err := setHAData(config); err != nil {
+	if err := storeBootstrapData(config); err != nil {
 		return err
 	}