Add k3s HA bootstrap

pkg/cli/server/server.go

@@ -157,8 +157,9 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		serverConfig.ControlConfig.ClusterDNS = net2.ParseIP(cfg.ClusterDNS)
 	}
 
-	// TODO: support etcd
-	serverConfig.ControlConfig.NoLeaderElect = true
+	if serverConfig.ControlConfig.StorageBackend != "etcd3" {
+		serverConfig.ControlConfig.NoLeaderElect = true
+	}
 
 	for _, noDeploy := range app.StringSlice("no-deploy") {
 		if noDeploy == "servicelb" {

pkg/daemons/control/ha.go

+package control
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"encoding/base64"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"go.etcd.io/etcd/clientv3"
+)
+
+const (
+	etcdDialTimeout    = 5 * time.Second
+	k3sRuntimeEtcdPath = "/k3s/runtime"
+)
+
+type serverHA struct {
+	ServerCAData           string `json:"serverCAData,omitempty"`
+	ServerCAKeyData        string `json:"serverCAKeyData,omitempty"`
+	ClientCAData           string `json:"clientCAData,omitempty"`
+	ClientCAKeyData        string `json:"clientCAKeyData,omitempty"`
+	ServiceKeyData         string `json:"serviceKeyData,omitempty"`
+	PasswdFileData         string `json:"passwdFileData,omitempty"`
+	RequestHeaderCAData    string `json:"requestHeaderCAData,omitempty"`
+	RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"`
+}
+
+func setHAData(cfg *config.Control) error {
+	if cfg.StorageBackend != "etcd3" {
+		return nil
+	}
+	endpoints := strings.Split(cfg.StorageEndpoint, ",")
+	cli, err := clientv3.New(clientv3.Config{
+		Endpoints:   endpoints,
+		DialTimeout: etcdDialTimeout,
+	})
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
+	if err != nil {
+		return err
+	}
+	if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" {
+		return nil
+	}
+	certData, err := readRuntimeCertData(cfg.Runtime)
+	if err != nil {
+		return err
+	}
+
+	runtimeBase64 := base64.StdEncoding.EncodeToString(certData)
+	_, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getHAData(cfg *config.Control) error {
+	serverRuntime := &serverHA{}
+	if cfg.StorageBackend != "etcd3" {
+		return nil
+	}
+	endpoints := strings.Split(cfg.StorageEndpoint, ",")
+	cli, err := clientv3.New(clientv3.Config{
+		Endpoints:   endpoints,
+		DialTimeout: etcdDialTimeout,
+	})
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
+	if err != nil {
+		return err
+	}
+	if len(gr.Kvs) == 0 {
+		return nil
+	}
+
+	runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value))
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil {
+		return err
+	}
+	return writeRuntimeCertData(cfg.Runtime, serverRuntime)
+}
+
+func readRuntimeCertData(runtime *config.ControlRuntime) ([]byte, error) {
+	serverHACerts := map[string]string{
+		runtime.ServerCA:           "",
+		runtime.ServerCAKey:        "",
+		runtime.ClientCA:           "",
+		runtime.ClientCAKey:        "",
+		runtime.ServiceKey:         "",
+		runtime.PasswdFile:         "",
+		runtime.RequestHeaderCA:    "",
+		runtime.RequestHeaderCAKey: "",
+	}
+	for k := range serverHACerts {
+		data, err := ioutil.ReadFile(k)
+		if err != nil {
+			return nil, err
+		}
+		serverHACerts[k] = string(data)
+	}
+	serverHACertsData := &serverHA{
+		ServerCAData:           serverHACerts[runtime.ServerCA],
+		ServerCAKeyData:        serverHACerts[runtime.ServerCAKey],
+		ClientCAData:           serverHACerts[runtime.ClientCA],
+		ClientCAKeyData:        serverHACerts[runtime.ClientCAKey],
+		ServiceKeyData:         serverHACerts[runtime.ServiceKey],
+		PasswdFileData:         serverHACerts[runtime.PasswdFile],
+		RequestHeaderCAData:    serverHACerts[runtime.RequestHeaderCA],
+		RequestHeaderCAKeyData: serverHACerts[runtime.RequestHeaderCAKey],
+	}
+	return json.Marshal(serverHACertsData)
+}
+
+func writeRuntimeCertData(runtime *config.ControlRuntime, runtimeData *serverHA) error {
+	runtimePathValue := map[string]string{
+		runtime.ServerCA:           runtimeData.ServerCAData,
+		runtime.ServerCAKey:        runtimeData.ServerCAKeyData,
+		runtime.ClientCA:           runtimeData.ClientCAData,
+		runtime.ClientCAKey:        runtimeData.ClientCAKeyData,
+		runtime.ServiceKey:         runtimeData.ServiceKeyData,
+		runtime.PasswdFile:         runtimeData.PasswdFileData,
+		runtime.RequestHeaderCA:    runtimeData.RequestHeaderCAData,
+		runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData,
+	}
+	for k, v := range runtimePathValue {
+		if _, err := os.Stat(k); os.IsNotExist(err) {
+			if err := ioutil.WriteFile(k, []byte(v), 600); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/daemons/control/server.go

@@ -290,6 +290,10 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
+	if err := getHAData(config); err != nil {
+		return err
+	}
+
 	if err := genCerts(config, runtime); err != nil {
 		return err
 	}
@@ -302,6 +306,10 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error {
 		return err
 	}
 
+	if err := setHAData(config); err != nil {
+		return err
+	}
+
 	return readTokens(runtime)
 }