Refactor tokens, bootstrap, and cli args

manifests/rolebindings.yaml

@@ -10,3 +10,53 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: kube-apiserver
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:k3s-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - networkpolicies
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - pods
+  verbs:
+  - list
+  - get
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:k3s-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:k3s-controller
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: system:k3s-controller

pkg/agent/config/config.go

@@ -6,7 +6,6 @@ import (
 	cryptorand "crypto/rand"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	sysnet "net"
@@ -27,7 +26,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
 )
 
@@ -183,17 +181,6 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
 	return name, ip, nil
 }
 
-func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) {
-	os.MkdirAll(envInfo.DataDir, 0700)
-	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
-	info.CACerts = pem.EncodeToMemory(&pem.Block{
-		Type:  cert.CertificateBlockType,
-		Bytes: tlsCert.Certificate[1],
-	})
-
-	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
-}
-
 func isValidResolvConf(resolvConfFile string) bool {
 	file, err := os.Open(resolvConfFile)
 	if err != nil {
@@ -293,11 +280,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
-	kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert)
-	if err != nil {
-		return nil, err
-	}
-
 	clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
 	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
 		return nil, err
@@ -328,6 +310,21 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
+	clientK3sControllerCert := filepath.Join(envInfo.DataDir, "client-k3s-controller.crt")
+	if err := getHostFile(clientK3sControllerCert, info); err != nil {
+		return nil, err
+	}
+
+	clientK3sControllerKey := filepath.Join(envInfo.DataDir, "client-k3s-controller.key")
+	if err := getHostFile(clientK3sControllerKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigK3sController := filepath.Join(envInfo.DataDir, "k3scontroller.kubeconfig")
+	if err := control.KubeConfig(kubeconfigK3sController, info.URL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
+		return nil, err
+	}
+
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
@@ -345,9 +342,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.ClientCA = clientCAFile
 	nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
-	nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
+	nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
 	if envInfo.Rootless {
 		nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	}

pkg/agent/flannel/setup.go

@@ -13,8 +13,7 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
 const (
@@ -74,21 +73,11 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error {
 	return createFlannelConf(nodeConfig)
 }
 
-func Run(ctx context.Context, nodeConfig *config.Node) error {
+func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) error {
 	nodeName := nodeConfig.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
-	if err != nil {
-		return err
-	}
-
-	client, err := kubernetes.NewForConfig(restConfig)
-	if err != nil {
-		return err
-	}
-
 	for {
-		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err == nil && node.Spec.PodCIDR != "" {
 			break
 		}
@@ -101,11 +90,11 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 
-	return err
+	return nil
 }
 
 func createCNIConf(dir string) error {

pkg/agent/netpol/network_policy.go

@@ -10,7 +10,7 @@ import (
 )
 
 func Run(ctx context.Context, nodeConfig *config.Node) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
 	if err != nil {
 		return err
 	}

pkg/agent/run.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/rancher/k3s/pkg/agent/config"
 	"github.com/rancher/k3s/pkg/agent/containerd"
 	"github.com/rancher/k3s/pkg/agent/flannel"
@@ -21,10 +22,11 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/rootless"
-	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
-	corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
-	"github.com/rancher/wrangler/pkg/start"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -64,14 +66,18 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 		return err
 	}
 
+	coreClient, err := coreClient(nodeConfig.AgentConfig.KubeConfigKubelet)
+	if err != nil {
+		return err
+	}
 	if !nodeConfig.NoFlannel {
-		if err := flannel.Run(ctx, nodeConfig); err != nil {
+		if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {
 			return err
 		}
 	}
 
 	if !nodeConfig.AgentConfig.DisableCCM {
-		if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil {
+		if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
 			return err
 		}
 	}
@@ -86,6 +92,15 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 	return ctx.Err()
 }
 
+func coreClient(cfg string) (kubernetes.Interface, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubernetes.NewForConfig(restConfig)
+}
+
 func Run(ctx context.Context, cfg cmds.Agent) error {
 	if err := validate(); err != nil {
 		return err
@@ -100,10 +115,6 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 	cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
 	os.MkdirAll(cfg.DataDir, 0700)
 
-	if cfg.ClusterSecret != "" {
-		cfg.Token = "K10node:" + cfg.ClusterSecret
-	}
-
 	lb, err := loadbalancer.Setup(ctx, cfg)
 	if err != nil {
 		return err
@@ -113,7 +124,7 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 	}
 
 	for {
-		tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token)
+		newToken, err := clientaccess.NormalizeAndValidateTokenForUser(cfg.ServerURL, cfg.Token, "node")
 		if err != nil {
 			logrus.Error(err)
 			select {
@@ -123,10 +134,11 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 			}
 			continue
 		}
-		os.Remove(tmpFile)
+		cfg.Token = newToken
 		break
 	}
 
+	systemd.SdNotify(true, "READY=1\n")
 	return run(ctx, cfg, lb)
 }
 
@@ -149,73 +161,51 @@ func validate() error {
 	return nil
 }
 
-func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error {
+func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
 	for {
-		nodeController, nodeCache, err := startNodeController(ctx, agentConfig)
-		if err != nil {
-			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		nodeCached, err := nodeCache.Get(agentConfig.NodeName)
+		node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{})
 		if err != nil {
 			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		node := nodeCached.DeepCopy()
-		updated := updateLabelMap(ctx, agentConfig, node.Labels)
-		if updated {
-			_, err = nodeController.Update(node)
-			if err == nil {
-				logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName)
-				break
+
+		newLabels, update := updateLabelMap(agentConfig, node.Labels)
+		if update {
+			node.Labels = newLabels
+			if _, err := nodes.Update(node); err != nil {
+				logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-time.After(time.Second):
+					continue
+				}
 			}
-			logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
-			time.Sleep(1 * time.Second)
-			continue
+			logrus.Infof("addresses labels has been set successfully on node: %s", agentConfig.NodeName)
+		} else {
+			logrus.Infof("addresses labels has already been set successfully on node: %s", agentConfig.NodeName)
 		}
-		logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName)
-		return nil
-	}
-	return nil
-}
 
-func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet)
-	if err != nil {
-		return nil, nil, err
-	}
-	coreFactory := core.NewFactoryFromConfigOrDie(restConfig)
-	nodeController := coreFactory.Core().V1().Node()
-	nodeCache := nodeController.Cache()
-	if err := start.All(ctx, 1, coreFactory); err != nil {
-		return nil, nil, err
+		break
 	}
 
-	return nodeController, nodeCache, nil
+	return nil
 }
 
-func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool {
-	if nodeLabels == nil {
-		nodeLabels = make(map[string]string)
-	}
-	updated := false
-	if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP {
-		nodeLabels[InternalIPLabel] = agentConfig.NodeIP
-		updated = true
-	}
-	if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName {
-		nodeLabels[HostnameLabel] = agentConfig.NodeName
-		updated = true
+func updateLabelMap(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) {
+	result := map[string]string{}
+	for k, v := range nodeLabels {
+		result[k] = v
 	}
-	nodeExternalIP := agentConfig.NodeExternalIP
-	if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" {
-		nodeLabels[ExternalIPLabel] = nodeExternalIP
-		updated = true
-	} else if nodeExternalIP == "" && externalIPLabel != "" {
-		delete(nodeLabels, ExternalIPLabel)
-		updated = true
+
+	result[InternalIPLabel] = agentConfig.NodeIP
+	result[HostnameLabel] = agentConfig.NodeName
+	if agentConfig.NodeExternalIP == "" {
+		delete(result, ExternalIPLabel)
+	} else {
+		result[ExternalIPLabel] = agentConfig.NodeExternalIP
 	}
-	return updated
+
+	return result, !equality.Semantic.DeepEqual(nodeLabels, result)
 }

pkg/agent/tunnel/tunnel.go

@@ -3,11 +3,8 @@ package tunnel
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
 	"fmt"
 	"net"
-	"net/http"
 	"reflect"
 	"sync"
 	"time"
@@ -21,8 +18,8 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	watchtypes "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/transport"
 )
 
 var (
@@ -54,17 +51,22 @@ func getAddresses(endpoint *v1.Endpoints) []string {
 }
 
 func Setup(ctx context.Context, config *config.Node, onChange func([]string)) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigK3sController)
 	if err != nil {
 		return err
 	}
 
-	transportConfig, err := restConfig.TransportConfig()
+	client, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return err
 	}
 
-	client, err := kubernetes.NewForConfig(restConfig)
+	nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet)
+	if err != nil {
+		return err
+	}
+
+	tlsConfig, err := rest.TLSConfigFor(nodeRestConfig)
 	if err != nil {
 		return err
 	}
@@ -84,7 +86,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 	wg := &sync.WaitGroup{}
 	for _, address := range addresses {
 		if _, ok := disconnect[address]; !ok {
-			disconnect[address] = connect(ctx, wg, address, config, transportConfig)
+			disconnect[address] = connect(ctx, wg, address, tlsConfig)
 		}
 	}
 
@@ -132,7 +134,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 					for _, address := range addresses {
 						validEndpoint[address] = true
 						if _, ok := disconnect[address]; !ok {
-							disconnect[address] = connect(ctx, nil, address, config, transportConfig)
+							disconnect[address] = connect(ctx, nil, address, tlsConfig)
 						}
 					}
 
@@ -164,25 +166,10 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 	return nil
 }
 
-func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc {
+func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, tlsConfig *tls.Config) context.CancelFunc {
 	wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address)
-	headers := map[string][]string{
-		"X-K3s-NodeName": {config.AgentConfig.NodeName},
-	}
-	ws := &websocket.Dialer{}
-
-	if len(config.CACerts) > 0 {
-		pool := x509.NewCertPool()
-		pool.AppendCertsFromPEM(config.CACerts)
-		ws.TLSClientConfig = &tls.Config{
-			RootCAs: pool,
-		}
-	}
-
-	if transportConfig.Username != "" {
-		auth := transportConfig.Username + ":" + transportConfig.Password
-		auth = base64.StdEncoding.EncodeToString([]byte(auth))
-		headers["Authorization"] = []string{"Basic " + auth}
+	ws := &websocket.Dialer{
+		TLSClientConfig: tlsConfig,
 	}
 
 	once := sync.Once{}
@@ -194,7 +181,7 @@ func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string,
 
 	go func() {
 		for {
-			remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool {
+			remotedialer.ClientConnect(ctx, wsURL, nil, ws, func(proto, address string) bool {
 				host, port, err := net.SplitHostPort(address)
 				return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
 			}, func(_ context.Context) error {

pkg/apis/k3s.cattle.io/v1/types.go

 package v1
 
 import (
-	"github.com/rancher/dynamiclistener"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -9,16 +8,6 @@ import (
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-type ListenerConfig struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Status dynamiclistener.ListenerStatus `json:"status,omitempty"`
-}
-
-// +genclient
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
 type Addon struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

pkg/daemons/control/bootstrap.go → pkg/bootstrap/bootstrap.go

-package control
+package bootstrap
 
 import (
-	"context"
 	"encoding/json"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/kine/pkg/client"
-	"github.com/sirupsen/logrus"
 )
 
-const (
-	k3sRuntimeEtcdPath = "/k3s/runtime"
-)
+func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("Content-Type", "application/json")
+		Write(rw, bootstrap)
+	})
+}
 
-// fetchBootstrapData copies the bootstrap data (certs, keys, passwords)
-// from etcd to individual files specified by cfg.Runtime.
-func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Client) error {
-	logrus.Info("Fetching bootstrap data from etcd")
-	gr, err := c.Get(ctx, k3sRuntimeEtcdPath)
+func Write(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {
+	paths, err := objToMap(bootstrap)
 	if err != nil {
-		return err
-	}
-	if gr.Modified == 0 {
 		return nil
 	}
 
-	paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
+	dataMap := map[string][]byte{}
+	for pathKey, path := range paths {
+		if path == "" {
+			continue
+		}
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read %s", path)
+		}
+
+		dataMap[pathKey] = data
+	}
+
+	return json.NewEncoder(w).Encode(dataMap)
+}
+
+func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
+	paths, err := objToMap(bootstrap)
 	if err != nil {
 		return err
 	}
 
 	files := map[string][]byte{}
-	if err := json.Unmarshal(gr.Data, &files); err != nil {
+	if err := json.NewDecoder(r).Decode(&files); err != nil {
 		return err
 	}
 
@@ -49,7 +62,7 @@ func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Clien
 			return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
 		}
 
-		if err := ioutil.WriteFile(path, data, 0700); err != nil {
+		if err := ioutil.WriteFile(path, data, 0600); err != nil {
 			return errors.Wrapf(err, "failed to write to %s", path)
 		}
 	}
@@ -57,39 +70,6 @@ func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Clien
 	return nil
 }
 
-// storeBootstrapData copies the bootstrap data in the opposite direction to
-// fetchBootstrapData.
-func storeBootstrapData(ctx context.Context, cfg *config.Control, client client.Client) error {
-	if cfg.BootstrapReadOnly {
-		return nil
-	}
-
-	paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
-	if err != nil {
-		return nil
-	}
-
-	dataMap := map[string][]byte{}
-	for pathKey, path := range paths {
-		if path == "" {
-			continue
-		}
-		data, err := ioutil.ReadFile(path)
-		if err != nil {
-			return errors.Wrapf(err, "failed to read %s", path)
-		}
-
-		dataMap[pathKey] = data
-	}
-
-	bytes, err := json.Marshal(dataMap)
-	if err != nil {
-		return err
-	}
-
-	return client.Put(ctx, k3sRuntimeEtcdPath, bytes)
-}
-
 func objToMap(obj interface{}) (map[string]string, error) {
 	bytes, err := json.Marshal(obj)
 	if err != nil {

pkg/cli/agent/agent.go

@@ -3,39 +3,18 @@ package agent
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"strings"
-	"time"
 
-	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/rancher/k3s/pkg/agent"
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/datadir"
 	"github.com/rancher/k3s/pkg/netutil"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-func readToken(path string) (string, error) {
-	if path == "" {
-		return "", nil
-	}
-
-	for {
-		tokenBytes, err := ioutil.ReadFile(path)
-		if err == nil {
-			return strings.TrimSpace(string(tokenBytes)), nil
-		} else if os.IsNotExist(err) {
-			logrus.Infof("Waiting for %s to be available\n", path)
-			time.Sleep(2 * time.Second)
-		} else {
-			return "", err
-		}
-	}
-}
-
 func Run(ctx *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err
@@ -45,14 +24,14 @@ func Run(ctx *cli.Context) error {
 	}
 
 	if cmds.AgentConfig.TokenFile != "" {
-		token, err := readToken(cmds.AgentConfig.TokenFile)
+		token, err := token.ReadFile(cmds.AgentConfig.TokenFile)
 		if err != nil {
 			return err
 		}
 		cmds.AgentConfig.Token = token
 	}
 
-	if cmds.AgentConfig.Token == "" && cmds.AgentConfig.ClusterSecret == "" {
+	if cmds.AgentConfig.Token == "" {
 		return fmt.Errorf("--token is required")
 	}
 
@@ -76,7 +55,6 @@ func Run(ctx *cli.Context) error {
 	cfg.DataDir = dataDir
 
 	contextCtx := signals.SetupSignalHandler(context.Background())
-	systemd.SdNotify(true, "READY=1\n")
 
 	return agent.Run(contextCtx, cfg)
 }

pkg/cli/cmds/agent.go

@@ -17,7 +17,6 @@ type Agent struct {
 	NodeIP                   string
 	NodeExternalIP           string
 	NodeName                 string
-	ClusterSecret            string
 	PauseImage               string
 	Docker                   bool
 	ContainerRuntimeEndpoint string
@@ -44,82 +43,82 @@ var (
 	AgentConfig Agent
 	NodeIPFlag  = cli.StringFlag{
 		Name:        "node-ip,i",
-		Usage:       "(agent) IP address to advertise for node",
+		Usage:       "(agent/networking) IP address to advertise for node",
 		Destination: &AgentConfig.NodeIP,
 	}
 	NodeExternalIPFlag = cli.StringFlag{
 		Name:        "node-external-ip",
-		Usage:       "(agent) External IP address to advertise for node",
+		Usage:       "(agent/networking) External IP address to advertise for node",
 		Destination: &AgentConfig.NodeExternalIP,
 	}
 	NodeNameFlag = cli.StringFlag{
 		Name:        "node-name",
-		Usage:       "(agent) Node name",
+		Usage:       "(agent/node) Node name",
 		EnvVar:      "K3S_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	}
 	DockerFlag = cli.BoolFlag{
 		Name:        "docker",
-		Usage:       "(agent) Use docker instead of containerd",
+		Usage:       "(agent/runtime) Use docker instead of containerd",
 		Destination: &AgentConfig.Docker,
 	}
+	CRIEndpointFlag = cli.StringFlag{
+		Name:        "container-runtime-endpoint",
+		Usage:       "(agent/runtime) Disable embedded containerd and use alternative CRI implementation",
+		Destination: &AgentConfig.ContainerRuntimeEndpoint,
+	}
+	PrivateRegistryFlag = cli.StringFlag{
+		Name:        "private-registry",
+		Usage:       "(agent/runtime) Private registry configuration file",
+		Destination: &AgentConfig.PrivateRegistry,
+		Value:       "/etc/rancher/k3s/registries.yaml",
+	}
+	PauseImageFlag = cli.StringFlag{
+		Name:        "pause-image",
+		Usage:       "(agent/runtime) Customized pause image for containerd sandbox",
+		Destination: &AgentConfig.PauseImage,
+	}
 	FlannelFlag = cli.BoolFlag{
 		Name:        "no-flannel",
-		Usage:       "(agent) Disable embedded flannel",
+		Usage:       "(deprecated) use --flannel-backend=none",
 		Destination: &AgentConfig.NoFlannel,
 	}
 	FlannelIfaceFlag = cli.StringFlag{
 		Name:        "flannel-iface",
-		Usage:       "(agent) Override default flannel interface",
+		Usage:       "(agent/networking) Override default flannel interface",
 		Destination: &AgentConfig.FlannelIface,
 	}
 	FlannelConfFlag = cli.StringFlag{
 		Name:        "flannel-conf",
-		Usage:       "(agent) (experimental) Override default flannel config file",
+		Usage:       "(agent/networking) Override default flannel config file",
 		Destination: &AgentConfig.FlannelConf,
 	}
-	CRIEndpointFlag = cli.StringFlag{
-		Name:        "container-runtime-endpoint",
-		Usage:       "(agent) Disable embedded containerd and use alternative CRI implementation",
-		Destination: &AgentConfig.ContainerRuntimeEndpoint,
-	}
-	PauseImageFlag = cli.StringFlag{
-		Name:        "pause-image",
-		Usage:       "(agent) Customized pause image for containerd sandbox",
-		Destination: &AgentConfig.PauseImage,
-	}
 	ResolvConfFlag = cli.StringFlag{
 		Name:        "resolv-conf",
-		Usage:       "(agent) Kubelet resolv.conf file",
+		Usage:       "(agent/networking) Kubelet resolv.conf file",
 		EnvVar:      "K3S_RESOLV_CONF",
 		Destination: &AgentConfig.ResolvConf,
 	}
 	ExtraKubeletArgs = cli.StringSliceFlag{
 		Name:  "kubelet-arg",
-		Usage: "(agent) Customized flag for kubelet process",
+		Usage: "(agent/flags) Customized flag for kubelet process",
 		Value: &AgentConfig.ExtraKubeletArgs,
 	}
 	ExtraKubeProxyArgs = cli.StringSliceFlag{
 		Name:  "kube-proxy-arg",
-		Usage: "(agent) Customized flag for kube-proxy process",
+		Usage: "(agent/flags) Customized flag for kube-proxy process",
 		Value: &AgentConfig.ExtraKubeProxyArgs,
 	}
 	NodeTaints = cli.StringSliceFlag{
 		Name:  "node-taint",
-		Usage: "(agent) Registering kubelet with set of taints",
+		Usage: "(agent/node) Registering kubelet with set of taints",
 		Value: &AgentConfig.Taints,
 	}
 	NodeLabels = cli.StringSliceFlag{
 		Name:  "node-label",
-		Usage: "(agent) Registering kubelet with set of labels",
+		Usage: "(agent/node) Registering kubelet with set of labels",
 		Value: &AgentConfig.Labels,
 	}
-	PrivateRegistryFlag = cli.StringFlag{
-		Name:        "private-registry",
-		Usage:       "(agent) Private registry configuration file",
-		Destination: &AgentConfig.PrivateRegistry,
-		Value:       "/etc/rancher/k3s/registries.yaml",
-	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -135,54 +134,57 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			AlsoLogToStderr,
 			cli.StringFlag{
 				Name:        "token,t",
-				Usage:       "Token to use for authentication",
+				Usage:       "(cluster) Token to use for authentication",
 				EnvVar:      "K3S_TOKEN",
 				Destination: &AgentConfig.Token,
 			},
 			cli.StringFlag{
 				Name:        "token-file",
-				Usage:       "Token file to use for authentication",
+				Usage:       "(cluster) Token file to use for authentication",
 				EnvVar:      "K3S_TOKEN_FILE",
 				Destination: &AgentConfig.TokenFile,
 			},
 			cli.StringFlag{
 				Name:        "server,s",
-				Usage:       "Server to connect to",
+				Usage:       "(cluster) Server to connect to",
 				EnvVar:      "K3S_URL",
 				Destination: &AgentConfig.ServerURL,
 			},
 			cli.StringFlag{
 				Name:        "data-dir,d",
-				Usage:       "Folder to hold state",
+				Usage:       "(agent/data) Folder to hold state",
 				Destination: &AgentConfig.DataDir,
 				Value:       "/var/lib/rancher/k3s",
 			},
-			cli.StringFlag{
-				Name:        "cluster-secret",
-				Usage:       "Shared secret used to bootstrap a cluster",
-				Destination: &AgentConfig.ClusterSecret,
-				EnvVar:      "K3S_CLUSTER_SECRET",
-			},
-			cli.BoolFlag{
-				Name:        "rootless",
-				Usage:       "(experimental) Run rootless",
-				Destination: &AgentConfig.Rootless,
-			},
-			DockerFlag,
-			FlannelFlag,
-			FlannelIfaceFlag,
-			FlannelConfFlag,
 			NodeNameFlag,
-			NodeIPFlag,
+			NodeLabels,
+			NodeTaints,
+			DockerFlag,
 			CRIEndpointFlag,
 			PauseImageFlag,
+			PrivateRegistryFlag,
+			NodeIPFlag,
+			NodeExternalIPFlag,
 			ResolvConfFlag,
+			FlannelIfaceFlag,
+			FlannelConfFlag,
 			ExtraKubeletArgs,
 			ExtraKubeProxyArgs,
-			NodeLabels,
-			NodeTaints,
-			PrivateRegistryFlag,
-			NodeExternalIPFlag,
+			cli.BoolFlag{
+				Name:        "rootless",
+				Usage:       "(experimental) Run rootless",
+				Destination: &AgentConfig.Rootless,
+			},
+
+			// Deprecated/hidden below
+
+			FlannelFlag,
+			cli.StringFlag{
+				Name:        "cluster-secret",
+				Usage:       "(deprecated) use --token",
+				Destination: &AgentConfig.Token,
+				EnvVar:      "K3S_CLUSTER_SECRET",
+			},
 		},
 	}
 }

pkg/cli/cmds/log.go

@@ -25,22 +25,22 @@ var (
 
 	VLevel = cli.IntFlag{
 		Name:        "v",
-		Usage:       "Number for the log level verbosity",
+		Usage:       "(logging) Number for the log level verbosity",
 		Destination: &LogConfig.VLevel,
 	}
 	VModule = cli.StringFlag{
 		Name:        "vmodule",
-		Usage:       "Comma-separated list of pattern=N settings for file-filtered logging",
+		Usage:       "(logging) Comma-separated list of pattern=N settings for file-filtered logging",
 		Destination: &LogConfig.VModule,
 	}
 	LogFile = cli.StringFlag{
 		Name:        "log,l",
-		Usage:       "Log to file",
+		Usage:       "(logging) Log to file",
 		Destination: &LogConfig.LogFile,
 	}
 	AlsoLogToStderr = cli.BoolFlag{
 		Name:        "alsologtostderr",
-		Usage:       "Log to standard error as well as file (if set)",
+		Usage:       "(logging) Log to standard error as well as file (if set)",
 		Destination: &LogConfig.AlsoLogToStderr,
 	}
 )

pkg/cli/cmds/nocluster.go

+// +build !dqlite
+
+package cmds
+
+const (
+	hideDqlite = true
+)

pkg/cli/server/server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rancher/k3s/pkg/netutil"
 	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -56,23 +57,28 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 
 	serverConfig := server.Config{}
 	serverConfig.DisableAgent = cfg.DisableAgent
-	serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret
+	serverConfig.ControlConfig.Token = cfg.Token
+	serverConfig.ControlConfig.AgentToken = cfg.AgentToken
+	serverConfig.ControlConfig.JoinURL = cfg.ServerURL
+	if cfg.AgentTokenFile != "" {
+		serverConfig.ControlConfig.AgentToken, err = token.ReadFile(cfg.AgentTokenFile)
+		if err != nil {
+			return err
+		}
+	}
+	if cfg.TokenFile != "" {
+		serverConfig.ControlConfig.Token, err = token.ReadFile(cfg.TokenFile)
+		if err != nil {
+			return err
+		}
+	}
 	serverConfig.ControlConfig.DataDir = cfg.DataDir
 	serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
 	serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
 	serverConfig.ControlConfig.NoScheduler = cfg.DisableScheduler
 	serverConfig.Rootless = cfg.Rootless
-	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
-	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
-	for _, san := range knownIPs(cfg.TLSSan) {
-		addr := net2.ParseIP(san)
-		if addr != nil {
-			serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
-		} else {
-			serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
-		}
-	}
-	serverConfig.TLSConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.SANs = knownIPs(cfg.TLSSan)
+	serverConfig.ControlConfig.BindAddress = cfg.BindAddress
 	serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
@@ -84,21 +90,25 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.Storage.KeyFile = cfg.StorageKeyFile
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
-	serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 	serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
 	serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM
 	serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
+	serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
+	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
 
 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
 	}
 
+	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeExternalIP != "" {
+		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeExternalIP
+	}
 	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
 		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
 	}
 	if serverConfig.ControlConfig.AdvertiseIP != "" {
-		serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
+		serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
 	}
 
 	_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
@@ -114,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
+	serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String())
 
 	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
@@ -160,8 +170,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	os.Unsetenv("NOTIFY_SOCKET")
 
 	ctx := signals.SetupSignalHandler(context.Background())
-	certs, err := server.StartServer(ctx, &serverConfig)
-	if err != nil {
+	if err := server.StartServer(ctx, &serverConfig); err != nil {
 		return err
 	}
 
@@ -175,12 +184,17 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		<-ctx.Done()
 		return nil
 	}
-	ip := serverConfig.TLSConfig.BindAddress
+
+	ip := serverConfig.ControlConfig.BindAddress
 	if ip == "" {
 		ip = "127.0.0.1"
 	}
-	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.TLSConfig.HTTPSPort)
-	token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs)
+
+	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.HTTPSPort)
+	token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
+	if err != nil {
+		return err
+	}
 
 	agentConfig := cmds.AgentConfig
 	agentConfig.Debug = app.GlobalBool("bool")

pkg/clientaccess/clientaccess.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -35,21 +34,6 @@ type clientToken struct {
 	password string
 }
 
-func AgentAccessInfoToTempKubeConfig(tempDir, server, token string) (string, error) {
-	f, err := ioutil.TempFile(tempDir, "tmp-")
-	if err != nil {
-		return "", err
-	}
-	if err := f.Close(); err != nil {
-		return "", err
-	}
-	err = accessInfoToKubeConfig(f.Name(), server, token)
-	if err != nil {
-		os.Remove(f.Name())
-	}
-	return f.Name(), err
-}
-
 func AgentAccessInfoToKubeConfig(destFile, server, token string) error {
 	return accessInfoToKubeConfig(destFile, server, token)
 }
@@ -62,6 +46,10 @@ type Info struct {
 	Token    string `json:"token,omitempty"`
 }
 
+func (i *Info) ToToken() string {
+	return fmt.Sprintf("K10%s::%s:%s", hashCA(i.CACerts), i.username, i.password)
+}
+
 func (i *Info) WriteKubeConfig(destFile string) error {
 	return clientcmd.WriteToFile(*i.KubeConfig(), destFile)
 }
@@ -98,6 +86,22 @@ func (i *Info) KubeConfig() *clientcmdapi.Config {
 	return config
 }
 
+func NormalizeAndValidateTokenForUser(server, token, user string) (string, error) {
+	if !strings.HasPrefix(token, "K10") {
+		token = "K10::" + user + ":" + token
+	}
+	info, err := ParseAndValidateToken(server, token)
+	if err != nil {
+		return "", err
+	}
+
+	if info.username != user {
+		info.username = user
+	}
+
+	return info.ToToken(), nil
+}
+
 func ParseAndValidateToken(server, token string) (*Info, error) {
 	url, err := url.Parse(server)
 	if err != nil {
@@ -132,13 +136,17 @@ func ParseAndValidateToken(server, token string) (*Info, error) {
 		return nil, err
 	}
 
-	return &Info{
+	i := &Info{
 		URL:      url.String(),
 		CACerts:  cacerts,
 		username: parsedToken.username,
 		password: parsedToken.password,
 		Token:    token,
-	}, nil
+	}
+
+	// normalize token
+	i.Token = i.ToToken()
+	return i, nil
 }
 
 func accessInfoToKubeConfig(destFile, server, token string) error {
@@ -164,11 +172,15 @@ func validateCACerts(cacerts []byte, hash string) (bool, string, string) {
 		return true, "", ""
 	}
 
-	digest := sha256.Sum256([]byte(cacerts))
-	newHash := hex.EncodeToString(digest[:])
+	newHash := hashCA(cacerts)
 	return hash == newHash, hash, newHash
 }
 
+func hashCA(cacerts []byte) string {
+	digest := sha256.Sum256(cacerts)
+	return hex.EncodeToString(digest[:])
+}
+
 func ParseUsernamePassword(token string) (string, string, bool) {
 	parsed, err := parseToken(token)
 	if err != nil {

pkg/cluster/cluster.go

+package cluster
+
+import (
+	"context"
+
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+type Cluster struct {
+	token            string
+	clientAccessInfo *clientaccess.Info
+	config           *config.Control
+	runtime          *config.ControlRuntime
+	db               interface{}
+}
+
+func (c *Cluster) Start(ctx context.Context) error {
+	join, err := c.shouldJoin()
+	if err != nil {
+		return err
+	}
+
+	if join {
+		if err := c.join(); err != nil {
+			return err
+		}
+	}
+
+	if err := c.startClusterAndHTTPS(ctx); err != nil {
+		return err
+	}
+
+	if join {
+		if err := c.postJoin(ctx); err != nil {
+			return err
+		}
+	}
+
+	return c.joined()
+}
+
+func New(config *config.Control) *Cluster {
+	return &Cluster{
+		config:  config,
+		runtime: config.Runtime,
+	}
+}

pkg/cluster/https.go

+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"path/filepath"
+
+	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/dynamiclistener/factory"
+	"github.com/rancher/dynamiclistener/storage/file"
+	"github.com/rancher/dynamiclistener/storage/kubernetes"
+	"github.com/rancher/dynamiclistener/storage/memory"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
+	tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.HTTPSPort)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	storage := tlsStorage(ctx, c.config.DataDir, c.runtime)
+	return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{
+		CN:           "k3s",
+		Organization: []string{"k3s"},
+		TLSConfig: tls.Config{
+			ClientAuth: tls.RequestClientCert,
+		},
+		SANs: c.config.SANs,
+	})
+}
+
+func (c *Cluster) startClusterAndHTTPS(ctx context.Context) error {
+	l, handler, err := c.newListener(ctx)
+	if err != nil {
+		return err
+	}
+
+	handler, err = c.getHandler(handler)
+	if err != nil {
+		return err
+	}
+
+	l, handler, err = c.initClusterDB(ctx, l, handler)
+	if err != nil {
+		return err
+	}
+
+	server := http.Server{
+		Handler: handler,
+	}
+
+	go func() {
+		err := server.Serve(l)
+		logrus.Fatalf("server stopped: %v", err)
+	}()
+
+	go func() {
+		<-ctx.Done()
+		server.Shutdown(context.Background())
+	}()
+
+	return nil
+}
+
+func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) dynamiclistener.TLSStorage {
+	fileStorage := file.New(filepath.Join(dataDir, "tls/dynamic-cert.json"))
+	cache := memory.NewBacked(fileStorage)
+	return kubernetes.New(ctx, func() *core.Factory {
+		return runtime.Core
+	}, "kube-system", "k3s-serving", cache)
+}

pkg/cluster/join.go

+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/rancher/k3s/pkg/bootstrap"
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *Cluster) shouldJoin() (bool, error) {
+	if c.config.JoinURL == "" {
+		return false, nil
+	}
+
+	stamp := filepath.Join(c.config.DataDir, "db/joined")
+	if _, err := os.Stat(stamp); err == nil {
+		logrus.Info("Already joined to cluster, not rejoining")
+		return false, nil
+	}
+
+	if c.config.Token == "" {
+		return false, fmt.Errorf("K3S_TOKEN is required to join a cluster")
+	}
+
+	return true, nil
+}
+
+func (c *Cluster) joined() error {
+	if err := os.MkdirAll(filepath.Dir(c.joinStamp()), 0700); err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(c.joinStamp()); err == nil {
+		return nil
+	}
+
+	f, err := os.Create(c.joinStamp())
+	if err != nil {
+		return err
+	}
+
+	return f.Close()
+}
+
+func (c *Cluster) join() error {
+	c.runtime.Cluster.Join = true
+
+	token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
+	if err != nil {
+		return err
+	}
+	c.token = token
+
+	info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
+	if err != nil {
+		return err
+	}
+	c.clientAccessInfo = info
+
+	content, err := clientaccess.Get("/v1-k3s/server-bootstrap", info)
+	if err != nil {
+		return err
+	}
+
+	return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
+}
+
+func (c *Cluster) joinStamp() string {
+	return filepath.Join(c.config.DataDir, "db/joined")
+}

pkg/cluster/nocluster.go

+// +build !dqlite
+
+package cluster
+
+import (
+	"context"
+	"net"
+	"net/http"
+)
+
+func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
+	return l, handler, nil
+}
+
+func (c *Cluster) postJoin(ctx context.Context) error {
+	return nil
+}

pkg/cluster/router.go

+package cluster
+
+import (
+	"net/http"
+)
+
+func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) {
+	next := c.router()
+
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		handler.ServeHTTP(rw, req)
+		next.ServeHTTP(rw, req)
+	}), nil
+}
+
+func (c *Cluster) router() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if c.runtime.Handler == nil {
+			http.Error(rw, "starting", http.StatusServiceUnavailable)
+			return
+		}
+
+		c.runtime.Handler.ServeHTTP(rw, req)
+	})
+}

pkg/codegen/main.go

@@ -70,7 +70,6 @@ func main() {
 		Groups: map[string]args.Group{
 			"k3s.cattle.io": {
 				Types: []interface{}{
-					v1.ListenerConfig{},
 					v1.Addon{},
 				},
 				GenerateTypes: true,

pkg/daemons/config/types.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/rancher/kine/pkg/endpoint"
-
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
@@ -47,40 +47,36 @@ type Containerd struct {
 }
 
 type Agent struct {
-	NodeName            string
-	ClientKubeletCert   string
-	ClientKubeletKey    string
-	ClientKubeProxyCert string
-	ClientKubeProxyKey  string
-	ServingKubeletCert  string
-	ServingKubeletKey   string
-	ClusterCIDR         net.IPNet
-	ClusterDNS          net.IP
-	ClusterDomain       string
-	ResolvConf          string
-	RootDir             string
-	KubeConfigNode      string
-	KubeConfigKubelet   string
-	KubeConfigKubeProxy string
-	NodeIP              string
-	NodeExternalIP      string
-	RuntimeSocket       string
-	ListenAddress       string
-	ClientCA            string
-	CNIBinDir           string
-	CNIConfDir          string
-	ExtraKubeletArgs    []string
-	ExtraKubeProxyArgs  []string
-	PauseImage          string
-	CNIPlugin           bool
-	NodeTaints          []string
-	NodeLabels          []string
-	IPSECPSK            string
-	StrongSwanDir       string
-	PrivateRegistry     string
-	DisableCCM          bool
-	DisableNPC          bool
-	Rootless            bool
+	NodeName                string
+	ServingKubeletCert      string
+	ServingKubeletKey       string
+	ClusterCIDR             net.IPNet
+	ClusterDNS              net.IP
+	ClusterDomain           string
+	ResolvConf              string
+	RootDir                 string
+	KubeConfigKubelet       string
+	KubeConfigKubeProxy     string
+	KubeConfigK3sController string
+	NodeIP                  string
+	NodeExternalIP          string
+	RuntimeSocket           string
+	ListenAddress           string
+	ClientCA                string
+	CNIBinDir               string
+	CNIConfDir              string
+	ExtraKubeletArgs        []string
+	ExtraKubeProxyArgs      []string
+	PauseImage              string
+	CNIPlugin               bool
+	NodeTaints              []string
+	NodeLabels              []string
+	IPSECPSK                string
+	StrongSwanDir           string
+	PrivateRegistry         string
+	DisableCCM              bool
+	DisableNPC              bool
+	Rootless                bool
 }
 
 type Control struct {
@@ -88,7 +84,8 @@ type Control struct {
 	AdvertiseIP              string
 	ListenPort               int
 	HTTPSPort                int
-	ClusterSecret            string
+	AgentToken               string
+	Token                    string
 	ClusterIPRange           *net.IPNet
 	ServiceIPRange           *net.IPNet
 	ClusterDNS               net.IP
@@ -98,19 +95,24 @@ type Control struct {
 	KubeConfigMode           string
 	DataDir                  string
 	Skips                    []string
-	BootstrapReadOnly        bool
 	Storage                  endpoint.Config
 	NoScheduler              bool
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
-	ExtraSchedulerAPIArgs    []string
 	ExtraCloudControllerArgs []string
+	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
+	JoinURL                  string
 	FlannelBackend           string
 	IPSECPSK                 string
 	DefaultLocalStoragePath  string
 	DisableCCM               bool
 	DisableNPC               bool
+	ClusterInit              bool
+	ClusterReset             bool
+
+	BindAddress string
+	SANs        []string
 
 	Runtime *ControlRuntime `json:"-"`
 }
@@ -124,9 +126,6 @@ type ControlRuntimeBootstrap struct {
 	PasswdFile         string
 	RequestHeaderCA    string
 	RequestHeaderCAKey string
-	ClientKubeletKey   string
-	ClientKubeProxyKey string
-	ServingKubeletKey  string
 	IPSECKey           string
 }
 
@@ -145,8 +144,10 @@ type ControlRuntime struct {
 
 	ServingKubeAPICert string
 	ServingKubeAPIKey  string
+	ServingKubeletKey  string
 	ClientToken        string
-	NodeToken          string
+	ServerToken        string
+	AgentToken         string
 	Handler            http.Handler
 	Tunnel             http.Handler
 	Authenticator      authenticator.Request
@@ -161,8 +162,19 @@ type ControlRuntime struct {
 	ClientSchedulerCert       string
 	ClientSchedulerKey        string
 	ClientKubeProxyCert       string
+	ClientKubeProxyKey        string
+	ClientKubeletKey          string
 	ClientCloudControllerCert string
 	ClientCloudControllerKey  string
+	ClientK3sControllerCert   string
+	ClientK3sControllerKey    string
+
+	Cluster ClusterConfig
+	Core    *core.Factory
+}
+
+type ClusterConfig struct {
+	Join bool
 }
 
 type ArgString []string

pkg/daemons/control/server.go

@@ -3,13 +3,9 @@ package control
 import (
 	"context"
 	"crypto"
-	cryptorand "crypto/rand"
 	"crypto/x509"
-	"encoding/csv"
-	"encoding/hex"
 	"fmt"
 	"html/template"
-	"io"
 	"io/ioutil"
 	"math/rand"
 	"net"
@@ -21,11 +17,14 @@ import (
 	"strings"
 	"time"
 
-	certutil "github.com/rancher/dynamiclistener/cert"
 	// registering k3s cloud provider
 	_ "github.com/rancher/k3s/pkg/cloudprovider"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3s/pkg/cluster"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/kine/pkg/client"
+	"github.com/rancher/k3s/pkg/passwd"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/kine/pkg/endpoint"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
@@ -121,6 +120,10 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
 		"cluster-signing-cert-file":        runtime.ServerCA,
 		"cluster-signing-key-file":         runtime.ServerCAKey,
 	}
+	offset := cfg.HTTPSPort - 6443
+	if offset > 0 {
+		argsMap["port"] = strconv.Itoa(10252 + offset)
+	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
@@ -143,6 +146,10 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
 		"bind-address": "127.0.0.1",
 		"secure-port":  "0",
 	}
+	offset := cfg.HTTPSPort - 6443
+	if offset > 0 {
+		argsMap["port"] = strconv.Itoa(10251 + offset)
+	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
@@ -291,6 +298,8 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key")
 	runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt")
 	runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key")
+	runtime.ClientK3sControllerCert = path.Join(config.DataDir, "tls", "client-k3s-controller.crt")
+	runtime.ClientK3sControllerKey = path.Join(config.DataDir, "tls", "client-k3s-controller.key")
 
 	runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
 	runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
@@ -301,17 +310,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
-	etcdClient, err := prepareStorageBackend(ctx, config)
-	if err != nil {
-		return err
-	}
-	defer etcdClient.Close()
-
-	if err := fetchBootstrapData(ctx, config, etcdClient); err != nil {
+	if err := genCerts(config, runtime); err != nil {
 		return err
 	}
 
-	if err := genCerts(config, runtime); err != nil {
+	if err := cluster.New(config).Start(ctx); err != nil {
 		return err
 	}
 
@@ -327,124 +330,44 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
-	if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
+	if err := prepareStorageBackend(ctx, config); err != nil {
 		return err
 	}
 
 	return readTokens(runtime)
 }
 
-func prepareStorageBackend(ctx context.Context, config *config.Control) (client.Client, error) {
+func prepareStorageBackend(ctx context.Context, config *config.Control) error {
 	etcdConfig, err := endpoint.Listen(ctx, config.Storage)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	config.Storage.Config = etcdConfig.TLSConfig
 	config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
 	config.NoLeaderElect = !etcdConfig.LeaderElect
-	return client.New(etcdConfig)
-}
-
-func readTokenFile(passwdFile string) (map[string]string, error) {
-	f, err := os.Open(passwdFile)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	reader := csv.NewReader(f)
-	reader.FieldsPerRecord = -1
-
-	tokens := map[string]string{}
-
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if len(record) < 2 {
-			continue
-		}
-		tokens[record[1]] = record[0]
-	}
-	return tokens, nil
+	return nil
 }
 
 func readTokens(runtime *config.ControlRuntime) error {
-	tokens, err := readTokenFile(runtime.PasswdFile)
+	tokens, err := passwd.Read(runtime.PasswdFile)
 	if err != nil {
 		return err
 	}
 
-	if nodeToken, ok := tokens["node"]; ok {
-		runtime.NodeToken = "node:" + nodeToken
+	if nodeToken, ok := tokens.Pass("node"); ok {
+		runtime.AgentToken = "node:" + nodeToken
+	}
+	if serverToken, ok := tokens.Pass("server"); ok {
+		runtime.AgentToken = "server:" + serverToken
 	}
-	if clientToken, ok := tokens["admin"]; ok {
+	if clientToken, ok := tokens.Pass("admin"); ok {
 		runtime.ClientToken = "admin:" + clientToken
 	}
 
 	return nil
 }
 
-func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) error {
-	if config.ClusterSecret == "" {
-		return nil
-	}
-
-	f, err := os.Open(runtime.PasswdFile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	records := [][]string{}
-	reader := csv.NewReader(f)
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		if len(record) < 3 {
-			return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record))
-		}
-		if record[1] == "node" {
-			if record[0] == config.ClusterSecret {
-				return nil
-			}
-			record[0] = config.ClusterSecret
-		}
-		records = append(records, record)
-	}
-
-	f.Close()
-	return WritePasswords(runtime.PasswdFile, records)
-}
-
-func WritePasswords(passwdFile string, records [][]string) error {
-	out, err := os.Create(passwdFile + ".tmp")
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	if err := out.Chmod(0600); err != nil {
-		return err
-	}
-
-	if err := csv.NewWriter(out).WriteAll(records); err != nil {
-		return err
-	}
-
-	return os.Rename(passwdFile+".tmp", passwdFile)
-}
-
 func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
 	if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
 		psk, err := ioutil.ReadFile(runtime.IPSECKey)
@@ -455,7 +378,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
 		return nil
 	}
 
-	psk, err := getToken(ipsecTokenSize)
+	psk, err := token.Random(ipsecTokenSize)
 	if err != nil {
 		return err
 	}
@@ -468,42 +391,71 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
 	return nil
 }
 
-func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
-	if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
-		return ensureNodeToken(config, runtime)
+func migratePassword(p *passwd.Passwd) error {
+	server, _ := p.Pass("server")
+	node, _ := p.Pass("node")
+	if server == "" && node != "" {
+		return p.EnsureUser("server", "k3s:server", node)
 	}
+	return nil
+}
+
+func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
+	var (
+		err error
+	)
 
-	adminToken, err := getToken(userTokenSize)
+	serverPass := config.Token
+	if serverPass == "" {
+		serverPass, _ = passwd.Pass("server")
+	}
+	if serverPass == "" {
+		serverPass, err = token.Random(16)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return serverPass, nil
+}
+
+func getNodePass(config *config.Control, serverPass string) string {
+	if config.AgentToken == "" {
+		return serverPass
+	}
+	return config.AgentToken
+}
+
+func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
+	passwd, err := passwd.Read(runtime.PasswdFile)
 	if err != nil {
 		return err
 	}
-	systemToken, err := getToken(userTokenSize)
-	if err != nil {
+
+	if err := migratePassword(passwd); err != nil {
 		return err
 	}
-	nodeToken, err := getToken(userTokenSize)
+
+	serverPass, err := getServerPass(passwd, config)
 	if err != nil {
 		return err
 	}
 
-	if config.ClusterSecret != "" {
-		nodeToken = config.ClusterSecret
+	nodePass := getNodePass(config, serverPass)
+
+	if err := passwd.EnsureUser("admin", "system:masters", ""); err != nil {
+		return err
 	}
 
-	return WritePasswords(runtime.PasswdFile, [][]string{
-		{adminToken, "admin", "admin", "system:masters"},
-		{systemToken, "system", "system", "system:masters"},
-		{nodeToken, "node", "node", "system:masters"},
-	})
-}
+	if err := passwd.EnsureUser("node", "k3s:agent", nodePass); err != nil {
+		return err
+	}
 
-func getToken(size int) (string, error) {
-	token := make([]byte, size, size)
-	_, err := cryptorand.Read(token)
-	if err != nil {
-		return "", err
+	if err := passwd.EnsureUser("server", "k3s:server", serverPass); err != nil {
+		return err
 	}
-	return hex.EncodeToString(token), err
+
+	return passwd.Write(runtime.PasswdFile)
 }
 
 func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
@@ -578,7 +530,10 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro
 		}
 	}
 
-	if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
+	if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
+		return err
+	}
+	if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
 		return err
 	}
 

pkg/daemons/control/tunnel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/rancher/remotedialer"
@@ -37,14 +38,9 @@ func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
 		return "", false, nil
 	}
 
-	if user.GetName() != "node" {
-		return "", false, nil
-	}
-
-	nodeName := req.Header.Get("X-K3s-NodeName")
-	if nodeName == "" {
-		return "", false, nil
+	if strings.HasPrefix(user.GetName(), "system:node:") {
+		return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil
 	}
 
-	return nodeName, true, nil
+	return "", false, nil
 }

pkg/datadir/datadir.go

@@ -2,6 +2,7 @@ package datadir
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
@@ -32,5 +33,5 @@ func LocalHome(dataDir string, forceLocal bool) (string, error) {
 		return "", errors.Wrapf(err, "resolving %s", dataDir)
 	}
 
-	return dataDir, nil
+	return filepath.Abs(dataDir)
 }

pkg/passwd/passwd.go

+package passwd
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/token"
+)
+
+type entry struct {
+	pass string
+	role string
+}
+
+type Passwd struct {
+	changed bool
+	names   map[string]entry
+}
+
+func Read(file string) (*Passwd, error) {
+	result := &Passwd{
+		names: map[string]entry{},
+	}
+
+	f, err := os.Open(file)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return result, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	reader := csv.NewReader(f)
+	reader.FieldsPerRecord = -1
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if len(record) < 3 {
+			return nil, fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", file, len(record))
+		}
+		e := entry{
+			pass: record[0],
+		}
+		if len(record) > 3 {
+			e.role = record[3]
+		}
+		result.names[record[1]] = e
+	}
+
+	return result, nil
+}
+
+func (p *Passwd) Check(name, pass string) (matches bool, exists bool) {
+	e, ok := p.names[name]
+	if !ok {
+		return false, false
+	}
+	return e.pass == pass, true
+}
+
+func (p *Passwd) EnsureUser(name, role, passwd string) error {
+	tokenPrefix := "::" + name + ":"
+	idx := strings.Index(passwd, tokenPrefix)
+	if idx > 0 && strings.HasPrefix(passwd, "K10") {
+		passwd = passwd[idx+len(tokenPrefix):]
+	}
+
+	if e, ok := p.names[name]; ok {
+		if passwd != "" && e.pass != passwd {
+			p.changed = true
+			e.pass = passwd
+		}
+
+		if e.role != role {
+			p.changed = true
+			e.role = role
+		}
+
+		p.names[name] = e
+		return nil
+	}
+
+	if passwd == "" {
+		token, err := token.Random(16)
+		if err != nil {
+			return err
+		}
+		passwd = token
+	}
+
+	p.changed = true
+	p.names[name] = entry{
+		pass: passwd,
+		role: role,
+	}
+
+	return nil
+}
+
+func (p *Passwd) Pass(name string) (string, bool) {
+	e, ok := p.names[name]
+	if !ok {
+		return "", false
+	}
+	return e.pass, true
+}
+
+func (p *Passwd) Write(passwdFile string) error {
+	if !p.changed {
+		return nil
+	}
+
+	var records [][]string
+	for name, e := range p.names {
+		records = append(records, []string{
+			e.pass,
+			name,
+			name,
+			e.role,
+		})
+	}
+
+	return writePasswords(passwdFile, records)
+}
+
+func writePasswords(passwdFile string, records [][]string) error {
+	out, err := os.Create(passwdFile + ".tmp")
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	if err := out.Chmod(0600); err != nil {
+		return err
+	}
+
+	if err := csv.NewWriter(out).WriteAll(records); err != nil {
+		return err
+	}
+
+	return os.Rename(passwdFile+".tmp", passwdFile)
+}

pkg/server/auth.go

@@ -9,9 +9,21 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
-func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
+func hasRole(mustRoles []string, roles []string) bool {
+	for _, check := range roles {
+		for _, role := range mustRoles {
+			if role == check {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
 	if serverConfig == nil || serverConfig.Runtime.Authenticator == nil {
-		next.ServeHTTP(rw, req)
+		logrus.Errorf("authenticate not initialized")
+		rw.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
@@ -22,7 +34,7 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
 		return
 	}
 
-	if !ok || resp.User.GetName() != "node" {
+	if !ok || !hasRole(roles, resp.User.GetGroups()) {
 		rw.WriteHeader(http.StatusUnauthorized)
 		return
 	}
@@ -32,10 +44,10 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
 	next.ServeHTTP(rw, req)
 }
 
-func authMiddleware(serverConfig *config.Control) mux.MiddlewareFunc {
+func authMiddleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-			doAuth(serverConfig, next, rw, req)
+			doAuth(roles, serverConfig, next, rw, req)
 		})
 	}
 }

pkg/server/context.go

@@ -62,7 +62,6 @@ func crds(ctx context.Context, config *rest.Config) error {
 	}
 
 	factory.BatchCreateCRDs(ctx, crd.NamespacedTypes(
-		"ListenerConfig.k3s.cattle.io/v1",
 		"Addon.k3s.cattle.io/v1",
 		"HelmChart.helm.cattle.io/v1")...)
 

pkg/server/router.go

@@ -3,70 +3,68 @@ package server
 import (
 	"crypto"
 	"crypto/x509"
-	"encoding/csv"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
 	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/k3s/pkg/daemons/control"
+	"github.com/rancher/k3s/pkg/passwd"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 )
 
 const (
-	jsonMediaType   = "application/json"
-	binaryMediaType = "application/octet-stream"
-	pbMediaType     = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf"
-	openapiPrefix   = "openapi."
-	staticURL       = "/static/"
+	staticURL = "/static/"
 )
 
-type CACertsGetter func() (string, error)
-
-func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CACertsGetter) http.Handler {
+func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.Handler {
 	authed := mux.NewRouter()
-	authed.Use(authMiddleware(serverConfig))
+	authed.Use(authMiddleware(serverConfig, "k3s:agent"))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
-	authed.Path("/v1-k3s/connect").Handler(tunnel)
 	authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
 	authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
 	authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
 	authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
 	authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
 	authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
+	authed.Path("/v1-k3s/client-k3s-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert))
+	authed.Path("/v1-k3s/client-k3s-controller.key").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
 	authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
 
+	nodeAuthed := mux.NewRouter()
+	nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))
+	nodeAuthed.Path("/v1-k3s/connect").Handler(tunnel)
+	nodeAuthed.NotFoundHandler = authed
+
+	serverAuthed := mux.NewRouter()
+	serverAuthed.Use(authMiddleware(serverConfig, "k3s:server"))
+	serverAuthed.NotFoundHandler = nodeAuthed
+	serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap))
+
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
 	router := mux.NewRouter()
-	router.NotFoundHandler = authed
+	router.NotFoundHandler = serverAuthed
 	router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
-	router.Path("/cacerts").Handler(cacerts(cacertsGetter))
+	router.Path("/cacerts").Handler(cacerts(ca))
 	router.Path("/ping").Handler(ping())
 
 	return router
 }
 
-func cacerts(getter CACertsGetter) http.Handler {
+func cacerts(ca []byte) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		content, err := getter()
-		if err != nil {
-			resp.WriteHeader(http.StatusInternalServerError)
-			resp.Write([]byte(err.Error()))
-		}
 		resp.Header().Set("content-type", "text/plain")
-		resp.Write([]byte(content))
+		resp.Write(ca)
 	})
 }
 
@@ -242,38 +240,19 @@ func sendError(err error, resp http.ResponseWriter, status ...int) {
 	resp.Write([]byte(err.Error()))
 }
 
-func ensureNodePassword(passwdFile, nodeName, passwd string) error {
-	records := [][]string{}
-
-	if _, err := os.Stat(passwdFile); !os.IsNotExist(err) {
-		f, err := os.Open(passwdFile)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		reader := csv.NewReader(f)
-		for {
-			record, err := reader.Read()
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				return err
-			}
-			if len(record) < 2 {
-				return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record))
-			}
-			if record[1] == nodeName {
-				if record[0] == passwd {
-					return nil
-				}
-				return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
-			}
-			records = append(records, record)
+func ensureNodePassword(passwdFile, nodeName, pass string) error {
+	passwd, err := passwd.Read(passwdFile)
+	if err != nil {
+		return err
+	}
+	match, exists := passwd.Check(nodeName, pass)
+	if exists {
+		if !match {
+			return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
 		}
-		f.Close()
+		return nil
 	}
-
-	records = append(records, []string{passwd, nodeName})
-	return control.WritePasswords(passwdFile, records)
+	// If user doesn't exist we save this password for future validation
+	passwd.EnsureUser(nodeName, "", pass)
+	return passwd.Write(passwdFile)
 }

pkg/server/server.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/helm-controller/pkg/helm"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -25,10 +24,11 @@ import (
 	"github.com/rancher/k3s/pkg/rootlessports"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
-	"github.com/rancher/k3s/pkg/tls"
+	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
@@ -39,79 +39,67 @@ func resolveDataDir(dataDir string) (string, error) {
 	return filepath.Join(dataDir, "server"), err
 }
 
-func StartServer(ctx context.Context, config *Config) (string, error) {
+func StartServer(ctx context.Context, config *Config) error {
 	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
-		return "", err
+		return err
 	}
 
 	if err := setNoProxyEnv(&config.ControlConfig); err != nil {
-		return "", err
+		return err
 	}
 
 	if err := control.Server(ctx, &config.ControlConfig); err != nil {
-		return "", errors.Wrap(err, "starting kubernetes")
+		return errors.Wrap(err, "starting kubernetes")
 	}
 
-	certs, err := startWrangler(ctx, config)
-	if err != nil {
-		return "", errors.Wrap(err, "starting tls server")
+	if err := startWrangler(ctx, config); err != nil {
+		return errors.Wrap(err, "starting tls server")
 	}
 
-	ip := net2.ParseIP(config.TLSConfig.BindAddress)
+	ip := net2.ParseIP(config.ControlConfig.BindAddress)
 	if ip == nil {
-		ip, err = net.ChooseHostInterface()
-		if err != nil {
+		hostIP, err := net.ChooseHostInterface()
+		if err == nil {
+			ip = hostIP
+		} else {
 			ip = net2.ParseIP("127.0.0.1")
 		}
 	}
-	printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
 
-	writeKubeConfig(certs, &config.TLSConfig, config)
+	if err := printTokens(ip.String(), &config.ControlConfig); err != nil {
+		return err
+	}
 
-	return certs, nil
+	return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
 }
 
-func startWrangler(ctx context.Context, config *Config) (string, error) {
+func startWrangler(ctx context.Context, config *Config) error {
 	var (
 		err           error
-		tlsConfig     = &config.TLSConfig
 		controlConfig = &config.ControlConfig
 	)
 
-	caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
+	ca, err := ioutil.ReadFile(config.ControlConfig.Runtime.ServerCA)
 	if err != nil {
-		return "", err
-	}
-	caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
-	if err != nil {
-		return "", err
+		return err
 	}
 
-	certs := string(caBytes)
-	tlsConfig.CACerts = certs
-	tlsConfig.CAKey = string(caKeyBytes)
-
-	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
-		return certs, nil
-	})
+	controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca)
 
 	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	if err := stageFiles(ctx, sc, controlConfig); err != nil {
-		return "", err
+		return err
 	}
 
-	_, err = tls.NewServer(ctx, sc.K3s.K3s().V1().ListenerConfig(), *tlsConfig)
-	if err != nil {
-		return "", err
+	if err := sc.Start(ctx); err != nil {
+		return err
 	}
 
-	if err := startNodeCache(ctx, sc); err != nil {
-		return "", err
-	}
+	controlConfig.Runtime.Core = sc.Core
 
 	start := func(ctx context.Context) {
 		if err := masterControllers(ctx, sc, config); err != nil {
@@ -122,7 +110,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		}
 	}
 	if !config.DisableAgent {
-		go setMasterRoleLabel(ctx, sc)
+		go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node())
 	}
 	if controlConfig.NoLeaderElect {
 		go func() {
@@ -134,7 +122,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		go leader.RunOrDie(ctx, "", "k3s", sc.K8s, start)
 	}
 
-	return certs, nil
+	return nil
 }
 
 func masterControllers(ctx context.Context, sc *Context, config *Config) error {
@@ -162,7 +150,7 @@ func masterControllers(ctx context.Context, sc *Context, config *Config) error {
 	}
 
 	if !config.DisableServiceLB && config.Rootless {
-		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.TLSConfig.HTTPSPort)
+		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.ControlConfig.HTTPSPort)
 	}
 
 	return nil
@@ -203,7 +191,7 @@ func HomeKubeConfig(write, rootless bool) (string, error) {
 	return resolvehome.Resolve(datadir.HomeConfig)
 }
 
-func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
+func printTokens(advertiseIP string, config *config.Control) error {
 	var (
 		nodeFile string
 	)
@@ -212,26 +200,43 @@ func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfi
 		advertiseIP = "127.0.0.1"
 	}
 
-	if len(config.Runtime.NodeToken) > 0 {
-		p := filepath.Join(config.DataDir, "node-token")
-		if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
+	if len(config.Runtime.AgentToken) > 0 {
+		p := filepath.Join(config.DataDir, "token")
+		if err := writeToken(config.Runtime.AgentToken, p, config.Runtime.ServerCA); err == nil {
 			logrus.Infof("Node token is available at %s", p)
 			nodeFile = p
 		}
+
+		// backwards compatibility
+		np := filepath.Join(config.DataDir, "node-token")
+		if !isSymlink(np) {
+			if err := os.RemoveAll(np); err != nil {
+				return err
+			}
+			if err := os.Symlink(p, np); err != nil {
+				return err
+			}
+		}
 	}
 
 	if len(nodeFile) > 0 {
-		printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
+		printToken(config.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
 	}
+
+	return nil
 }
 
-func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
-	clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
-	ip := tlsConfig.BindAddress
+func writeKubeConfig(certs string, config *Config) error {
+	clientToken, err := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
+	if err != nil {
+		return err
+	}
+
+	ip := config.ControlConfig.BindAddress
 	if ip == "" {
 		ip = "127.0.0.1"
 	}
-	url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
+	url := fmt.Sprintf("https://%s:%d", ip, config.ControlConfig.HTTPSPort)
 	kubeConfig, err := HomeKubeConfig(true, config.Rootless)
 	def := true
 	if err != nil {
@@ -274,6 +279,8 @@ func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config
 	if def {
 		logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
 	}
+
+	return nil
 }
 
 func setupDataDirAndChdir(config *config.Control) error {
@@ -312,18 +319,22 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
 	logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
 }
 
-func FormatToken(token string, certs string) string {
+func FormatToken(token string, certFile string) (string, error) {
 	if len(token) == 0 {
-		return token
+		return token, nil
 	}
 
 	prefix := "K10"
-	if len(certs) > 0 {
-		digest := sha256.Sum256([]byte(certs))
+	if len(certFile) > 0 {
+		bytes, err := ioutil.ReadFile(certFile)
+		if err != nil {
+			return "", nil
+		}
+		digest := sha256.Sum256(bytes)
 		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
 	}
 
-	return prefix + token
+	return prefix + token, nil
 }
 
 func writeToken(token, file, certs string) error {
@@ -331,7 +342,10 @@ func writeToken(token, file, certs string) error {
 		return nil
 	}
 
-	token = FormatToken(token, certs)
+	token, err := FormatToken(token, certs)
+	if err != nil {
+		return err
+	}
 	return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
 }
 
@@ -364,26 +378,23 @@ func isSymlink(config string) bool {
 	return false
 }
 
-func setMasterRoleLabel(ctx context.Context, sc *Context) error {
+func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 	for {
 		nodeName := os.Getenv("NODE_NAME")
-		nodeController := sc.Core.Core().V1().Node()
-		nodeCache := nodeController.Cache()
-		nodeCached, err := nodeCache.Get(nodeName)
+		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err != nil {
 			logrus.Infof("Waiting for master node %s startup: %v", nodeName, err)
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		if v, ok := nodeCached.Labels[MasterRoleLabelKey]; ok && v == "true" {
+		if v, ok := node.Labels[MasterRoleLabelKey]; ok && v == "true" {
 			break
 		}
-		node := nodeCached.DeepCopy()
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
 		node.Labels[MasterRoleLabelKey] = "true"
-		_, err = nodeController.Update(node)
+		_, err = nodes.Update(node)
 		if err == nil {
 			logrus.Infof("master role label has been set succesfully on node: %s", nodeName)
 			break
@@ -396,9 +407,3 @@ func setMasterRoleLabel(ctx context.Context, sc *Context) error {
 	}
 	return nil
 }
-
-func startNodeCache(ctx context.Context, sc *Context) error {
-	sc.Core.Core().V1().Node().Cache()
-
-	return sc.Start(ctx)
-}

pkg/server/types.go

 package server
 
 import (
-	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/k3s/pkg/daemons/config"
 )
 
 type Config struct {
 	DisableAgent     bool
 	DisableServiceLB bool
-	TLSConfig        dynamiclistener.UserConfig
 	ControlConfig    config.Control
 	Rootless         bool
 }

pkg/token/read.go

+package token
+
+import (
+	cryptorand "crypto/rand"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func Random(size int) (string, error) {
+	token := make([]byte, size, size)
+	_, err := cryptorand.Read(token)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(token), err
+}
+
+func ReadFile(path string) (string, error) {
+	if path == "" {
+		return "", nil
+	}
+
+	for {
+		tokenBytes, err := ioutil.ReadFile(path)
+		if err == nil {
+			return strings.TrimSpace(string(tokenBytes)), nil
+		} else if os.IsNotExist(err) {
+			logrus.Infof("Waiting for %s to be available\n", path)
+			time.Sleep(2 * time.Second)
+		} else {
+			return "", err
+		}
+	}
+}