Merge pull request #995 from ibuildthecloud/master

go.mod

@@ -97,7 +97,7 @@ require (
 	github.com/opencontainers/runc v1.0.0-rc2.0.20190611121236-6cc515888830
 	github.com/pkg/errors v0.8.1
 	github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect
-	github.com/rancher/dynamiclistener v0.1.1-0.20191010011134-8a2488bc860a
+	github.com/rancher/dynamiclistener v0.1.1-0.20191031022009-6224794ef3cb
 	github.com/rancher/helm-controller v0.2.2
 	github.com/rancher/kine v0.0.0-00010101000000-000000000000
 	github.com/rancher/remotedialer v0.2.0

go.sum

@@ -416,9 +416,8 @@ github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
 github.com/knative/build v0.6.0/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo=
 github.com/knative/pkg v0.0.0-20190514205332-5e4512dcb2ca/go.mod h1:7Ijfhw7rfB+H9VtosIsDYvZQ+qYTz7auK3fHW/5z4ww=
 github.com/knative/serving v0.6.1/go.mod h1:ljvMfwQy2qanaM/8xnBSK4Mz3Vv2NawC2fo5kFRJS1A=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -563,8 +562,8 @@ github.com/rancher/cri v1.3.0-k3s.2 h1:k2XFyD+ZdsGvNfugdvqD38KSMANT3JmTFULFM2CtI
 github.com/rancher/cri v1.3.0-k3s.2/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY=
 github.com/rancher/cri-tools v1.16.1-k3s.1 h1:iporgQ46noE6dtLzq6fWcIO2qjyPZy2m42d2P+UnGJg=
 github.com/rancher/cri-tools v1.16.1-k3s.1/go.mod h1:TEKhKv2EJIZp+p9jnEy4C63g8CosJzsI4kyKKkHag+8=
-github.com/rancher/dynamiclistener v0.1.1-0.20191010011134-8a2488bc860a h1:1bUYAv5U/Ky4YJ9o8gWxX+vNcjpIL3JWNBao70OlkFE=
-github.com/rancher/dynamiclistener v0.1.1-0.20191010011134-8a2488bc860a/go.mod h1:8hbGf35mB7ormKEFqsAgjgeI5rLbj5N764jG41dNhps=
+github.com/rancher/dynamiclistener v0.1.1-0.20191031022009-6224794ef3cb h1:bMoA9UHr1QNTWVrf0fSJCba6YDU1xmt2jmeohpiugKg=
+github.com/rancher/dynamiclistener v0.1.1-0.20191031022009-6224794ef3cb/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s=
 github.com/rancher/flannel v0.11.0-k3s.1 h1:mIwnfWDafjzQgFkZeJ1AkFrrAT3EdBaA1giE0eLJKo8=
 github.com/rancher/flannel v0.11.0-k3s.1/go.mod h1:Hn4ZV+eq0LhLZP63xZnxdGwXEoRSxs5sxELxu27M3UA=
 github.com/rancher/helm-controller v0.2.2 h1:MUqisy53/Ay1EYOF2uTCYBbGpgtZLNKKrI01BdxIbQo=
@@ -682,7 +681,6 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stripe/safesql v0.2.0/go.mod h1:q7b2n0JmzM1mVGfcYpanfVb2j23cXZeWFxcILPn3JV4=
 github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -741,7 +739,6 @@ golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -808,7 +805,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190730183949-1393eb018365/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo=

manifests/rolebindings.yaml

@@ -10,3 +10,53 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: kube-apiserver
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:k3s-controller
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - "networking.k8s.io"
+  resources:
+  - networkpolicies
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  - pods
+  verbs:
+  - list
+  - get
+  - watch
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:k3s-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:k3s-controller
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: User
+    name: system:k3s-controller

pkg/agent/config/config.go

@@ -6,7 +6,6 @@ import (
 	cryptorand "crypto/rand"
 	"crypto/tls"
 	"encoding/hex"
-	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	sysnet "net"
@@ -27,7 +26,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/client-go/util/cert"
 	"k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
 )
 
@@ -183,17 +181,6 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
 	return name, ip, nil
 }
 
-func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) {
-	os.MkdirAll(envInfo.DataDir, 0700)
-	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
-	info.CACerts = pem.EncodeToMemory(&pem.Block{
-		Type:  cert.CertificateBlockType,
-		Bytes: tlsCert.Certificate[1],
-	})
-
-	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
-}
-
 func isValidResolvConf(resolvConfFile string) bool {
 	file, err := os.Open(resolvConfFile)
 	if err != nil {
@@ -293,11 +280,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
-	kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert)
-	if err != nil {
-		return nil, err
-	}
-
 	clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
 	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
 		return nil, err
@@ -328,6 +310,21 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
+	clientK3sControllerCert := filepath.Join(envInfo.DataDir, "client-k3s-controller.crt")
+	if err := getHostFile(clientK3sControllerCert, info); err != nil {
+		return nil, err
+	}
+
+	clientK3sControllerKey := filepath.Join(envInfo.DataDir, "client-k3s-controller.key")
+	if err := getHostFile(clientK3sControllerKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigK3sController := filepath.Join(envInfo.DataDir, "k3scontroller.kubeconfig")
+	if err := control.KubeConfig(kubeconfigK3sController, info.URL, serverCAFile, clientK3sControllerCert, clientK3sControllerKey); err != nil {
+		return nil, err
+	}
+
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
@@ -345,9 +342,9 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.ClientCA = clientCAFile
 	nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
-	nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
+	nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController
 	if envInfo.Rootless {
 		nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	}

pkg/agent/flannel/setup.go

@@ -13,8 +13,7 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/clientcmd"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
 const (
@@ -74,21 +73,11 @@ func Prepare(ctx context.Context, nodeConfig *config.Node) error {
 	return createFlannelConf(nodeConfig)
 }
 
-func Run(ctx context.Context, nodeConfig *config.Node) error {
+func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) error {
 	nodeName := nodeConfig.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
-	if err != nil {
-		return err
-	}
-
-	client, err := kubernetes.NewForConfig(restConfig)
-	if err != nil {
-		return err
-	}
-
 	for {
-		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err == nil && node.Spec.PodCIDR != "" {
 			break
 		}
@@ -101,11 +90,11 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 
-	return err
+	return nil
 }
 
 func createCNIConf(dir string) error {

pkg/agent/netpol/network_policy.go

@@ -10,7 +10,7 @@ import (
 )
 
 func Run(ctx context.Context, nodeConfig *config.Node) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController)
 	if err != nil {
 		return err
 	}

pkg/agent/run.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/rancher/k3s/pkg/agent/config"
 	"github.com/rancher/k3s/pkg/agent/containerd"
 	"github.com/rancher/k3s/pkg/agent/flannel"
@@ -21,10 +22,11 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/agent"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/rootless"
-	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
-	corev1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
-	"github.com/rancher/wrangler/pkg/start"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -64,14 +66,18 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 		return err
 	}
 
+	coreClient, err := coreClient(nodeConfig.AgentConfig.KubeConfigKubelet)
+	if err != nil {
+		return err
+	}
 	if !nodeConfig.NoFlannel {
-		if err := flannel.Run(ctx, nodeConfig); err != nil {
+		if err := flannel.Run(ctx, nodeConfig, coreClient.CoreV1().Nodes()); err != nil {
 			return err
 		}
 	}
 
 	if !nodeConfig.AgentConfig.DisableCCM {
-		if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig); err != nil {
+		if err := syncAddressesLabels(ctx, &nodeConfig.AgentConfig, coreClient.CoreV1().Nodes()); err != nil {
 			return err
 		}
 	}
@@ -86,6 +92,15 @@ func run(ctx context.Context, cfg cmds.Agent, lb *loadbalancer.LoadBalancer) err
 	return ctx.Err()
 }
 
+func coreClient(cfg string) (kubernetes.Interface, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return kubernetes.NewForConfig(restConfig)
+}
+
 func Run(ctx context.Context, cfg cmds.Agent) error {
 	if err := validate(); err != nil {
 		return err
@@ -100,10 +115,6 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 	cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
 	os.MkdirAll(cfg.DataDir, 0700)
 
-	if cfg.ClusterSecret != "" {
-		cfg.Token = "K10node:" + cfg.ClusterSecret
-	}
-
 	lb, err := loadbalancer.Setup(ctx, cfg)
 	if err != nil {
 		return err
@@ -113,7 +124,7 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 	}
 
 	for {
-		tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token)
+		newToken, err := clientaccess.NormalizeAndValidateTokenForUser(cfg.ServerURL, cfg.Token, "node")
 		if err != nil {
 			logrus.Error(err)
 			select {
@@ -123,10 +134,11 @@ func Run(ctx context.Context, cfg cmds.Agent) error {
 			}
 			continue
 		}
-		os.Remove(tmpFile)
+		cfg.Token = newToken
 		break
 	}
 
+	systemd.SdNotify(true, "READY=1\n")
 	return run(ctx, cfg, lb)
 }
 
@@ -149,73 +161,51 @@ func validate() error {
 	return nil
 }
 
-func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent) error {
+func syncAddressesLabels(ctx context.Context, agentConfig *daemonconfig.Agent, nodes v1.NodeInterface) error {
 	for {
-		nodeController, nodeCache, err := startNodeController(ctx, agentConfig)
-		if err != nil {
-			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		nodeCached, err := nodeCache.Get(agentConfig.NodeName)
+		node, err := nodes.Get(agentConfig.NodeName, metav1.GetOptions{})
 		if err != nil {
 			logrus.Infof("Waiting for kubelet to be ready on node %s: %v", agentConfig.NodeName, err)
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		node := nodeCached.DeepCopy()
-		updated := updateLabelMap(ctx, agentConfig, node.Labels)
-		if updated {
-			_, err = nodeController.Update(node)
-			if err == nil {
-				logrus.Infof("addresses labels has been set succesfully on node: %s", agentConfig.NodeName)
-				break
+
+		newLabels, update := updateLabelMap(agentConfig, node.Labels)
+		if update {
+			node.Labels = newLabels
+			if _, err := nodes.Update(node); err != nil {
+				logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				case <-time.After(time.Second):
+					continue
+				}
 			}
-			logrus.Infof("Failed to update node %s: %v", agentConfig.NodeName, err)
-			time.Sleep(1 * time.Second)
-			continue
+			logrus.Infof("addresses labels has been set successfully on node: %s", agentConfig.NodeName)
+		} else {
+			logrus.Infof("addresses labels has already been set successfully on node: %s", agentConfig.NodeName)
 		}
-		logrus.Infof("addresses labels has already been set succesfully on node: %s", agentConfig.NodeName)
-		return nil
-	}
-	return nil
-}
 
-func startNodeController(ctx context.Context, agentConfig *daemonconfig.Agent) (corev1.NodeController, corev1.NodeCache, error) {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", agentConfig.KubeConfigKubelet)
-	if err != nil {
-		return nil, nil, err
-	}
-	coreFactory := core.NewFactoryFromConfigOrDie(restConfig)
-	nodeController := coreFactory.Core().V1().Node()
-	nodeCache := nodeController.Cache()
-	if err := start.All(ctx, 1, coreFactory); err != nil {
-		return nil, nil, err
+		break
 	}
 
-	return nodeController, nodeCache, nil
+	return nil
 }
 
-func updateLabelMap(ctx context.Context, agentConfig *daemonconfig.Agent, nodeLabels map[string]string) bool {
-	if nodeLabels == nil {
-		nodeLabels = make(map[string]string)
-	}
-	updated := false
-	if internalIPLabel, ok := nodeLabels[InternalIPLabel]; !ok || internalIPLabel != agentConfig.NodeIP {
-		nodeLabels[InternalIPLabel] = agentConfig.NodeIP
-		updated = true
-	}
-	if hostnameLabel, ok := nodeLabels[HostnameLabel]; !ok || hostnameLabel != agentConfig.NodeName {
-		nodeLabels[HostnameLabel] = agentConfig.NodeName
-		updated = true
+func updateLabelMap(agentConfig *daemonconfig.Agent, nodeLabels map[string]string) (map[string]string, bool) {
+	result := map[string]string{}
+	for k, v := range nodeLabels {
+		result[k] = v
 	}
-	nodeExternalIP := agentConfig.NodeExternalIP
-	if externalIPLabel := nodeLabels[ExternalIPLabel]; externalIPLabel != nodeExternalIP && nodeExternalIP != "" {
-		nodeLabels[ExternalIPLabel] = nodeExternalIP
-		updated = true
-	} else if nodeExternalIP == "" && externalIPLabel != "" {
-		delete(nodeLabels, ExternalIPLabel)
-		updated = true
+
+	result[InternalIPLabel] = agentConfig.NodeIP
+	result[HostnameLabel] = agentConfig.NodeName
+	if agentConfig.NodeExternalIP == "" {
+		delete(result, ExternalIPLabel)
+	} else {
+		result[ExternalIPLabel] = agentConfig.NodeExternalIP
 	}
-	return updated
+
+	return result, !equality.Semantic.DeepEqual(nodeLabels, result)
 }

pkg/agent/tunnel/tunnel.go

@@ -3,11 +3,8 @@ package tunnel
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
-	"encoding/base64"
 	"fmt"
 	"net"
-	"net/http"
 	"reflect"
 	"sync"
 	"time"
@@ -21,8 +18,8 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	watchtypes "k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/transport"
 )
 
 var (
@@ -54,17 +51,22 @@ func getAddresses(endpoint *v1.Endpoints) []string {
 }
 
 func Setup(ctx context.Context, config *config.Node, onChange func([]string)) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigK3sController)
 	if err != nil {
 		return err
 	}
 
-	transportConfig, err := restConfig.TransportConfig()
+	client, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return err
 	}
 
-	client, err := kubernetes.NewForConfig(restConfig)
+	nodeRestConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigKubelet)
+	if err != nil {
+		return err
+	}
+
+	tlsConfig, err := rest.TLSConfigFor(nodeRestConfig)
 	if err != nil {
 		return err
 	}
@@ -84,7 +86,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 	wg := &sync.WaitGroup{}
 	for _, address := range addresses {
 		if _, ok := disconnect[address]; !ok {
-			disconnect[address] = connect(ctx, wg, address, config, transportConfig)
+			disconnect[address] = connect(ctx, wg, address, tlsConfig)
 		}
 	}
 
@@ -132,7 +134,7 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 					for _, address := range addresses {
 						validEndpoint[address] = true
 						if _, ok := disconnect[address]; !ok {
-							disconnect[address] = connect(ctx, nil, address, config, transportConfig)
+							disconnect[address] = connect(ctx, nil, address, tlsConfig)
 						}
 					}
 
@@ -164,25 +166,10 @@ func Setup(ctx context.Context, config *config.Node, onChange func([]string)) er
 	return nil
 }
 
-func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc {
+func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string, tlsConfig *tls.Config) context.CancelFunc {
 	wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address)
-	headers := map[string][]string{
-		"X-K3s-NodeName": {config.AgentConfig.NodeName},
-	}
-	ws := &websocket.Dialer{}
-
-	if len(config.CACerts) > 0 {
-		pool := x509.NewCertPool()
-		pool.AppendCertsFromPEM(config.CACerts)
-		ws.TLSClientConfig = &tls.Config{
-			RootCAs: pool,
-		}
-	}
-
-	if transportConfig.Username != "" {
-		auth := transportConfig.Username + ":" + transportConfig.Password
-		auth = base64.StdEncoding.EncodeToString([]byte(auth))
-		headers["Authorization"] = []string{"Basic " + auth}
+	ws := &websocket.Dialer{
+		TLSClientConfig: tlsConfig,
 	}
 
 	once := sync.Once{}
@@ -194,7 +181,7 @@ func connect(rootCtx context.Context, waitGroup *sync.WaitGroup, address string,
 
 	go func() {
 		for {
-			remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool {
+			remotedialer.ClientConnect(ctx, wsURL, nil, ws, func(proto, address string) bool {
 				host, port, err := net.SplitHostPort(address)
 				return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
 			}, func(_ context.Context) error {

pkg/apis/k3s.cattle.io/v1/types.go

 package v1
 
 import (
-	"github.com/rancher/dynamiclistener"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 )
@@ -9,16 +8,6 @@ import (
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-type ListenerConfig struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Status dynamiclistener.ListenerStatus `json:"status,omitempty"`
-}
-
-// +genclient
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
 type Addon struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go

@@ -122,63 +122,3 @@ func (in *AddonStatus) DeepCopy() *AddonStatus {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ListenerConfig) DeepCopyInto(out *ListenerConfig) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	in.Status.DeepCopyInto(&out.Status)
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListenerConfig.
-func (in *ListenerConfig) DeepCopy() *ListenerConfig {
-	if in == nil {
-		return nil
-	}
-	out := new(ListenerConfig)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *ListenerConfig) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ListenerConfigList) DeepCopyInto(out *ListenerConfigList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]ListenerConfig, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListenerConfigList.
-func (in *ListenerConfigList) DeepCopy() *ListenerConfigList {
-	if in == nil {
-		return nil
-	}
-	out := new(ListenerConfigList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *ListenerConfigList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}

pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go

@@ -26,23 +26,6 @@ import (
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// ListenerConfigList is a list of ListenerConfig resources
-type ListenerConfigList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata"`
-
-	Items []ListenerConfig `json:"items"`
-}
-
-func NewListenerConfig(namespace, name string, obj ListenerConfig) *ListenerConfig {
-	obj.APIVersion, obj.Kind = SchemeGroupVersion.WithKind("ListenerConfig").ToAPIVersionAndKind()
-	obj.Name = name
-	obj.Namespace = namespace
-	return &obj
-}
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
 // AddonList is a list of Addon resources
 type AddonList struct {
 	metav1.TypeMeta `json:",inline"`

pkg/apis/k3s.cattle.io/v1/zz_generated_register.go

@@ -28,8 +28,7 @@ import (
 )
 
 var (
-	AddonResourceName          = "addons"
-	ListenerConfigResourceName = "listenerconfigs"
+	AddonResourceName = "addons"
 )
 
 // SchemeGroupVersion is group version used to register these objects
@@ -55,8 +54,6 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&Addon{},
 		&AddonList{},
-		&ListenerConfig{},
-		&ListenerConfigList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/daemons/control/bootstrap.go → pkg/bootstrap/bootstrap.go

-package control
+package bootstrap
 
 import (
-	"context"
 	"encoding/json"
+	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/kine/pkg/client"
-	"github.com/sirupsen/logrus"
 )
 
-const (
-	k3sRuntimeEtcdPath = "/k3s/runtime"
-)
+func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("Content-Type", "application/json")
+		Write(rw, bootstrap)
+	})
+}
 
-// fetchBootstrapData copies the bootstrap data (certs, keys, passwords)
-// from etcd to individual files specified by cfg.Runtime.
-func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Client) error {
-	logrus.Info("Fetching bootstrap data from etcd")
-	gr, err := c.Get(ctx, k3sRuntimeEtcdPath)
+func Write(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {
+	paths, err := objToMap(bootstrap)
 	if err != nil {
-		return err
-	}
-	if gr.Modified == 0 {
 		return nil
 	}
 
-	paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
+	dataMap := map[string][]byte{}
+	for pathKey, path := range paths {
+		if path == "" {
+			continue
+		}
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read %s", path)
+		}
+
+		dataMap[pathKey] = data
+	}
+
+	return json.NewEncoder(w).Encode(dataMap)
+}
+
+func Read(r io.Reader, bootstrap *config.ControlRuntimeBootstrap) error {
+	paths, err := objToMap(bootstrap)
 	if err != nil {
 		return err
 	}
 
 	files := map[string][]byte{}
-	if err := json.Unmarshal(gr.Data, &files); err != nil {
+	if err := json.NewDecoder(r).Decode(&files); err != nil {
 		return err
 	}
 
@@ -49,7 +62,7 @@ func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Clien
 			return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
 		}
 
-		if err := ioutil.WriteFile(path, data, 0700); err != nil {
+		if err := ioutil.WriteFile(path, data, 0600); err != nil {
 			return errors.Wrapf(err, "failed to write to %s", path)
 		}
 	}
@@ -57,39 +70,6 @@ func fetchBootstrapData(ctx context.Context, cfg *config.Control, c client.Clien
 	return nil
 }
 
-// storeBootstrapData copies the bootstrap data in the opposite direction to
-// fetchBootstrapData.
-func storeBootstrapData(ctx context.Context, cfg *config.Control, client client.Client) error {
-	if cfg.BootstrapReadOnly {
-		return nil
-	}
-
-	paths, err := objToMap(&cfg.Runtime.ControlRuntimeBootstrap)
-	if err != nil {
-		return nil
-	}
-
-	dataMap := map[string][]byte{}
-	for pathKey, path := range paths {
-		if path == "" {
-			continue
-		}
-		data, err := ioutil.ReadFile(path)
-		if err != nil {
-			return errors.Wrapf(err, "failed to read %s", path)
-		}
-
-		dataMap[pathKey] = data
-	}
-
-	bytes, err := json.Marshal(dataMap)
-	if err != nil {
-		return err
-	}
-
-	return client.Put(ctx, k3sRuntimeEtcdPath, bytes)
-}
-
 func objToMap(obj interface{}) (map[string]string, error) {
 	bytes, err := json.Marshal(obj)
 	if err != nil {

pkg/cli/agent/agent.go

@@ -3,39 +3,18 @@ package agent
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"strings"
-	"time"
 
-	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/rancher/k3s/pkg/agent"
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/datadir"
 	"github.com/rancher/k3s/pkg/netutil"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
-func readToken(path string) (string, error) {
-	if path == "" {
-		return "", nil
-	}
-
-	for {
-		tokenBytes, err := ioutil.ReadFile(path)
-		if err == nil {
-			return strings.TrimSpace(string(tokenBytes)), nil
-		} else if os.IsNotExist(err) {
-			logrus.Infof("Waiting for %s to be available\n", path)
-			time.Sleep(2 * time.Second)
-		} else {
-			return "", err
-		}
-	}
-}
-
 func Run(ctx *cli.Context) error {
 	if err := cmds.InitLogging(); err != nil {
 		return err
@@ -45,14 +24,14 @@ func Run(ctx *cli.Context) error {
 	}
 
 	if cmds.AgentConfig.TokenFile != "" {
-		token, err := readToken(cmds.AgentConfig.TokenFile)
+		token, err := token.ReadFile(cmds.AgentConfig.TokenFile)
 		if err != nil {
 			return err
 		}
 		cmds.AgentConfig.Token = token
 	}
 
-	if cmds.AgentConfig.Token == "" && cmds.AgentConfig.ClusterSecret == "" {
+	if cmds.AgentConfig.Token == "" {
 		return fmt.Errorf("--token is required")
 	}
 
@@ -76,7 +55,6 @@ func Run(ctx *cli.Context) error {
 	cfg.DataDir = dataDir
 
 	contextCtx := signals.SetupSignalHandler(context.Background())
-	systemd.SdNotify(true, "READY=1\n")
 
 	return agent.Run(contextCtx, cfg)
 }

pkg/cli/cmds/agent.go

@@ -17,7 +17,6 @@ type Agent struct {
 	NodeIP                   string
 	NodeExternalIP           string
 	NodeName                 string
-	ClusterSecret            string
 	PauseImage               string
 	Docker                   bool
 	ContainerRuntimeEndpoint string
@@ -44,82 +43,82 @@ var (
 	AgentConfig Agent
 	NodeIPFlag  = cli.StringFlag{
 		Name:        "node-ip,i",
-		Usage:       "(agent) IP address to advertise for node",
+		Usage:       "(agent/networking) IP address to advertise for node",
 		Destination: &AgentConfig.NodeIP,
 	}
 	NodeExternalIPFlag = cli.StringFlag{
 		Name:        "node-external-ip",
-		Usage:       "(agent) External IP address to advertise for node",
+		Usage:       "(agent/networking) External IP address to advertise for node",
 		Destination: &AgentConfig.NodeExternalIP,
 	}
 	NodeNameFlag = cli.StringFlag{
 		Name:        "node-name",
-		Usage:       "(agent) Node name",
+		Usage:       "(agent/node) Node name",
 		EnvVar:      "K3S_NODE_NAME",
 		Destination: &AgentConfig.NodeName,
 	}
 	DockerFlag = cli.BoolFlag{
 		Name:        "docker",
-		Usage:       "(agent) Use docker instead of containerd",
+		Usage:       "(agent/runtime) Use docker instead of containerd",
 		Destination: &AgentConfig.Docker,
 	}
+	CRIEndpointFlag = cli.StringFlag{
+		Name:        "container-runtime-endpoint",
+		Usage:       "(agent/runtime) Disable embedded containerd and use alternative CRI implementation",
+		Destination: &AgentConfig.ContainerRuntimeEndpoint,
+	}
+	PrivateRegistryFlag = cli.StringFlag{
+		Name:        "private-registry",
+		Usage:       "(agent/runtime) Private registry configuration file",
+		Destination: &AgentConfig.PrivateRegistry,
+		Value:       "/etc/rancher/k3s/registries.yaml",
+	}
+	PauseImageFlag = cli.StringFlag{
+		Name:        "pause-image",
+		Usage:       "(agent/runtime) Customized pause image for containerd sandbox",
+		Destination: &AgentConfig.PauseImage,
+	}
 	FlannelFlag = cli.BoolFlag{
 		Name:        "no-flannel",
-		Usage:       "(agent) Disable embedded flannel",
+		Usage:       "(deprecated) use --flannel-backend=none",
 		Destination: &AgentConfig.NoFlannel,
 	}
 	FlannelIfaceFlag = cli.StringFlag{
 		Name:        "flannel-iface",
-		Usage:       "(agent) Override default flannel interface",
+		Usage:       "(agent/networking) Override default flannel interface",
 		Destination: &AgentConfig.FlannelIface,
 	}
 	FlannelConfFlag = cli.StringFlag{
 		Name:        "flannel-conf",
-		Usage:       "(agent) (experimental) Override default flannel config file",
+		Usage:       "(agent/networking) Override default flannel config file",
 		Destination: &AgentConfig.FlannelConf,
 	}
-	CRIEndpointFlag = cli.StringFlag{
-		Name:        "container-runtime-endpoint",
-		Usage:       "(agent) Disable embedded containerd and use alternative CRI implementation",
-		Destination: &AgentConfig.ContainerRuntimeEndpoint,
-	}
-	PauseImageFlag = cli.StringFlag{
-		Name:        "pause-image",
-		Usage:       "(agent) Customized pause image for containerd sandbox",
-		Destination: &AgentConfig.PauseImage,
-	}
 	ResolvConfFlag = cli.StringFlag{
 		Name:        "resolv-conf",
-		Usage:       "(agent) Kubelet resolv.conf file",
+		Usage:       "(agent/networking) Kubelet resolv.conf file",
 		EnvVar:      "K3S_RESOLV_CONF",
 		Destination: &AgentConfig.ResolvConf,
 	}
 	ExtraKubeletArgs = cli.StringSliceFlag{
 		Name:  "kubelet-arg",
-		Usage: "(agent) Customized flag for kubelet process",
+		Usage: "(agent/flags) Customized flag for kubelet process",
 		Value: &AgentConfig.ExtraKubeletArgs,
 	}
 	ExtraKubeProxyArgs = cli.StringSliceFlag{
 		Name:  "kube-proxy-arg",
-		Usage: "(agent) Customized flag for kube-proxy process",
+		Usage: "(agent/flags) Customized flag for kube-proxy process",
 		Value: &AgentConfig.ExtraKubeProxyArgs,
 	}
 	NodeTaints = cli.StringSliceFlag{
 		Name:  "node-taint",
-		Usage: "(agent) Registering kubelet with set of taints",
+		Usage: "(agent/node) Registering kubelet with set of taints",
 		Value: &AgentConfig.Taints,
 	}
 	NodeLabels = cli.StringSliceFlag{
 		Name:  "node-label",
-		Usage: "(agent) Registering kubelet with set of labels",
+		Usage: "(agent/node) Registering kubelet with set of labels",
 		Value: &AgentConfig.Labels,
 	}
-	PrivateRegistryFlag = cli.StringFlag{
-		Name:        "private-registry",
-		Usage:       "(agent) Private registry configuration file",
-		Destination: &AgentConfig.PrivateRegistry,
-		Value:       "/etc/rancher/k3s/registries.yaml",
-	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -135,54 +134,57 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			AlsoLogToStderr,
 			cli.StringFlag{
 				Name:        "token,t",
-				Usage:       "Token to use for authentication",
+				Usage:       "(cluster) Token to use for authentication",
 				EnvVar:      "K3S_TOKEN",
 				Destination: &AgentConfig.Token,
 			},
 			cli.StringFlag{
 				Name:        "token-file",
-				Usage:       "Token file to use for authentication",
+				Usage:       "(cluster) Token file to use for authentication",
 				EnvVar:      "K3S_TOKEN_FILE",
 				Destination: &AgentConfig.TokenFile,
 			},
 			cli.StringFlag{
 				Name:        "server,s",
-				Usage:       "Server to connect to",
+				Usage:       "(cluster) Server to connect to",
 				EnvVar:      "K3S_URL",
 				Destination: &AgentConfig.ServerURL,
 			},
 			cli.StringFlag{
 				Name:        "data-dir,d",
-				Usage:       "Folder to hold state",
+				Usage:       "(agent/data) Folder to hold state",
 				Destination: &AgentConfig.DataDir,
 				Value:       "/var/lib/rancher/k3s",
 			},
-			cli.StringFlag{
-				Name:        "cluster-secret",
-				Usage:       "Shared secret used to bootstrap a cluster",
-				Destination: &AgentConfig.ClusterSecret,
-				EnvVar:      "K3S_CLUSTER_SECRET",
-			},
-			cli.BoolFlag{
-				Name:        "rootless",
-				Usage:       "(experimental) Run rootless",
-				Destination: &AgentConfig.Rootless,
-			},
-			DockerFlag,
-			FlannelFlag,
-			FlannelIfaceFlag,
-			FlannelConfFlag,
 			NodeNameFlag,
-			NodeIPFlag,
+			NodeLabels,
+			NodeTaints,
+			DockerFlag,
 			CRIEndpointFlag,
 			PauseImageFlag,
+			PrivateRegistryFlag,
+			NodeIPFlag,
+			NodeExternalIPFlag,
 			ResolvConfFlag,
+			FlannelIfaceFlag,
+			FlannelConfFlag,
 			ExtraKubeletArgs,
 			ExtraKubeProxyArgs,
-			NodeLabels,
-			NodeTaints,
-			PrivateRegistryFlag,
-			NodeExternalIPFlag,
+			cli.BoolFlag{
+				Name:        "rootless",
+				Usage:       "(experimental) Run rootless",
+				Destination: &AgentConfig.Rootless,
+			},
+
+			// Deprecated/hidden below
+
+			FlannelFlag,
+			cli.StringFlag{
+				Name:        "cluster-secret",
+				Usage:       "(deprecated) use --token",
+				Destination: &AgentConfig.Token,
+				EnvVar:      "K3S_CLUSTER_SECRET",
+			},
 		},
 	}
 }

pkg/cli/cmds/log.go

@@ -25,22 +25,22 @@ var (
 
 	VLevel = cli.IntFlag{
 		Name:        "v",
-		Usage:       "Number for the log level verbosity",
+		Usage:       "(logging) Number for the log level verbosity",
 		Destination: &LogConfig.VLevel,
 	}
 	VModule = cli.StringFlag{
 		Name:        "vmodule",
-		Usage:       "Comma-separated list of pattern=N settings for file-filtered logging",
+		Usage:       "(logging) Comma-separated list of pattern=N settings for file-filtered logging",
 		Destination: &LogConfig.VModule,
 	}
 	LogFile = cli.StringFlag{
 		Name:        "log,l",
-		Usage:       "Log to file",
+		Usage:       "(logging) Log to file",
 		Destination: &LogConfig.LogFile,
 	}
 	AlsoLogToStderr = cli.BoolFlag{
 		Name:        "alsologtostderr",
-		Usage:       "Log to standard error as well as file (if set)",
+		Usage:       "(logging) Log to standard error as well as file (if set)",
 		Destination: &LogConfig.AlsoLogToStderr,
 	}
 )

pkg/cli/cmds/nocluster.go

+// +build !dqlite
+
+package cmds
+
+const (
+	hideDqlite = true
+)

pkg/cli/server/server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rancher/k3s/pkg/netutil"
 	"github.com/rancher/k3s/pkg/rootless"
 	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -56,23 +57,28 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 
 	serverConfig := server.Config{}
 	serverConfig.DisableAgent = cfg.DisableAgent
-	serverConfig.ControlConfig.ClusterSecret = cfg.ClusterSecret
+	serverConfig.ControlConfig.Token = cfg.Token
+	serverConfig.ControlConfig.AgentToken = cfg.AgentToken
+	serverConfig.ControlConfig.JoinURL = cfg.ServerURL
+	if cfg.AgentTokenFile != "" {
+		serverConfig.ControlConfig.AgentToken, err = token.ReadFile(cfg.AgentTokenFile)
+		if err != nil {
+			return err
+		}
+	}
+	if cfg.TokenFile != "" {
+		serverConfig.ControlConfig.Token, err = token.ReadFile(cfg.TokenFile)
+		if err != nil {
+			return err
+		}
+	}
 	serverConfig.ControlConfig.DataDir = cfg.DataDir
 	serverConfig.ControlConfig.KubeConfigOutput = cfg.KubeConfigOutput
 	serverConfig.ControlConfig.KubeConfigMode = cfg.KubeConfigMode
 	serverConfig.ControlConfig.NoScheduler = cfg.DisableScheduler
 	serverConfig.Rootless = cfg.Rootless
-	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
-	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
-	for _, san := range knownIPs(cfg.TLSSan) {
-		addr := net2.ParseIP(san)
-		if addr != nil {
-			serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
-		} else {
-			serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
-		}
-	}
-	serverConfig.TLSConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.SANs = knownIPs(cfg.TLSSan)
+	serverConfig.ControlConfig.BindAddress = cfg.BindAddress
 	serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
@@ -84,21 +90,25 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.Storage.KeyFile = cfg.StorageKeyFile
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
-	serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
 	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 	serverConfig.ControlConfig.ExtraCloudControllerArgs = cfg.ExtraCloudControllerArgs
 	serverConfig.ControlConfig.DisableCCM = cfg.DisableCCM
 	serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
+	serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
+	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
 
 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)
 	}
 
+	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeExternalIP != "" {
+		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeExternalIP
+	}
 	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
 		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
 	}
 	if serverConfig.ControlConfig.AdvertiseIP != "" {
-		serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
+		serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
 	}
 
 	_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
@@ -114,7 +124,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	if err != nil {
 		return err
 	}
-	serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
+	serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, apiServerServiceIP.String())
 
 	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
@@ -160,8 +170,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	os.Unsetenv("NOTIFY_SOCKET")
 
 	ctx := signals.SetupSignalHandler(context.Background())
-	certs, err := server.StartServer(ctx, &serverConfig)
-	if err != nil {
+	if err := server.StartServer(ctx, &serverConfig); err != nil {
 		return err
 	}
 
@@ -175,12 +184,17 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		<-ctx.Done()
 		return nil
 	}
-	ip := serverConfig.TLSConfig.BindAddress
+
+	ip := serverConfig.ControlConfig.BindAddress
 	if ip == "" {
 		ip = "127.0.0.1"
 	}
-	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.TLSConfig.HTTPSPort)
-	token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs)
+
+	url := fmt.Sprintf("https://%s:%d", ip, serverConfig.ControlConfig.HTTPSPort)
+	token, err := server.FormatToken(serverConfig.ControlConfig.Runtime.AgentToken, serverConfig.ControlConfig.Runtime.ServerCA)
+	if err != nil {
+		return err
+	}
 
 	agentConfig := cmds.AgentConfig
 	agentConfig.Debug = app.GlobalBool("bool")

pkg/clientaccess/clientaccess.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -35,21 +34,6 @@ type clientToken struct {
 	password string
 }
 
-func AgentAccessInfoToTempKubeConfig(tempDir, server, token string) (string, error) {
-	f, err := ioutil.TempFile(tempDir, "tmp-")
-	if err != nil {
-		return "", err
-	}
-	if err := f.Close(); err != nil {
-		return "", err
-	}
-	err = accessInfoToKubeConfig(f.Name(), server, token)
-	if err != nil {
-		os.Remove(f.Name())
-	}
-	return f.Name(), err
-}
-
 func AgentAccessInfoToKubeConfig(destFile, server, token string) error {
 	return accessInfoToKubeConfig(destFile, server, token)
 }
@@ -62,6 +46,10 @@ type Info struct {
 	Token    string `json:"token,omitempty"`
 }
 
+func (i *Info) ToToken() string {
+	return fmt.Sprintf("K10%s::%s:%s", hashCA(i.CACerts), i.username, i.password)
+}
+
 func (i *Info) WriteKubeConfig(destFile string) error {
 	return clientcmd.WriteToFile(*i.KubeConfig(), destFile)
 }
@@ -98,6 +86,22 @@ func (i *Info) KubeConfig() *clientcmdapi.Config {
 	return config
 }
 
+func NormalizeAndValidateTokenForUser(server, token, user string) (string, error) {
+	if !strings.HasPrefix(token, "K10") {
+		token = "K10::" + user + ":" + token
+	}
+	info, err := ParseAndValidateToken(server, token)
+	if err != nil {
+		return "", err
+	}
+
+	if info.username != user {
+		info.username = user
+	}
+
+	return info.ToToken(), nil
+}
+
 func ParseAndValidateToken(server, token string) (*Info, error) {
 	url, err := url.Parse(server)
 	if err != nil {
@@ -132,13 +136,17 @@ func ParseAndValidateToken(server, token string) (*Info, error) {
 		return nil, err
 	}
 
-	return &Info{
+	i := &Info{
 		URL:      url.String(),
 		CACerts:  cacerts,
 		username: parsedToken.username,
 		password: parsedToken.password,
 		Token:    token,
-	}, nil
+	}
+
+	// normalize token
+	i.Token = i.ToToken()
+	return i, nil
 }
 
 func accessInfoToKubeConfig(destFile, server, token string) error {
@@ -164,11 +172,15 @@ func validateCACerts(cacerts []byte, hash string) (bool, string, string) {
 		return true, "", ""
 	}
 
-	digest := sha256.Sum256([]byte(cacerts))
-	newHash := hex.EncodeToString(digest[:])
+	newHash := hashCA(cacerts)
 	return hash == newHash, hash, newHash
 }
 
+func hashCA(cacerts []byte) string {
+	digest := sha256.Sum256(cacerts)
+	return hex.EncodeToString(digest[:])
+}
+
 func ParseUsernamePassword(token string) (string, string, bool) {
 	parsed, err := parseToken(token)
 	if err != nil {

pkg/cluster/cluster.go

+package cluster
+
+import (
+	"context"
+
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+type Cluster struct {
+	token            string
+	clientAccessInfo *clientaccess.Info
+	config           *config.Control
+	runtime          *config.ControlRuntime
+	db               interface{}
+}
+
+func (c *Cluster) Start(ctx context.Context) error {
+	join, err := c.shouldJoin()
+	if err != nil {
+		return err
+	}
+
+	if join {
+		if err := c.join(); err != nil {
+			return err
+		}
+	}
+
+	if err := c.startClusterAndHTTPS(ctx); err != nil {
+		return err
+	}
+
+	if join {
+		if err := c.postJoin(ctx); err != nil {
+			return err
+		}
+	}
+
+	return c.joined()
+}
+
+func New(config *config.Control) *Cluster {
+	return &Cluster{
+		config:  config,
+		runtime: config.Runtime,
+	}
+}

pkg/cluster/https.go

+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
+	"path/filepath"
+
+	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/dynamiclistener/factory"
+	"github.com/rancher/dynamiclistener/storage/file"
+	"github.com/rancher/dynamiclistener/storage/kubernetes"
+	"github.com/rancher/dynamiclistener/storage/memory"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *Cluster) newListener(ctx context.Context) (net.Listener, http.Handler, error) {
+	tcp, err := dynamiclistener.NewTCPListener(c.config.BindAddress, c.config.HTTPSPort)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert, key, err := factory.LoadCerts(c.runtime.ServerCA, c.runtime.ServerCAKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	storage := tlsStorage(ctx, c.config.DataDir, c.runtime)
+	return dynamiclistener.NewListener(tcp, storage, cert, key, dynamiclistener.Config{
+		CN:           "k3s",
+		Organization: []string{"k3s"},
+		TLSConfig: tls.Config{
+			ClientAuth: tls.RequestClientCert,
+		},
+		SANs: c.config.SANs,
+	})
+}
+
+func (c *Cluster) startClusterAndHTTPS(ctx context.Context) error {
+	l, handler, err := c.newListener(ctx)
+	if err != nil {
+		return err
+	}
+
+	handler, err = c.getHandler(handler)
+	if err != nil {
+		return err
+	}
+
+	l, handler, err = c.initClusterDB(ctx, l, handler)
+	if err != nil {
+		return err
+	}
+
+	server := http.Server{
+		Handler: handler,
+	}
+
+	go func() {
+		err := server.Serve(l)
+		logrus.Fatalf("server stopped: %v", err)
+	}()
+
+	go func() {
+		<-ctx.Done()
+		server.Shutdown(context.Background())
+	}()
+
+	return nil
+}
+
+func tlsStorage(ctx context.Context, dataDir string, runtime *config.ControlRuntime) dynamiclistener.TLSStorage {
+	fileStorage := file.New(filepath.Join(dataDir, "tls/dynamic-cert.json"))
+	cache := memory.NewBacked(fileStorage)
+	return kubernetes.New(ctx, func() *core.Factory {
+		return runtime.Core
+	}, "kube-system", "k3s-serving", cache)
+}

pkg/cluster/join.go

+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/rancher/k3s/pkg/bootstrap"
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *Cluster) shouldJoin() (bool, error) {
+	if c.config.JoinURL == "" {
+		return false, nil
+	}
+
+	stamp := filepath.Join(c.config.DataDir, "db/joined")
+	if _, err := os.Stat(stamp); err == nil {
+		logrus.Info("Already joined to cluster, not rejoining")
+		return false, nil
+	}
+
+	if c.config.Token == "" {
+		return false, fmt.Errorf("K3S_TOKEN is required to join a cluster")
+	}
+
+	return true, nil
+}
+
+func (c *Cluster) joined() error {
+	if err := os.MkdirAll(filepath.Dir(c.joinStamp()), 0700); err != nil {
+		return err
+	}
+
+	if _, err := os.Stat(c.joinStamp()); err == nil {
+		return nil
+	}
+
+	f, err := os.Create(c.joinStamp())
+	if err != nil {
+		return err
+	}
+
+	return f.Close()
+}
+
+func (c *Cluster) join() error {
+	c.runtime.Cluster.Join = true
+
+	token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
+	if err != nil {
+		return err
+	}
+	c.token = token
+
+	info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
+	if err != nil {
+		return err
+	}
+	c.clientAccessInfo = info
+
+	content, err := clientaccess.Get("/v1-k3s/server-bootstrap", info)
+	if err != nil {
+		return err
+	}
+
+	return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
+}
+
+func (c *Cluster) joinStamp() string {
+	return filepath.Join(c.config.DataDir, "db/joined")
+}

pkg/cluster/nocluster.go

+// +build !dqlite
+
+package cluster
+
+import (
+	"context"
+	"net"
+	"net/http"
+)
+
+func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
+	return l, handler, nil
+}
+
+func (c *Cluster) postJoin(ctx context.Context) error {
+	return nil
+}

pkg/cluster/router.go

+package cluster
+
+import (
+	"net/http"
+)
+
+func (c *Cluster) getHandler(handler http.Handler) (http.Handler, error) {
+	next := c.router()
+
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		handler.ServeHTTP(rw, req)
+		next.ServeHTTP(rw, req)
+	}), nil
+}
+
+func (c *Cluster) router() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if c.runtime.Handler == nil {
+			http.Error(rw, "starting", http.StatusServiceUnavailable)
+			return
+		}
+
+		c.runtime.Handler.ServeHTTP(rw, req)
+	})
+}

pkg/codegen/main.go

@@ -70,7 +70,6 @@ func main() {
 		Groups: map[string]args.Group{
 			"k3s.cattle.io": {
 				Types: []interface{}{
-					v1.ListenerConfig{},
 					v1.Addon{},
 				},
 				GenerateTypes: true,

pkg/daemons/config/types.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/rancher/kine/pkg/endpoint"
-
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
@@ -47,40 +47,36 @@ type Containerd struct {
 }
 
 type Agent struct {
-	NodeName            string
-	ClientKubeletCert   string
-	ClientKubeletKey    string
-	ClientKubeProxyCert string
-	ClientKubeProxyKey  string
-	ServingKubeletCert  string
-	ServingKubeletKey   string
-	ClusterCIDR         net.IPNet
-	ClusterDNS          net.IP
-	ClusterDomain       string
-	ResolvConf          string
-	RootDir             string
-	KubeConfigNode      string
-	KubeConfigKubelet   string
-	KubeConfigKubeProxy string
-	NodeIP              string
-	NodeExternalIP      string
-	RuntimeSocket       string
-	ListenAddress       string
-	ClientCA            string
-	CNIBinDir           string
-	CNIConfDir          string
-	ExtraKubeletArgs    []string
-	ExtraKubeProxyArgs  []string
-	PauseImage          string
-	CNIPlugin           bool
-	NodeTaints          []string
-	NodeLabels          []string
-	IPSECPSK            string
-	StrongSwanDir       string
-	PrivateRegistry     string
-	DisableCCM          bool
-	DisableNPC          bool
-	Rootless            bool
+	NodeName                string
+	ServingKubeletCert      string
+	ServingKubeletKey       string
+	ClusterCIDR             net.IPNet
+	ClusterDNS              net.IP
+	ClusterDomain           string
+	ResolvConf              string
+	RootDir                 string
+	KubeConfigKubelet       string
+	KubeConfigKubeProxy     string
+	KubeConfigK3sController string
+	NodeIP                  string
+	NodeExternalIP          string
+	RuntimeSocket           string
+	ListenAddress           string
+	ClientCA                string
+	CNIBinDir               string
+	CNIConfDir              string
+	ExtraKubeletArgs        []string
+	ExtraKubeProxyArgs      []string
+	PauseImage              string
+	CNIPlugin               bool
+	NodeTaints              []string
+	NodeLabels              []string
+	IPSECPSK                string
+	StrongSwanDir           string
+	PrivateRegistry         string
+	DisableCCM              bool
+	DisableNPC              bool
+	Rootless                bool
 }
 
 type Control struct {
@@ -88,7 +84,8 @@ type Control struct {
 	AdvertiseIP              string
 	ListenPort               int
 	HTTPSPort                int
-	ClusterSecret            string
+	AgentToken               string
+	Token                    string
 	ClusterIPRange           *net.IPNet
 	ServiceIPRange           *net.IPNet
 	ClusterDNS               net.IP
@@ -98,19 +95,24 @@ type Control struct {
 	KubeConfigMode           string
 	DataDir                  string
 	Skips                    []string
-	BootstrapReadOnly        bool
 	Storage                  endpoint.Config
 	NoScheduler              bool
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
-	ExtraSchedulerAPIArgs    []string
 	ExtraCloudControllerArgs []string
+	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
+	JoinURL                  string
 	FlannelBackend           string
 	IPSECPSK                 string
 	DefaultLocalStoragePath  string
 	DisableCCM               bool
 	DisableNPC               bool
+	ClusterInit              bool
+	ClusterReset             bool
+
+	BindAddress string
+	SANs        []string
 
 	Runtime *ControlRuntime `json:"-"`
 }
@@ -124,9 +126,6 @@ type ControlRuntimeBootstrap struct {
 	PasswdFile         string
 	RequestHeaderCA    string
 	RequestHeaderCAKey string
-	ClientKubeletKey   string
-	ClientKubeProxyKey string
-	ServingKubeletKey  string
 	IPSECKey           string
 }
 
@@ -145,8 +144,10 @@ type ControlRuntime struct {
 
 	ServingKubeAPICert string
 	ServingKubeAPIKey  string
+	ServingKubeletKey  string
 	ClientToken        string
-	NodeToken          string
+	ServerToken        string
+	AgentToken         string
 	Handler            http.Handler
 	Tunnel             http.Handler
 	Authenticator      authenticator.Request
@@ -161,8 +162,19 @@ type ControlRuntime struct {
 	ClientSchedulerCert       string
 	ClientSchedulerKey        string
 	ClientKubeProxyCert       string
+	ClientKubeProxyKey        string
+	ClientKubeletKey          string
 	ClientCloudControllerCert string
 	ClientCloudControllerKey  string
+	ClientK3sControllerCert   string
+	ClientK3sControllerKey    string
+
+	Cluster ClusterConfig
+	Core    *core.Factory
+}
+
+type ClusterConfig struct {
+	Join bool
 }
 
 type ArgString []string

pkg/daemons/control/server.go

@@ -3,13 +3,9 @@ package control
 import (
 	"context"
 	"crypto"
-	cryptorand "crypto/rand"
 	"crypto/x509"
-	"encoding/csv"
-	"encoding/hex"
 	"fmt"
 	"html/template"
-	"io"
 	"io/ioutil"
 	"math/rand"
 	"net"
@@ -21,11 +17,14 @@ import (
 	"strings"
 	"time"
 
-	certutil "github.com/rancher/dynamiclistener/cert"
 	// registering k3s cloud provider
 	_ "github.com/rancher/k3s/pkg/cloudprovider"
+
+	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3s/pkg/cluster"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/kine/pkg/client"
+	"github.com/rancher/k3s/pkg/passwd"
+	"github.com/rancher/k3s/pkg/token"
 	"github.com/rancher/kine/pkg/endpoint"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
@@ -121,6 +120,10 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
 		"cluster-signing-cert-file":        runtime.ServerCA,
 		"cluster-signing-key-file":         runtime.ServerCAKey,
 	}
+	offset := cfg.HTTPSPort - 6443
+	if offset > 0 {
+		argsMap["port"] = strconv.Itoa(10252 + offset)
+	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
@@ -143,6 +146,10 @@ func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
 		"bind-address": "127.0.0.1",
 		"secure-port":  "0",
 	}
+	offset := cfg.HTTPSPort - 6443
+	if offset > 0 {
+		argsMap["port"] = strconv.Itoa(10251 + offset)
+	}
 	if cfg.NoLeaderElect {
 		argsMap["leader-elect"] = "false"
 	}
@@ -291,6 +298,8 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key")
 	runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt")
 	runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key")
+	runtime.ClientK3sControllerCert = path.Join(config.DataDir, "tls", "client-k3s-controller.crt")
+	runtime.ClientK3sControllerKey = path.Join(config.DataDir, "tls", "client-k3s-controller.key")
 
 	runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt")
 	runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key")
@@ -301,17 +310,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
-	etcdClient, err := prepareStorageBackend(ctx, config)
-	if err != nil {
-		return err
-	}
-	defer etcdClient.Close()
-
-	if err := fetchBootstrapData(ctx, config, etcdClient); err != nil {
+	if err := genCerts(config, runtime); err != nil {
 		return err
 	}
 
-	if err := genCerts(config, runtime); err != nil {
+	if err := cluster.New(config).Start(ctx); err != nil {
 		return err
 	}
 
@@ -327,124 +330,44 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
-	if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
+	if err := prepareStorageBackend(ctx, config); err != nil {
 		return err
 	}
 
 	return readTokens(runtime)
 }
 
-func prepareStorageBackend(ctx context.Context, config *config.Control) (client.Client, error) {
+func prepareStorageBackend(ctx context.Context, config *config.Control) error {
 	etcdConfig, err := endpoint.Listen(ctx, config.Storage)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	config.Storage.Config = etcdConfig.TLSConfig
 	config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
 	config.NoLeaderElect = !etcdConfig.LeaderElect
-	return client.New(etcdConfig)
-}
-
-func readTokenFile(passwdFile string) (map[string]string, error) {
-	f, err := os.Open(passwdFile)
-	if err != nil {
-		return nil, err
-	}
-	defer f.Close()
-
-	reader := csv.NewReader(f)
-	reader.FieldsPerRecord = -1
-
-	tokens := map[string]string{}
-
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if len(record) < 2 {
-			continue
-		}
-		tokens[record[1]] = record[0]
-	}
-	return tokens, nil
+	return nil
 }
 
 func readTokens(runtime *config.ControlRuntime) error {
-	tokens, err := readTokenFile(runtime.PasswdFile)
+	tokens, err := passwd.Read(runtime.PasswdFile)
 	if err != nil {
 		return err
 	}
 
-	if nodeToken, ok := tokens["node"]; ok {
-		runtime.NodeToken = "node:" + nodeToken
+	if nodeToken, ok := tokens.Pass("node"); ok {
+		runtime.AgentToken = "node:" + nodeToken
+	}
+	if serverToken, ok := tokens.Pass("server"); ok {
+		runtime.AgentToken = "server:" + serverToken
 	}
-	if clientToken, ok := tokens["admin"]; ok {
+	if clientToken, ok := tokens.Pass("admin"); ok {
 		runtime.ClientToken = "admin:" + clientToken
 	}
 
 	return nil
 }
 
-func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) error {
-	if config.ClusterSecret == "" {
-		return nil
-	}
-
-	f, err := os.Open(runtime.PasswdFile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	records := [][]string{}
-	reader := csv.NewReader(f)
-	for {
-		record, err := reader.Read()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		if len(record) < 3 {
-			return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record))
-		}
-		if record[1] == "node" {
-			if record[0] == config.ClusterSecret {
-				return nil
-			}
-			record[0] = config.ClusterSecret
-		}
-		records = append(records, record)
-	}
-
-	f.Close()
-	return WritePasswords(runtime.PasswdFile, records)
-}
-
-func WritePasswords(passwdFile string, records [][]string) error {
-	out, err := os.Create(passwdFile + ".tmp")
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	if err := out.Chmod(0600); err != nil {
-		return err
-	}
-
-	if err := csv.NewWriter(out).WriteAll(records); err != nil {
-		return err
-	}
-
-	return os.Rename(passwdFile+".tmp", passwdFile)
-}
-
 func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
 	if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
 		psk, err := ioutil.ReadFile(runtime.IPSECKey)
@@ -455,7 +378,7 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
 		return nil
 	}
 
-	psk, err := getToken(ipsecTokenSize)
+	psk, err := token.Random(ipsecTokenSize)
 	if err != nil {
 		return err
 	}
@@ -468,42 +391,71 @@ func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.Cont
 	return nil
 }
 
-func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
-	if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
-		return ensureNodeToken(config, runtime)
+func migratePassword(p *passwd.Passwd) error {
+	server, _ := p.Pass("server")
+	node, _ := p.Pass("node")
+	if server == "" && node != "" {
+		return p.EnsureUser("server", "k3s:server", node)
 	}
+	return nil
+}
+
+func getServerPass(passwd *passwd.Passwd, config *config.Control) (string, error) {
+	var (
+		err error
+	)
 
-	adminToken, err := getToken(userTokenSize)
+	serverPass := config.Token
+	if serverPass == "" {
+		serverPass, _ = passwd.Pass("server")
+	}
+	if serverPass == "" {
+		serverPass, err = token.Random(16)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return serverPass, nil
+}
+
+func getNodePass(config *config.Control, serverPass string) string {
+	if config.AgentToken == "" {
+		return serverPass
+	}
+	return config.AgentToken
+}
+
+func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
+	passwd, err := passwd.Read(runtime.PasswdFile)
 	if err != nil {
 		return err
 	}
-	systemToken, err := getToken(userTokenSize)
-	if err != nil {
+
+	if err := migratePassword(passwd); err != nil {
 		return err
 	}
-	nodeToken, err := getToken(userTokenSize)
+
+	serverPass, err := getServerPass(passwd, config)
 	if err != nil {
 		return err
 	}
 
-	if config.ClusterSecret != "" {
-		nodeToken = config.ClusterSecret
+	nodePass := getNodePass(config, serverPass)
+
+	if err := passwd.EnsureUser("admin", "system:masters", ""); err != nil {
+		return err
 	}
 
-	return WritePasswords(runtime.PasswdFile, [][]string{
-		{adminToken, "admin", "admin", "system:masters"},
-		{systemToken, "system", "system", "system:masters"},
-		{nodeToken, "node", "node", "system:masters"},
-	})
-}
+	if err := passwd.EnsureUser("node", "k3s:agent", nodePass); err != nil {
+		return err
+	}
 
-func getToken(size int) (string, error) {
-	token := make([]byte, size, size)
-	_, err := cryptorand.Read(token)
-	if err != nil {
-		return "", err
+	if err := passwd.EnsureUser("server", "k3s:server", serverPass); err != nil {
+		return err
 	}
-	return hex.EncodeToString(token), err
+
+	return passwd.Write(runtime.PasswdFile)
 }
 
 func genCerts(config *config.Control, runtime *config.ControlRuntime) error {
@@ -578,7 +530,10 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro
 		}
 	}
 
-	if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
+	if _, err = factory("system:kube-proxy", nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
+		return err
+	}
+	if _, err = factory("system:k3s-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
 		return err
 	}
 

pkg/daemons/control/tunnel.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/rancher/remotedialer"
@@ -37,14 +38,9 @@ func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
 		return "", false, nil
 	}
 
-	if user.GetName() != "node" {
-		return "", false, nil
-	}
-
-	nodeName := req.Header.Get("X-K3s-NodeName")
-	if nodeName == "" {
-		return "", false, nil
+	if strings.HasPrefix(user.GetName(), "system:node:") {
+		return strings.TrimPrefix(user.GetName(), "system:node:"), true, nil
 	}
 
-	return nodeName, true, nil
+	return "", false, nil
 }

pkg/datadir/datadir.go

@@ -2,6 +2,7 @@ package datadir
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
@@ -32,5 +33,5 @@ func LocalHome(dataDir string, forceLocal bool) (string, error) {
 		return "", errors.Wrapf(err, "resolving %s", dataDir)
 	}
 
-	return dataDir, nil
+	return filepath.Abs(dataDir)
 }

pkg/deploy/zz_generated_bindata.go

@@ -141,7 +141,7 @@ func localStorageYaml() (*asset, error) {
 	return a, nil
 }
 
-var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\xcf\xbd\x0a\xc2\x40\x10\x04\xe0\xfe\x9e\xe2\x5e\xe0\x22\x76\x72\xa5\x16\xf6\x01\xed\x37\xb9\x55\xd7\xdc\x1f\xbb\x7b\x01\x7d\x7a\x09\x48\x1a\x51\xb0\x1c\x18\xe6\x63\xa0\xd2\x19\x59\xa8\x64\x6f\x79\x80\xb1\x83\xa6\xb7\xc2\xf4\x04\xa5\x92\xbb\x69\x27\x1d\x95\xcd\xbc\x35\x13\xe5\xe0\xed\x21\x36\x51\xe4\xbe\x44\xdc\x53\x0e\x94\xaf\x26\xa1\x42\x00\x05\x6f\xac\xcd\x90\xd0\xdb\xa9\x0d\xe8\xa0\x92\x20\xcf\xc8\x6e\x89\x11\xd5\x41\x48\x94\x0d\x97\x88\x3d\x5e\x96\x36\x54\x3a\x72\x69\xf5\x87\x6c\xac\xfd\x80\x57\x47\x1e\xa2\x98\xfc\xba\x5f\xe9\x6d\x48\x1b\xee\x38\xaa\x78\xe3\xfe\x42\x4e\x82\xfc\xe5\x85\x79\x05\x00\x00\xff\xff\x54\xf2\x55\xe2\x29\x01\x00\x00")
+var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x92\x31\x6f\xe3\x30\x0c\x85\x77\xfd\x0a\x21\xbb\x72\x38\xdc\x72\xf0\xd8\x0e\xdd\x03\xb4\x3b\x6d\xb3\x09\x6b\x59\x14\x48\x2a\x41\xfb\xeb\x0b\xa7\x6e\x82\xa4\x76\x90\xb4\xdd\x24\x41\x7c\x1f\x1f\xf9\x20\xd3\x13\x8a\x12\xa7\xca\x4b\x0d\xcd\x12\x8a\x6d\x58\xe8\x0d\x8c\x38\x2d\xbb\xff\xba\x24\xfe\xb3\xfd\xeb\x3a\x4a\x6d\xe5\xef\x63\x51\x43\x59\x71\xc4\x3b\x4a\x2d\xa5\xb5\xeb\xd1\xa0\x05\x83\xca\x79\x9f\xa0\xc7\xca\x77\xa5\xc6\x00\x99\x14\x65\x8b\x12\x86\x6b\x44\x0b\xd0\xf6\x94\x9c\x70\xc4\x15\x3e\x0f\xbf\x21\xd3\x83\x70\xc9\x17\xc8\xce\xfb\x2f\xe0\x03\x47\x5f\xd5\xb0\xaf\x0e\xfa\x99\x46\x86\x96\xfa\x05\x1b\xd3\xca\x85\x9b\x20\x8f\x8a\x32\xe3\xc2\xb9\x10\x82\xfb\xfe\xb4\x26\xc6\xf4\xd9\xfe\x3f\x0d\x0d\x27\x13\x8e\x11\xc5\x49\x89\x78\xd2\xb8\x0e\x15\xc1\x2f\x16\xce\x7b\x41\xe5\x22\x0d\x8e\x6f\x89\x5b\x54\xe7\xfd\x16\xa5\x1e\x9f\xd6\x68\x57\xd6\x42\x8f\x9a\xa1\x39\x17\x88\xa4\xb6\x3f\xec\xc0\x9a\xcd\x84\x56\x42\xdb\xb1\x74\x94\xd6\xa3\xdf\x29\xf1\x8f\x3f\x99\x23\x35\x74\x33\x61\x42\x10\x53\x9b\x99\x92\xe9\xfe\x96\xb9\x9d\xd3\x1c\xfc\x1f\xb5\x7f\xb8\xb4\xf9\x88\xcf\xec\xee\xf7\xb3\x7d\x0a\x38\x06\x7b\xf0\x78\x1d\xe3\x2c\xdc\x97\x01\xef\x01\x00\x00\xff\xff\x46\xd3\x6d\x9d\x0f\x04\x00\x00")
 
 func rolebindingsYamlBytes() ([]byte, error) {
 	return bindataRead(

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go

@@ -32,10 +32,6 @@ func (c *FakeK3sV1) Addons(namespace string) v1.AddonInterface {
 	return &FakeAddons{c, namespace}
 }
 
-func (c *FakeK3sV1) ListenerConfigs(namespace string) v1.ListenerConfigInterface {
-	return &FakeListenerConfigs{c, namespace}
-}
-
 // RESTClient returns a RESTClient that is used to communicate
 // with API server by this client implementation.
 func (c *FakeK3sV1) RESTClient() rest.Interface {

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_listenerconfig.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	k3scattleiov1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	labels "k8s.io/apimachinery/pkg/labels"
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	testing "k8s.io/client-go/testing"
-)
-
-// FakeListenerConfigs implements ListenerConfigInterface
-type FakeListenerConfigs struct {
-	Fake *FakeK3sV1
-	ns   string
-}
-
-var listenerconfigsResource = schema.GroupVersionResource{Group: "k3s.cattle.io", Version: "v1", Resource: "listenerconfigs"}
-
-var listenerconfigsKind = schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "ListenerConfig"}
-
-// Get takes name of the listenerConfig, and returns the corresponding listenerConfig object, and an error if there is any.
-func (c *FakeListenerConfigs) Get(name string, options v1.GetOptions) (result *k3scattleiov1.ListenerConfig, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewGetAction(listenerconfigsResource, c.ns, name), &k3scattleiov1.ListenerConfig{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*k3scattleiov1.ListenerConfig), err
-}
-
-// List takes label and field selectors, and returns the list of ListenerConfigs that match those selectors.
-func (c *FakeListenerConfigs) List(opts v1.ListOptions) (result *k3scattleiov1.ListenerConfigList, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewListAction(listenerconfigsResource, listenerconfigsKind, c.ns, opts), &k3scattleiov1.ListenerConfigList{})
-
-	if obj == nil {
-		return nil, err
-	}
-
-	label, _, _ := testing.ExtractFromListOptions(opts)
-	if label == nil {
-		label = labels.Everything()
-	}
-	list := &k3scattleiov1.ListenerConfigList{ListMeta: obj.(*k3scattleiov1.ListenerConfigList).ListMeta}
-	for _, item := range obj.(*k3scattleiov1.ListenerConfigList).Items {
-		if label.Matches(labels.Set(item.Labels)) {
-			list.Items = append(list.Items, item)
-		}
-	}
-	return list, err
-}
-
-// Watch returns a watch.Interface that watches the requested listenerConfigs.
-func (c *FakeListenerConfigs) Watch(opts v1.ListOptions) (watch.Interface, error) {
-	return c.Fake.
-		InvokesWatch(testing.NewWatchAction(listenerconfigsResource, c.ns, opts))
-
-}
-
-// Create takes the representation of a listenerConfig and creates it.  Returns the server's representation of the listenerConfig, and an error, if there is any.
-func (c *FakeListenerConfigs) Create(listenerConfig *k3scattleiov1.ListenerConfig) (result *k3scattleiov1.ListenerConfig, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewCreateAction(listenerconfigsResource, c.ns, listenerConfig), &k3scattleiov1.ListenerConfig{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*k3scattleiov1.ListenerConfig), err
-}
-
-// Update takes the representation of a listenerConfig and updates it. Returns the server's representation of the listenerConfig, and an error, if there is any.
-func (c *FakeListenerConfigs) Update(listenerConfig *k3scattleiov1.ListenerConfig) (result *k3scattleiov1.ListenerConfig, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewUpdateAction(listenerconfigsResource, c.ns, listenerConfig), &k3scattleiov1.ListenerConfig{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*k3scattleiov1.ListenerConfig), err
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *FakeListenerConfigs) UpdateStatus(listenerConfig *k3scattleiov1.ListenerConfig) (*k3scattleiov1.ListenerConfig, error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewUpdateSubresourceAction(listenerconfigsResource, "status", c.ns, listenerConfig), &k3scattleiov1.ListenerConfig{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*k3scattleiov1.ListenerConfig), err
-}
-
-// Delete takes name of the listenerConfig and deletes it. Returns an error if one occurs.
-func (c *FakeListenerConfigs) Delete(name string, options *v1.DeleteOptions) error {
-	_, err := c.Fake.
-		Invokes(testing.NewDeleteAction(listenerconfigsResource, c.ns, name), &k3scattleiov1.ListenerConfig{})
-
-	return err
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *FakeListenerConfigs) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
-	action := testing.NewDeleteCollectionAction(listenerconfigsResource, c.ns, listOptions)
-
-	_, err := c.Fake.Invokes(action, &k3scattleiov1.ListenerConfigList{})
-	return err
-}
-
-// Patch applies the patch and returns the patched listenerConfig.
-func (c *FakeListenerConfigs) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *k3scattleiov1.ListenerConfig, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewPatchSubresourceAction(listenerconfigsResource, c.ns, name, pt, data, subresources...), &k3scattleiov1.ListenerConfig{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*k3scattleiov1.ListenerConfig), err
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go

@@ -19,5 +19,3 @@ limitations under the License.
 package v1
 
 type AddonExpansion interface{}
-
-type ListenerConfigExpansion interface{}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go

@@ -27,7 +27,6 @@ import (
 type K3sV1Interface interface {
 	RESTClient() rest.Interface
 	AddonsGetter
-	ListenerConfigsGetter
 }
 
 // K3sV1Client is used to interact with features provided by the k3s.cattle.io group.
@@ -39,10 +38,6 @@ func (c *K3sV1Client) Addons(namespace string) AddonInterface {
 	return newAddons(c, namespace)
 }
 
-func (c *K3sV1Client) ListenerConfigs(namespace string) ListenerConfigInterface {
-	return newListenerConfigs(c, namespace)
-}
-
 // NewForConfig creates a new K3sV1Client for the given config.
 func NewForConfig(c *rest.Config) (*K3sV1Client, error) {
 	config := *c

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/listenerconfig.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"time"
-
-	v1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
-	scheme "github.com/rancher/k3s/pkg/generated/clientset/versioned/scheme"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	rest "k8s.io/client-go/rest"
-)
-
-// ListenerConfigsGetter has a method to return a ListenerConfigInterface.
-// A group's client should implement this interface.
-type ListenerConfigsGetter interface {
-	ListenerConfigs(namespace string) ListenerConfigInterface
-}
-
-// ListenerConfigInterface has methods to work with ListenerConfig resources.
-type ListenerConfigInterface interface {
-	Create(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	Update(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	UpdateStatus(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	Delete(name string, options *metav1.DeleteOptions) error
-	DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error
-	Get(name string, options metav1.GetOptions) (*v1.ListenerConfig, error)
-	List(opts metav1.ListOptions) (*v1.ListenerConfigList, error)
-	Watch(opts metav1.ListOptions) (watch.Interface, error)
-	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ListenerConfig, err error)
-	ListenerConfigExpansion
-}
-
-// listenerConfigs implements ListenerConfigInterface
-type listenerConfigs struct {
-	client rest.Interface
-	ns     string
-}
-
-// newListenerConfigs returns a ListenerConfigs
-func newListenerConfigs(c *K3sV1Client, namespace string) *listenerConfigs {
-	return &listenerConfigs{
-		client: c.RESTClient(),
-		ns:     namespace,
-	}
-}
-
-// Get takes name of the listenerConfig, and returns the corresponding listenerConfig object, and an error if there is any.
-func (c *listenerConfigs) Get(name string, options metav1.GetOptions) (result *v1.ListenerConfig, err error) {
-	result = &v1.ListenerConfig{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		Name(name).
-		VersionedParams(&options, scheme.ParameterCodec).
-		Do().
-		Into(result)
-	return
-}
-
-// List takes label and field selectors, and returns the list of ListenerConfigs that match those selectors.
-func (c *listenerConfigs) List(opts metav1.ListOptions) (result *v1.ListenerConfigList, err error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	result = &v1.ListenerConfigList{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Do().
-		Into(result)
-	return
-}
-
-// Watch returns a watch.Interface that watches the requested listenerConfigs.
-func (c *listenerConfigs) Watch(opts metav1.ListOptions) (watch.Interface, error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	opts.Watch = true
-	return c.client.Get().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Watch()
-}
-
-// Create takes the representation of a listenerConfig and creates it.  Returns the server's representation of the listenerConfig, and an error, if there is any.
-func (c *listenerConfigs) Create(listenerConfig *v1.ListenerConfig) (result *v1.ListenerConfig, err error) {
-	result = &v1.ListenerConfig{}
-	err = c.client.Post().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		Body(listenerConfig).
-		Do().
-		Into(result)
-	return
-}
-
-// Update takes the representation of a listenerConfig and updates it. Returns the server's representation of the listenerConfig, and an error, if there is any.
-func (c *listenerConfigs) Update(listenerConfig *v1.ListenerConfig) (result *v1.ListenerConfig, err error) {
-	result = &v1.ListenerConfig{}
-	err = c.client.Put().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		Name(listenerConfig.Name).
-		Body(listenerConfig).
-		Do().
-		Into(result)
-	return
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-
-func (c *listenerConfigs) UpdateStatus(listenerConfig *v1.ListenerConfig) (result *v1.ListenerConfig, err error) {
-	result = &v1.ListenerConfig{}
-	err = c.client.Put().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		Name(listenerConfig.Name).
-		SubResource("status").
-		Body(listenerConfig).
-		Do().
-		Into(result)
-	return
-}
-
-// Delete takes name of the listenerConfig and deletes it. Returns an error if one occurs.
-func (c *listenerConfigs) Delete(name string, options *metav1.DeleteOptions) error {
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		Name(name).
-		Body(options).
-		Do().
-		Error()
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *listenerConfigs) DeleteCollection(options *metav1.DeleteOptions, listOptions metav1.ListOptions) error {
-	var timeout time.Duration
-	if listOptions.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
-	}
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		VersionedParams(&listOptions, scheme.ParameterCodec).
-		Timeout(timeout).
-		Body(options).
-		Do().
-		Error()
-}
-
-// Patch applies the patch and returns the patched listenerConfig.
-func (c *listenerConfigs) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ListenerConfig, err error) {
-	result = &v1.ListenerConfig{}
-	err = c.client.Patch(pt).
-		Namespace(c.ns).
-		Resource("listenerconfigs").
-		SubResource(subresources...).
-		Name(name).
-		Body(data).
-		Do().
-		Into(result)
-	return
-}

pkg/generated/controllers/k3s.cattle.io/v1/interface.go

@@ -27,7 +27,6 @@ import (
 
 type Interface interface {
 	Addon() AddonController
-	ListenerConfig() ListenerConfigController
 }
 
 func New(controllerManager *generic.ControllerManager, client clientset.K3sV1Interface,
@@ -48,6 +47,3 @@ type version struct {
 func (c *version) Addon() AddonController {
 	return NewAddonController(v1.SchemeGroupVersion.WithKind("Addon"), c.controllerManager, c.client, c.informers.Addons())
 }
-func (c *version) ListenerConfig() ListenerConfigController {
-	return NewListenerConfigController(v1.SchemeGroupVersion.WithKind("ListenerConfig"), c.controllerManager, c.client, c.informers.ListenerConfigs())
-}

pkg/generated/controllers/k3s.cattle.io/v1/listenerconfig.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"context"
-
-	v1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
-	clientset "github.com/rancher/k3s/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1"
-	informers "github.com/rancher/k3s/pkg/generated/informers/externalversions/k3s.cattle.io/v1"
-	listers "github.com/rancher/k3s/pkg/generated/listers/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/generic"
-	"k8s.io/apimachinery/pkg/api/equality"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/types"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/tools/cache"
-)
-
-type ListenerConfigHandler func(string, *v1.ListenerConfig) (*v1.ListenerConfig, error)
-
-type ListenerConfigController interface {
-	ListenerConfigClient
-
-	OnChange(ctx context.Context, name string, sync ListenerConfigHandler)
-	OnRemove(ctx context.Context, name string, sync ListenerConfigHandler)
-	Enqueue(namespace, name string)
-
-	Cache() ListenerConfigCache
-
-	Informer() cache.SharedIndexInformer
-	GroupVersionKind() schema.GroupVersionKind
-
-	AddGenericHandler(ctx context.Context, name string, handler generic.Handler)
-	AddGenericRemoveHandler(ctx context.Context, name string, handler generic.Handler)
-	Updater() generic.Updater
-}
-
-type ListenerConfigClient interface {
-	Create(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	Update(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	UpdateStatus(*v1.ListenerConfig) (*v1.ListenerConfig, error)
-	Delete(namespace, name string, options *metav1.DeleteOptions) error
-	Get(namespace, name string, options metav1.GetOptions) (*v1.ListenerConfig, error)
-	List(namespace string, opts metav1.ListOptions) (*v1.ListenerConfigList, error)
-	Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error)
-	Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ListenerConfig, err error)
-}
-
-type ListenerConfigCache interface {
-	Get(namespace, name string) (*v1.ListenerConfig, error)
-	List(namespace string, selector labels.Selector) ([]*v1.ListenerConfig, error)
-
-	AddIndexer(indexName string, indexer ListenerConfigIndexer)
-	GetByIndex(indexName, key string) ([]*v1.ListenerConfig, error)
-}
-
-type ListenerConfigIndexer func(obj *v1.ListenerConfig) ([]string, error)
-
-type listenerConfigController struct {
-	controllerManager *generic.ControllerManager
-	clientGetter      clientset.ListenerConfigsGetter
-	informer          informers.ListenerConfigInformer
-	gvk               schema.GroupVersionKind
-}
-
-func NewListenerConfigController(gvk schema.GroupVersionKind, controllerManager *generic.ControllerManager, clientGetter clientset.ListenerConfigsGetter, informer informers.ListenerConfigInformer) ListenerConfigController {
-	return &listenerConfigController{
-		controllerManager: controllerManager,
-		clientGetter:      clientGetter,
-		informer:          informer,
-		gvk:               gvk,
-	}
-}
-
-func FromListenerConfigHandlerToHandler(sync ListenerConfigHandler) generic.Handler {
-	return func(key string, obj runtime.Object) (ret runtime.Object, err error) {
-		var v *v1.ListenerConfig
-		if obj == nil {
-			v, err = sync(key, nil)
-		} else {
-			v, err = sync(key, obj.(*v1.ListenerConfig))
-		}
-		if v == nil {
-			return nil, err
-		}
-		return v, err
-	}
-}
-
-func (c *listenerConfigController) Updater() generic.Updater {
-	return func(obj runtime.Object) (runtime.Object, error) {
-		newObj, err := c.Update(obj.(*v1.ListenerConfig))
-		if newObj == nil {
-			return nil, err
-		}
-		return newObj, err
-	}
-}
-
-func UpdateListenerConfigOnChange(updater generic.Updater, handler ListenerConfigHandler) ListenerConfigHandler {
-	return func(key string, obj *v1.ListenerConfig) (*v1.ListenerConfig, error) {
-		if obj == nil {
-			return handler(key, nil)
-		}
-
-		copyObj := obj.DeepCopy()
-		newObj, err := handler(key, copyObj)
-		if newObj != nil {
-			copyObj = newObj
-		}
-		if obj.ResourceVersion == copyObj.ResourceVersion && !equality.Semantic.DeepEqual(obj, copyObj) {
-			newObj, err := updater(copyObj)
-			if newObj != nil && err == nil {
-				copyObj = newObj.(*v1.ListenerConfig)
-			}
-		}
-
-		return copyObj, err
-	}
-}
-
-func (c *listenerConfigController) AddGenericHandler(ctx context.Context, name string, handler generic.Handler) {
-	c.controllerManager.AddHandler(ctx, c.gvk, c.informer.Informer(), name, handler)
-}
-
-func (c *listenerConfigController) AddGenericRemoveHandler(ctx context.Context, name string, handler generic.Handler) {
-	removeHandler := generic.NewRemoveHandler(name, c.Updater(), handler)
-	c.controllerManager.AddHandler(ctx, c.gvk, c.informer.Informer(), name, removeHandler)
-}
-
-func (c *listenerConfigController) OnChange(ctx context.Context, name string, sync ListenerConfigHandler) {
-	c.AddGenericHandler(ctx, name, FromListenerConfigHandlerToHandler(sync))
-}
-
-func (c *listenerConfigController) OnRemove(ctx context.Context, name string, sync ListenerConfigHandler) {
-	removeHandler := generic.NewRemoveHandler(name, c.Updater(), FromListenerConfigHandlerToHandler(sync))
-	c.AddGenericHandler(ctx, name, removeHandler)
-}
-
-func (c *listenerConfigController) Enqueue(namespace, name string) {
-	c.controllerManager.Enqueue(c.gvk, c.informer.Informer(), namespace, name)
-}
-
-func (c *listenerConfigController) Informer() cache.SharedIndexInformer {
-	return c.informer.Informer()
-}
-
-func (c *listenerConfigController) GroupVersionKind() schema.GroupVersionKind {
-	return c.gvk
-}
-
-func (c *listenerConfigController) Cache() ListenerConfigCache {
-	return &listenerConfigCache{
-		lister:  c.informer.Lister(),
-		indexer: c.informer.Informer().GetIndexer(),
-	}
-}
-
-func (c *listenerConfigController) Create(obj *v1.ListenerConfig) (*v1.ListenerConfig, error) {
-	return c.clientGetter.ListenerConfigs(obj.Namespace).Create(obj)
-}
-
-func (c *listenerConfigController) Update(obj *v1.ListenerConfig) (*v1.ListenerConfig, error) {
-	return c.clientGetter.ListenerConfigs(obj.Namespace).Update(obj)
-}
-
-func (c *listenerConfigController) UpdateStatus(obj *v1.ListenerConfig) (*v1.ListenerConfig, error) {
-	return c.clientGetter.ListenerConfigs(obj.Namespace).UpdateStatus(obj)
-}
-
-func (c *listenerConfigController) Delete(namespace, name string, options *metav1.DeleteOptions) error {
-	return c.clientGetter.ListenerConfigs(namespace).Delete(name, options)
-}
-
-func (c *listenerConfigController) Get(namespace, name string, options metav1.GetOptions) (*v1.ListenerConfig, error) {
-	return c.clientGetter.ListenerConfigs(namespace).Get(name, options)
-}
-
-func (c *listenerConfigController) List(namespace string, opts metav1.ListOptions) (*v1.ListenerConfigList, error) {
-	return c.clientGetter.ListenerConfigs(namespace).List(opts)
-}
-
-func (c *listenerConfigController) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) {
-	return c.clientGetter.ListenerConfigs(namespace).Watch(opts)
-}
-
-func (c *listenerConfigController) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.ListenerConfig, err error) {
-	return c.clientGetter.ListenerConfigs(namespace).Patch(name, pt, data, subresources...)
-}
-
-type listenerConfigCache struct {
-	lister  listers.ListenerConfigLister
-	indexer cache.Indexer
-}
-
-func (c *listenerConfigCache) Get(namespace, name string) (*v1.ListenerConfig, error) {
-	return c.lister.ListenerConfigs(namespace).Get(name)
-}
-
-func (c *listenerConfigCache) List(namespace string, selector labels.Selector) ([]*v1.ListenerConfig, error) {
-	return c.lister.ListenerConfigs(namespace).List(selector)
-}
-
-func (c *listenerConfigCache) AddIndexer(indexName string, indexer ListenerConfigIndexer) {
-	utilruntime.Must(c.indexer.AddIndexers(map[string]cache.IndexFunc{
-		indexName: func(obj interface{}) (strings []string, e error) {
-			return indexer(obj.(*v1.ListenerConfig))
-		},
-	}))
-}
-
-func (c *listenerConfigCache) GetByIndex(indexName, key string) (result []*v1.ListenerConfig, err error) {
-	objs, err := c.indexer.ByIndex(indexName, key)
-	if err != nil {
-		return nil, err
-	}
-	for _, obj := range objs {
-		result = append(result, obj.(*v1.ListenerConfig))
-	}
-	return result, nil
-}

pkg/generated/informers/externalversions/generic.go

@@ -55,8 +55,6 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 	// Group=k3s.cattle.io, Version=v1
 	case v1.SchemeGroupVersion.WithResource("addons"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.K3s().V1().Addons().Informer()}, nil
-	case v1.SchemeGroupVersion.WithResource("listenerconfigs"):
-		return &genericInformer{resource: resource.GroupResource(), informer: f.K3s().V1().ListenerConfigs().Informer()}, nil
 
 	}
 

pkg/generated/informers/externalversions/k3s.cattle.io/v1/interface.go

@@ -26,8 +26,6 @@ import (
 type Interface interface {
 	// Addons returns a AddonInformer.
 	Addons() AddonInformer
-	// ListenerConfigs returns a ListenerConfigInformer.
-	ListenerConfigs() ListenerConfigInformer
 }
 
 type version struct {
@@ -45,8 +43,3 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList
 func (v *version) Addons() AddonInformer {
 	return &addonInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
 }
-
-// ListenerConfigs returns a ListenerConfigInformer.
-func (v *version) ListenerConfigs() ListenerConfigInformer {
-	return &listenerConfigInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
-}

pkg/generated/informers/externalversions/k3s.cattle.io/v1/listenerconfig.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	time "time"
-
-	k3scattleiov1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
-	versioned "github.com/rancher/k3s/pkg/generated/clientset/versioned"
-	internalinterfaces "github.com/rancher/k3s/pkg/generated/informers/externalversions/internalinterfaces"
-	v1 "github.com/rancher/k3s/pkg/generated/listers/k3s.cattle.io/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
-	watch "k8s.io/apimachinery/pkg/watch"
-	cache "k8s.io/client-go/tools/cache"
-)
-
-// ListenerConfigInformer provides access to a shared informer and lister for
-// ListenerConfigs.
-type ListenerConfigInformer interface {
-	Informer() cache.SharedIndexInformer
-	Lister() v1.ListenerConfigLister
-}
-
-type listenerConfigInformer struct {
-	factory          internalinterfaces.SharedInformerFactory
-	tweakListOptions internalinterfaces.TweakListOptionsFunc
-	namespace        string
-}
-
-// NewListenerConfigInformer constructs a new informer for ListenerConfig type.
-// Always prefer using an informer factory to get a shared informer instead of getting an independent
-// one. This reduces memory footprint and number of connections to the server.
-func NewListenerConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
-	return NewFilteredListenerConfigInformer(client, namespace, resyncPeriod, indexers, nil)
-}
-
-// NewFilteredListenerConfigInformer constructs a new informer for ListenerConfig type.
-// Always prefer using an informer factory to get a shared informer instead of getting an independent
-// one. This reduces memory footprint and number of connections to the server.
-func NewFilteredListenerConfigInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
-	return cache.NewSharedIndexInformer(
-		&cache.ListWatch{
-			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
-				if tweakListOptions != nil {
-					tweakListOptions(&options)
-				}
-				return client.K3sV1().ListenerConfigs(namespace).List(options)
-			},
-			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
-				if tweakListOptions != nil {
-					tweakListOptions(&options)
-				}
-				return client.K3sV1().ListenerConfigs(namespace).Watch(options)
-			},
-		},
-		&k3scattleiov1.ListenerConfig{},
-		resyncPeriod,
-		indexers,
-	)
-}
-
-func (f *listenerConfigInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	return NewFilteredListenerConfigInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
-}
-
-func (f *listenerConfigInformer) Informer() cache.SharedIndexInformer {
-	return f.factory.InformerFor(&k3scattleiov1.ListenerConfig{}, f.defaultInformer)
-}
-
-func (f *listenerConfigInformer) Lister() v1.ListenerConfigLister {
-	return v1.NewListenerConfigLister(f.Informer().GetIndexer())
-}

pkg/generated/listers/k3s.cattle.io/v1/expansion_generated.go

@@ -25,11 +25,3 @@ type AddonListerExpansion interface{}
 // AddonNamespaceListerExpansion allows custom methods to be added to
 // AddonNamespaceLister.
 type AddonNamespaceListerExpansion interface{}
-
-// ListenerConfigListerExpansion allows custom methods to be added to
-// ListenerConfigLister.
-type ListenerConfigListerExpansion interface{}
-
-// ListenerConfigNamespaceListerExpansion allows custom methods to be added to
-// ListenerConfigNamespaceLister.
-type ListenerConfigNamespaceListerExpansion interface{}

pkg/generated/listers/k3s.cattle.io/v1/listenerconfig.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	v1 "github.com/rancher/k3s/pkg/apis/k3s.cattle.io/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/client-go/tools/cache"
-)
-
-// ListenerConfigLister helps list ListenerConfigs.
-type ListenerConfigLister interface {
-	// List lists all ListenerConfigs in the indexer.
-	List(selector labels.Selector) (ret []*v1.ListenerConfig, err error)
-	// ListenerConfigs returns an object that can list and get ListenerConfigs.
-	ListenerConfigs(namespace string) ListenerConfigNamespaceLister
-	ListenerConfigListerExpansion
-}
-
-// listenerConfigLister implements the ListenerConfigLister interface.
-type listenerConfigLister struct {
-	indexer cache.Indexer
-}
-
-// NewListenerConfigLister returns a new ListenerConfigLister.
-func NewListenerConfigLister(indexer cache.Indexer) ListenerConfigLister {
-	return &listenerConfigLister{indexer: indexer}
-}
-
-// List lists all ListenerConfigs in the indexer.
-func (s *listenerConfigLister) List(selector labels.Selector) (ret []*v1.ListenerConfig, err error) {
-	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
-		ret = append(ret, m.(*v1.ListenerConfig))
-	})
-	return ret, err
-}
-
-// ListenerConfigs returns an object that can list and get ListenerConfigs.
-func (s *listenerConfigLister) ListenerConfigs(namespace string) ListenerConfigNamespaceLister {
-	return listenerConfigNamespaceLister{indexer: s.indexer, namespace: namespace}
-}
-
-// ListenerConfigNamespaceLister helps list and get ListenerConfigs.
-type ListenerConfigNamespaceLister interface {
-	// List lists all ListenerConfigs in the indexer for a given namespace.
-	List(selector labels.Selector) (ret []*v1.ListenerConfig, err error)
-	// Get retrieves the ListenerConfig from the indexer for a given namespace and name.
-	Get(name string) (*v1.ListenerConfig, error)
-	ListenerConfigNamespaceListerExpansion
-}
-
-// listenerConfigNamespaceLister implements the ListenerConfigNamespaceLister
-// interface.
-type listenerConfigNamespaceLister struct {
-	indexer   cache.Indexer
-	namespace string
-}
-
-// List lists all ListenerConfigs in the indexer for a given namespace.
-func (s listenerConfigNamespaceLister) List(selector labels.Selector) (ret []*v1.ListenerConfig, err error) {
-	err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) {
-		ret = append(ret, m.(*v1.ListenerConfig))
-	})
-	return ret, err
-}
-
-// Get retrieves the ListenerConfig from the indexer for a given namespace and name.
-func (s listenerConfigNamespaceLister) Get(name string) (*v1.ListenerConfig, error) {
-	obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name)
-	if err != nil {
-		return nil, err
-	}
-	if !exists {
-		return nil, errors.NewNotFound(v1.Resource("listenerconfig"), name)
-	}
-	return obj.(*v1.ListenerConfig), nil
-}

pkg/passwd/passwd.go

+package passwd
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/token"
+)
+
+type entry struct {
+	pass string
+	role string
+}
+
+type Passwd struct {
+	changed bool
+	names   map[string]entry
+}
+
+func Read(file string) (*Passwd, error) {
+	result := &Passwd{
+		names: map[string]entry{},
+	}
+
+	f, err := os.Open(file)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return result, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+
+	reader := csv.NewReader(f)
+	reader.FieldsPerRecord = -1
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if len(record) < 3 {
+			return nil, fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", file, len(record))
+		}
+		e := entry{
+			pass: record[0],
+		}
+		if len(record) > 3 {
+			e.role = record[3]
+		}
+		result.names[record[1]] = e
+	}
+
+	return result, nil
+}
+
+func (p *Passwd) Check(name, pass string) (matches bool, exists bool) {
+	e, ok := p.names[name]
+	if !ok {
+		return false, false
+	}
+	return e.pass == pass, true
+}
+
+func (p *Passwd) EnsureUser(name, role, passwd string) error {
+	tokenPrefix := "::" + name + ":"
+	idx := strings.Index(passwd, tokenPrefix)
+	if idx > 0 && strings.HasPrefix(passwd, "K10") {
+		passwd = passwd[idx+len(tokenPrefix):]
+	}
+
+	if e, ok := p.names[name]; ok {
+		if passwd != "" && e.pass != passwd {
+			p.changed = true
+			e.pass = passwd
+		}
+
+		if e.role != role {
+			p.changed = true
+			e.role = role
+		}
+
+		p.names[name] = e
+		return nil
+	}
+
+	if passwd == "" {
+		token, err := token.Random(16)
+		if err != nil {
+			return err
+		}
+		passwd = token
+	}
+
+	p.changed = true
+	p.names[name] = entry{
+		pass: passwd,
+		role: role,
+	}
+
+	return nil
+}
+
+func (p *Passwd) Pass(name string) (string, bool) {
+	e, ok := p.names[name]
+	if !ok {
+		return "", false
+	}
+	return e.pass, true
+}
+
+func (p *Passwd) Write(passwdFile string) error {
+	if !p.changed {
+		return nil
+	}
+
+	var records [][]string
+	for name, e := range p.names {
+		records = append(records, []string{
+			e.pass,
+			name,
+			name,
+			e.role,
+		})
+	}
+
+	return writePasswords(passwdFile, records)
+}
+
+func writePasswords(passwdFile string, records [][]string) error {
+	out, err := os.Create(passwdFile + ".tmp")
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	if err := out.Chmod(0600); err != nil {
+		return err
+	}
+
+	if err := csv.NewWriter(out).WriteAll(records); err != nil {
+		return err
+	}
+
+	return os.Rename(passwdFile+".tmp", passwdFile)
+}

pkg/server/auth.go

@@ -9,9 +9,21 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
-func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
+func hasRole(mustRoles []string, roles []string) bool {
+	for _, check := range roles {
+		for _, role := range mustRoles {
+			if role == check {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
 	if serverConfig == nil || serverConfig.Runtime.Authenticator == nil {
-		next.ServeHTTP(rw, req)
+		logrus.Errorf("authenticate not initialized")
+		rw.WriteHeader(http.StatusUnauthorized)
 		return
 	}
 
@@ -22,7 +34,7 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
 		return
 	}
 
-	if !ok || resp.User.GetName() != "node" {
+	if !ok || !hasRole(roles, resp.User.GetGroups()) {
 		rw.WriteHeader(http.StatusUnauthorized)
 		return
 	}
@@ -32,10 +44,10 @@ func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWri
 	next.ServeHTTP(rw, req)
 }
 
-func authMiddleware(serverConfig *config.Control) mux.MiddlewareFunc {
+func authMiddleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-			doAuth(serverConfig, next, rw, req)
+			doAuth(roles, serverConfig, next, rw, req)
 		})
 	}
 }

pkg/server/context.go

@@ -62,7 +62,6 @@ func crds(ctx context.Context, config *rest.Config) error {
 	}
 
 	factory.BatchCreateCRDs(ctx, crd.NamespacedTypes(
-		"ListenerConfig.k3s.cattle.io/v1",
 		"Addon.k3s.cattle.io/v1",
 		"HelmChart.helm.cattle.io/v1")...)
 

pkg/server/router.go

@@ -3,70 +3,68 @@ package server
 import (
 	"crypto"
 	"crypto/x509"
-	"encoding/csv"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/gorilla/mux"
 	certutil "github.com/rancher/dynamiclistener/cert"
+	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/k3s/pkg/daemons/control"
+	"github.com/rancher/k3s/pkg/passwd"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 )
 
 const (
-	jsonMediaType   = "application/json"
-	binaryMediaType = "application/octet-stream"
-	pbMediaType     = "application/com.github.proto-openapi.spec.v2@v1.0+protobuf"
-	openapiPrefix   = "openapi."
-	staticURL       = "/static/"
+	staticURL = "/static/"
 )
 
-type CACertsGetter func() (string, error)
-
-func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CACertsGetter) http.Handler {
+func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.Handler {
 	authed := mux.NewRouter()
-	authed.Use(authMiddleware(serverConfig))
+	authed.Use(authMiddleware(serverConfig, "k3s:agent"))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
-	authed.Path("/v1-k3s/connect").Handler(tunnel)
 	authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
 	authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
 	authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
 	authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
 	authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
 	authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
+	authed.Path("/v1-k3s/client-k3s-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert))
+	authed.Path("/v1-k3s/client-k3s-controller.key").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
 	authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
 
+	nodeAuthed := mux.NewRouter()
+	nodeAuthed.Use(authMiddleware(serverConfig, "system:nodes"))
+	nodeAuthed.Path("/v1-k3s/connect").Handler(tunnel)
+	nodeAuthed.NotFoundHandler = authed
+
+	serverAuthed := mux.NewRouter()
+	serverAuthed.Use(authMiddleware(serverConfig, "k3s:server"))
+	serverAuthed.NotFoundHandler = nodeAuthed
+	serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap))
+
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
 	router := mux.NewRouter()
-	router.NotFoundHandler = authed
+	router.NotFoundHandler = serverAuthed
 	router.PathPrefix(staticURL).Handler(serveStatic(staticURL, staticDir))
-	router.Path("/cacerts").Handler(cacerts(cacertsGetter))
+	router.Path("/cacerts").Handler(cacerts(ca))
 	router.Path("/ping").Handler(ping())
 
 	return router
 }
 
-func cacerts(getter CACertsGetter) http.Handler {
+func cacerts(ca []byte) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
-		content, err := getter()
-		if err != nil {
-			resp.WriteHeader(http.StatusInternalServerError)
-			resp.Write([]byte(err.Error()))
-		}
 		resp.Header().Set("content-type", "text/plain")
-		resp.Write([]byte(content))
+		resp.Write(ca)
 	})
 }
 
@@ -242,38 +240,19 @@ func sendError(err error, resp http.ResponseWriter, status ...int) {
 	resp.Write([]byte(err.Error()))
 }
 
-func ensureNodePassword(passwdFile, nodeName, passwd string) error {
-	records := [][]string{}
-
-	if _, err := os.Stat(passwdFile); !os.IsNotExist(err) {
-		f, err := os.Open(passwdFile)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		reader := csv.NewReader(f)
-		for {
-			record, err := reader.Read()
-			if err == io.EOF {
-				break
-			}
-			if err != nil {
-				return err
-			}
-			if len(record) < 2 {
-				return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record))
-			}
-			if record[1] == nodeName {
-				if record[0] == passwd {
-					return nil
-				}
-				return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
-			}
-			records = append(records, record)
+func ensureNodePassword(passwdFile, nodeName, pass string) error {
+	passwd, err := passwd.Read(passwdFile)
+	if err != nil {
+		return err
+	}
+	match, exists := passwd.Check(nodeName, pass)
+	if exists {
+		if !match {
+			return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
 		}
-		f.Close()
+		return nil
 	}
-
-	records = append(records, []string{passwd, nodeName})
-	return control.WritePasswords(passwdFile, records)
+	// If user doesn't exist we save this password for future validation
+	passwd.EnsureUser(nodeName, "", pass)
+	return passwd.Write(passwdFile)
 }

pkg/server/server.go

@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/helm-controller/pkg/helm"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
@@ -25,10 +24,11 @@ import (
 	"github.com/rancher/k3s/pkg/rootlessports"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
-	"github.com/rancher/k3s/pkg/tls"
+	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
@@ -39,79 +39,67 @@ func resolveDataDir(dataDir string) (string, error) {
 	return filepath.Join(dataDir, "server"), err
 }
 
-func StartServer(ctx context.Context, config *Config) (string, error) {
+func StartServer(ctx context.Context, config *Config) error {
 	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
-		return "", err
+		return err
 	}
 
 	if err := setNoProxyEnv(&config.ControlConfig); err != nil {
-		return "", err
+		return err
 	}
 
 	if err := control.Server(ctx, &config.ControlConfig); err != nil {
-		return "", errors.Wrap(err, "starting kubernetes")
+		return errors.Wrap(err, "starting kubernetes")
 	}
 
-	certs, err := startWrangler(ctx, config)
-	if err != nil {
-		return "", errors.Wrap(err, "starting tls server")
+	if err := startWrangler(ctx, config); err != nil {
+		return errors.Wrap(err, "starting tls server")
 	}
 
-	ip := net2.ParseIP(config.TLSConfig.BindAddress)
+	ip := net2.ParseIP(config.ControlConfig.BindAddress)
 	if ip == nil {
-		ip, err = net.ChooseHostInterface()
-		if err != nil {
+		hostIP, err := net.ChooseHostInterface()
+		if err == nil {
+			ip = hostIP
+		} else {
 			ip = net2.ParseIP("127.0.0.1")
 		}
 	}
-	printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
 
-	writeKubeConfig(certs, &config.TLSConfig, config)
+	if err := printTokens(ip.String(), &config.ControlConfig); err != nil {
+		return err
+	}
 
-	return certs, nil
+	return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
 }
 
-func startWrangler(ctx context.Context, config *Config) (string, error) {
+func startWrangler(ctx context.Context, config *Config) error {
 	var (
 		err           error
-		tlsConfig     = &config.TLSConfig
 		controlConfig = &config.ControlConfig
 	)
 
-	caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
+	ca, err := ioutil.ReadFile(config.ControlConfig.Runtime.ServerCA)
 	if err != nil {
-		return "", err
-	}
-	caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
-	if err != nil {
-		return "", err
+		return err
 	}
 
-	certs := string(caBytes)
-	tlsConfig.CACerts = certs
-	tlsConfig.CAKey = string(caKeyBytes)
-
-	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
-		return certs, nil
-	})
+	controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca)
 
 	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
-		return "", err
+		return err
 	}
 
 	if err := stageFiles(ctx, sc, controlConfig); err != nil {
-		return "", err
+		return err
 	}
 
-	_, err = tls.NewServer(ctx, sc.K3s.K3s().V1().ListenerConfig(), *tlsConfig)
-	if err != nil {
-		return "", err
+	if err := sc.Start(ctx); err != nil {
+		return err
 	}
 
-	if err := startNodeCache(ctx, sc); err != nil {
-		return "", err
-	}
+	controlConfig.Runtime.Core = sc.Core
 
 	start := func(ctx context.Context) {
 		if err := masterControllers(ctx, sc, config); err != nil {
@@ -122,7 +110,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		}
 	}
 	if !config.DisableAgent {
-		go setMasterRoleLabel(ctx, sc)
+		go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node())
 	}
 	if controlConfig.NoLeaderElect {
 		go func() {
@@ -134,7 +122,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		go leader.RunOrDie(ctx, "", "k3s", sc.K8s, start)
 	}
 
-	return certs, nil
+	return nil
 }
 
 func masterControllers(ctx context.Context, sc *Context, config *Config) error {
@@ -162,7 +150,7 @@ func masterControllers(ctx context.Context, sc *Context, config *Config) error {
 	}
 
 	if !config.DisableServiceLB && config.Rootless {
-		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.TLSConfig.HTTPSPort)
+		return rootlessports.Register(ctx, sc.Core.Core().V1().Service(), config.ControlConfig.HTTPSPort)
 	}
 
 	return nil
@@ -203,7 +191,7 @@ func HomeKubeConfig(write, rootless bool) (string, error) {
 	return resolvehome.Resolve(datadir.HomeConfig)
 }
 
-func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
+func printTokens(advertiseIP string, config *config.Control) error {
 	var (
 		nodeFile string
 	)
@@ -212,26 +200,43 @@ func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfi
 		advertiseIP = "127.0.0.1"
 	}
 
-	if len(config.Runtime.NodeToken) > 0 {
-		p := filepath.Join(config.DataDir, "node-token")
-		if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
+	if len(config.Runtime.AgentToken) > 0 {
+		p := filepath.Join(config.DataDir, "token")
+		if err := writeToken(config.Runtime.AgentToken, p, config.Runtime.ServerCA); err == nil {
 			logrus.Infof("Node token is available at %s", p)
 			nodeFile = p
 		}
+
+		// backwards compatibility
+		np := filepath.Join(config.DataDir, "node-token")
+		if !isSymlink(np) {
+			if err := os.RemoveAll(np); err != nil {
+				return err
+			}
+			if err := os.Symlink(p, np); err != nil {
+				return err
+			}
+		}
 	}
 
 	if len(nodeFile) > 0 {
-		printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
+		printToken(config.HTTPSPort, advertiseIP, "To join node to cluster:", "agent")
 	}
+
+	return nil
 }
 
-func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *Config) {
-	clientToken := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
-	ip := tlsConfig.BindAddress
+func writeKubeConfig(certs string, config *Config) error {
+	clientToken, err := FormatToken(config.ControlConfig.Runtime.ClientToken, certs)
+	if err != nil {
+		return err
+	}
+
+	ip := config.ControlConfig.BindAddress
 	if ip == "" {
 		ip = "127.0.0.1"
 	}
-	url := fmt.Sprintf("https://%s:%d", ip, tlsConfig.HTTPSPort)
+	url := fmt.Sprintf("https://%s:%d", ip, config.ControlConfig.HTTPSPort)
 	kubeConfig, err := HomeKubeConfig(true, config.Rootless)
 	def := true
 	if err != nil {
@@ -274,6 +279,8 @@ func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config
 	if def {
 		logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
 	}
+
+	return nil
 }
 
 func setupDataDirAndChdir(config *config.Control) error {
@@ -312,18 +319,22 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd string) {
 	logrus.Infof("%s k3s %s -s https://%s:%d -t ${NODE_TOKEN}", prefix, cmd, ip, httpsPort)
 }
 
-func FormatToken(token string, certs string) string {
+func FormatToken(token string, certFile string) (string, error) {
 	if len(token) == 0 {
-		return token
+		return token, nil
 	}
 
 	prefix := "K10"
-	if len(certs) > 0 {
-		digest := sha256.Sum256([]byte(certs))
+	if len(certFile) > 0 {
+		bytes, err := ioutil.ReadFile(certFile)
+		if err != nil {
+			return "", nil
+		}
+		digest := sha256.Sum256(bytes)
 		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
 	}
 
-	return prefix + token
+	return prefix + token, nil
 }
 
 func writeToken(token, file, certs string) error {
@@ -331,7 +342,10 @@ func writeToken(token, file, certs string) error {
 		return nil
 	}
 
-	token = FormatToken(token, certs)
+	token, err := FormatToken(token, certs)
+	if err != nil {
+		return err
+	}
 	return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
 }
 
@@ -364,26 +378,23 @@ func isSymlink(config string) bool {
 	return false
 }
 
-func setMasterRoleLabel(ctx context.Context, sc *Context) error {
+func setMasterRoleLabel(ctx context.Context, nodes v1.NodeClient) error {
 	for {
 		nodeName := os.Getenv("NODE_NAME")
-		nodeController := sc.Core.Core().V1().Node()
-		nodeCache := nodeController.Cache()
-		nodeCached, err := nodeCache.Get(nodeName)
+		node, err := nodes.Get(nodeName, metav1.GetOptions{})
 		if err != nil {
 			logrus.Infof("Waiting for master node %s startup: %v", nodeName, err)
 			time.Sleep(1 * time.Second)
 			continue
 		}
-		if v, ok := nodeCached.Labels[MasterRoleLabelKey]; ok && v == "true" {
+		if v, ok := node.Labels[MasterRoleLabelKey]; ok && v == "true" {
 			break
 		}
-		node := nodeCached.DeepCopy()
 		if node.Labels == nil {
 			node.Labels = make(map[string]string)
 		}
 		node.Labels[MasterRoleLabelKey] = "true"
-		_, err = nodeController.Update(node)
+		_, err = nodes.Update(node)
 		if err == nil {
 			logrus.Infof("master role label has been set succesfully on node: %s", nodeName)
 			break
@@ -396,9 +407,3 @@ func setMasterRoleLabel(ctx context.Context, sc *Context) error {
 	}
 	return nil
 }
-
-func startNodeCache(ctx context.Context, sc *Context) error {
-	sc.Core.Core().V1().Node().Cache()
-
-	return sc.Start(ctx)
-}

pkg/server/types.go

 package server
 
 import (
-	"github.com/rancher/dynamiclistener"
 	"github.com/rancher/k3s/pkg/daemons/config"
 )
 
 type Config struct {
 	DisableAgent     bool
 	DisableServiceLB bool
-	TLSConfig        dynamiclistener.UserConfig
 	ControlConfig    config.Control
 	Rootless         bool
 }

pkg/token/read.go

+package token
+
+import (
+	cryptorand "crypto/rand"
+	"encoding/hex"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+func Random(size int) (string, error) {
+	token := make([]byte, size, size)
+	_, err := cryptorand.Read(token)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(token), err
+}
+
+func ReadFile(path string) (string, error) {
+	if path == "" {
+		return "", nil
+	}
+
+	for {
+		tokenBytes, err := ioutil.ReadFile(path)
+		if err == nil {
+			return strings.TrimSpace(string(tokenBytes)), nil
+		} else if os.IsNotExist(err) {
+			logrus.Infof("Waiting for %s to be available\n", path)
+			time.Sleep(2 * time.Second)
+		} else {
+			return "", err
+		}
+	}
+}

vendor/github.com/konsorten/go-windows-terminal-sequences/README.md

@@ -26,7 +26,6 @@ The tool is sponsored by the [marvin + konsorten GmbH](http://www.konsorten.de).
 We thank all the authors who provided code to this library:
 
 * Felix Kollmann
-* Nicolas Perraut
 
 ## License
 

vendor/github.com/konsorten/go-windows-terminal-sequences/sequences_dummy.go

-// +build linux darwin
-
-package sequences
-
-import (
-	"fmt"
-)
-
-func EnableVirtualTerminalProcessing(stream uintptr, enable bool) error {
-	return fmt.Errorf("windows only package")
-}

vendor/github.com/rancher/dynamiclistener/factory/ca.go

+package factory
+
+import (
+	"crypto/ecdsa"
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+)
+
+func GenCA() (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	caKey, err := NewPrivateKey()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caCert, err := NewSelfSignedCACert(caKey, "dynamiclistener-ca", "dynamiclistener-org")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return caCert, caKey, nil
+}
+
+func LoadOrGenCA() (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	cert, key, err := loadCA()
+	if err == nil {
+		return cert, key, nil
+	}
+
+	cert, key, err = GenCA()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	certBytes, keyBytes, err := Marshal(cert, key)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if err := os.MkdirAll("./certs", 0700); err != nil {
+		return nil, nil, err
+	}
+
+	if err := ioutil.WriteFile("./certs/ca.pem", certBytes, 0600); err != nil {
+		return nil, nil, err
+	}
+
+	if err := ioutil.WriteFile("./certs/ca.key", keyBytes, 0600); err != nil {
+		return nil, nil, err
+	}
+
+	return cert, key, nil
+}
+
+func loadCA() (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	return LoadCerts("./certs/ca.pem", "./certs/ca.key")
+}
+
+func LoadCerts(certFile, keyFile string) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	caPem, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, nil, err
+	}
+	caKey, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	key, err := ParseECPrivateKeyPEM(caKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cert, err := ParseCertPEM(caPem)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return cert, key, nil
+}

vendor/github.com/rancher/dynamiclistener/factory/cert_utils.go

+package factory
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math"
+	"math/big"
+	"net"
+	"time"
+)
+
+const (
+	ECPrivateKeyBlockType = "EC PRIVATE KEY"
+	CertificateBlockType  = "CERTIFICATE"
+)
+
+func NewSelfSignedCACert(key crypto.Signer, cn string, org ...string) (*x509.Certificate, error) {
+	now := time.Now()
+	tmpl := x509.Certificate{
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		NotAfter:              now.Add(time.Hour * 24 * 365 * 10).UTC(),
+		NotBefore:             now.UTC(),
+		SerialNumber:          new(big.Int).SetInt64(0),
+		Subject: pkix.Name{
+			CommonName:   cn,
+			Organization: org,
+		},
+	}
+
+	certDERBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, key.Public(), key)
+	if err != nil {
+		return nil, err
+	}
+
+	return x509.ParseCertificate(certDERBytes)
+}
+
+func NewSignedCert(signer crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer, cn string, orgs []string,
+	domains []string, ips []net.IP) (*x509.Certificate, error) {
+
+	serialNumber, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+
+	parent := x509.Certificate{
+		DNSNames:     domains,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses:  ips,
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		NotAfter:     time.Now().Add(time.Hour * 24 * 365).UTC(),
+		NotBefore:    caCert.NotBefore,
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   cn,
+			Organization: orgs,
+		},
+	}
+
+	cert, err := x509.CreateCertificate(rand.Reader, &parent, caCert, signer.Public(), caKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return x509.ParseCertificate(cert)
+}
+
+func ParseECPrivateKeyPEM(keyData []byte) (*ecdsa.PrivateKey, error) {
+	var privateKeyPemBlock *pem.Block
+	for {
+		privateKeyPemBlock, keyData = pem.Decode(keyData)
+		if privateKeyPemBlock == nil {
+			break
+		}
+
+		if privateKeyPemBlock.Type == ECPrivateKeyBlockType {
+			return x509.ParseECPrivateKey(privateKeyPemBlock.Bytes)
+		}
+	}
+
+	return nil, fmt.Errorf("pem does not include a valid EC private key")
+}
+
+func ParseCertPEM(pemCerts []byte) (*x509.Certificate, error) {
+	var pemBlock *pem.Block
+	for {
+		pemBlock, pemCerts = pem.Decode(pemCerts)
+		if pemBlock == nil {
+			break
+		}
+
+		if pemBlock.Type == CertificateBlockType {
+			return x509.ParseCertificate(pemBlock.Bytes)
+		}
+	}
+
+	return nil, fmt.Errorf("pem does not include a valid x509 cert")
+}

vendor/github.com/rancher/dynamiclistener/factory/gen.go

+package factory
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/pem"
+	"net"
+	"sort"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	cnPrefix = "listener.cattle.io/cn-"
+	static   = "listener.cattle.io/static"
+	hashKey  = "listener.cattle.io/hash"
+)
+
+type TLS struct {
+	CACert       *x509.Certificate
+	CAKey        crypto.Signer
+	CN           string
+	Organization []string
+}
+
+func collectCNs(secret *v1.Secret) (domains []string, ips []net.IP, hash string, err error) {
+	var (
+		cns    []string
+		digest = sha256.New()
+	)
+	for k, v := range secret.Annotations {
+		if strings.HasPrefix(k, cnPrefix) {
+			cns = append(cns, v)
+		}
+	}
+
+	sort.Strings(cns)
+
+	for _, v := range cns {
+		digest.Write([]byte(v))
+		ip := net.ParseIP(v)
+		if ip == nil {
+			domains = append(domains, v)
+		} else {
+			ips = append(ips, ip)
+		}
+	}
+
+	hash = hex.EncodeToString(digest.Sum(nil))
+	return
+}
+
+func (t *TLS) AddCN(secret *v1.Secret, cn ...string) (*v1.Secret, bool, error) {
+	var (
+		err error
+	)
+
+	if !NeedsUpdate(secret, cn...) {
+		return secret, false, nil
+	}
+
+	secret = populateCN(secret, cn...)
+
+	privateKey, err := getPrivateKey(secret)
+	if err != nil {
+		return nil, false, err
+	}
+
+	domains, ips, hash, err := collectCNs(secret)
+	if err != nil {
+		return nil, false, err
+	}
+
+	newCert, err := t.newCert(domains, ips, privateKey)
+	if err != nil {
+		return nil, false, err
+	}
+
+	certBytes, keyBytes, err := Marshal(newCert, privateKey)
+	if err != nil {
+		return nil, false, err
+	}
+
+	if secret.Data == nil {
+		secret.Data = map[string][]byte{}
+	}
+	secret.Data[v1.TLSCertKey] = certBytes
+	secret.Data[v1.TLSPrivateKeyKey] = keyBytes
+	secret.Annotations[hashKey] = hash
+
+	return secret, true, nil
+}
+
+func (t *TLS) newCert(domains []string, ips []net.IP, privateKey *ecdsa.PrivateKey) (*x509.Certificate, error) {
+	return NewSignedCert(privateKey, t.CACert, t.CAKey, t.CN, t.Organization, domains, ips)
+}
+
+func populateCN(secret *v1.Secret, cn ...string) *v1.Secret {
+	secret = secret.DeepCopy()
+	if secret.Annotations == nil {
+		secret.Annotations = map[string]string{}
+	}
+	for _, cn := range cn {
+		secret.Annotations[cnPrefix+cn] = cn
+	}
+	return secret
+}
+
+func NeedsUpdate(secret *v1.Secret, cn ...string) bool {
+	if secret.Annotations[static] == "true" {
+		return false
+	}
+
+	for _, cn := range cn {
+		if secret.Annotations[cnPrefix+cn] == "" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getPrivateKey(secret *v1.Secret) (*ecdsa.PrivateKey, error) {
+	keyBytes := secret.Data[v1.TLSPrivateKeyKey]
+	if len(keyBytes) == 0 {
+		return NewPrivateKey()
+	}
+
+	privateKey, err := ParseECPrivateKeyPEM(keyBytes)
+	if err == nil {
+		return privateKey, nil
+	}
+
+	return NewPrivateKey()
+}
+
+func Marshal(x509Cert *x509.Certificate, privateKey *ecdsa.PrivateKey) ([]byte, []byte, error) {
+	certBlock := pem.Block{
+		Type:  CertificateBlockType,
+		Bytes: x509Cert.Raw,
+	}
+
+	keyBytes, err := x509.MarshalECPrivateKey(privateKey)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	keyBlock := pem.Block{
+		Type:  ECPrivateKeyBlockType,
+		Bytes: keyBytes,
+	}
+
+	return pem.EncodeToMemory(&certBlock), pem.EncodeToMemory(&keyBlock), nil
+}
+
+func NewPrivateKey() (*ecdsa.PrivateKey, error) {
+	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+}

vendor/github.com/rancher/dynamiclistener/go.mod

@@ -3,16 +3,9 @@ module github.com/rancher/dynamiclistener
 go 1.12
 
 require (
-	github.com/hashicorp/golang-lru v0.5.1
-	github.com/kisielk/gotool v1.0.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/rancher/wrangler v0.1.4
+	github.com/rancher/wrangler-api v0.2.0
 	github.com/sirupsen/logrus v1.4.1
-	github.com/stretchr/testify v1.3.0 // indirect
-	github.com/stripe/safesql v0.2.0 // indirect
-	golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284
-	golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c // indirect
-	golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect
-	golang.org/x/text v0.3.2 // indirect
-	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
-	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
+	k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b
+	k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
 )

vendor/github.com/rancher/dynamiclistener/listener.go

+package dynamiclistener
+
+import (
+	"crypto"
+	"crypto/tls"
+	"crypto/x509"
+	"net"
+	"net/http"
+	"sync"
+
+	"github.com/rancher/dynamiclistener/factory"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+)
+
+type TLSStorage interface {
+	Get() (*v1.Secret, error)
+	Update(secret *v1.Secret) error
+}
+
+type Config struct {
+	CN           string
+	Organization []string
+	TLSConfig    tls.Config
+	SANs         []string
+}
+
+func NewListener(l net.Listener, storage TLSStorage, caCert *x509.Certificate, caKey crypto.Signer, config Config) (net.Listener, http.Handler, error) {
+	if config.CN == "" {
+		config.CN = "dynamic"
+	}
+	if len(config.Organization) == 0 {
+		config.Organization = []string{"dynamic"}
+	}
+
+	dynamicListener := &listener{
+		factory: &factory.TLS{
+			CACert:       caCert,
+			CAKey:        caKey,
+			CN:           config.CN,
+			Organization: config.Organization,
+		},
+		Listener:  l,
+		storage:   &nonNil{storage: storage},
+		sans:      config.SANs,
+		tlsConfig: config.TLSConfig,
+	}
+	dynamicListener.tlsConfig.GetCertificate = dynamicListener.getCertificate
+
+	return tls.NewListener(dynamicListener, &dynamicListener.tlsConfig), dynamicListener.cacheHandler(), nil
+}
+
+type listener struct {
+	sync.RWMutex
+	net.Listener
+
+	factory   *factory.TLS
+	storage   TLSStorage
+	version   string
+	tlsConfig tls.Config
+	cert      *tls.Certificate
+	sans      []string
+}
+
+func (l *listener) Accept() (net.Conn, error) {
+	conn, err := l.Listener.Accept()
+	if err != nil {
+		return conn, err
+	}
+
+	addr := conn.RemoteAddr()
+	if addr == nil {
+		return conn, nil
+	}
+
+	host, _, err := net.SplitHostPort(addr.String())
+	if err != nil {
+		logrus.Errorf("failed to parse network %s: %v", addr.Network(), err)
+		return conn, nil
+	}
+
+	return conn, l.updateCert(host)
+}
+
+func (l *listener) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	if hello.ServerName != "" {
+		if err := l.updateCert(hello.ServerName); err != nil {
+			return nil, err
+		}
+	}
+
+	return l.loadCert()
+}
+
+func (l *listener) updateCert(cn string) error {
+	l.RLock()
+	defer l.RUnlock()
+
+	secret, err := l.storage.Get()
+	if err != nil {
+		return err
+	}
+
+	if !factory.NeedsUpdate(secret, append(l.sans, cn)...) {
+		return nil
+	}
+
+	l.RUnlock()
+	l.Lock()
+	defer l.RLock()
+	defer l.Unlock()
+
+	secret, updated, err := l.factory.AddCN(secret, append(l.sans, cn)...)
+	if err != nil {
+		return err
+	}
+
+	if updated {
+		if err := l.storage.Update(secret); err != nil {
+			return err
+		}
+		// clear version to force cert reload
+		l.version = ""
+	}
+
+	return nil
+}
+
+func (l *listener) loadCert() (*tls.Certificate, error) {
+	l.RLock()
+	defer l.RUnlock()
+
+	secret, err := l.storage.Get()
+	if err != nil {
+		return nil, err
+	}
+	if l.cert != nil && l.version == secret.ResourceVersion {
+		return l.cert, nil
+	}
+
+	defer l.RLock()
+	l.RUnlock()
+	l.Lock()
+	defer l.Unlock()
+
+	secret, err = l.storage.Get()
+	if err != nil {
+		return nil, err
+	}
+	if l.cert != nil && l.version == secret.ResourceVersion {
+		return l.cert, nil
+	}
+
+	cert, err := tls.X509KeyPair(secret.Data[v1.TLSCertKey], secret.Data[v1.TLSPrivateKeyKey])
+	if err != nil {
+		return nil, err
+	}
+
+	l.cert = &cert
+	return l.cert, nil
+}
+
+func (l *listener) cacheHandler() http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		h, _, err := net.SplitHostPort(req.Host)
+		if err != nil {
+			h = req.Host
+		}
+
+		ip := net.ParseIP(h)
+		if len(ip) > 0 {
+			l.updateCert(h)
+		}
+	})
+}
+
+type nonNil struct {
+	sync.Mutex
+	storage TLSStorage
+}
+
+func (n *nonNil) Get() (*v1.Secret, error) {
+	n.Lock()
+	defer n.Unlock()
+
+	s, err := n.storage.Get()
+	if err != nil || s == nil {
+		return &v1.Secret{}, err
+	}
+	return s, nil
+}
+
+func (n *nonNil) Update(secret *v1.Secret) error {
+	n.Lock()
+	defer n.Unlock()
+
+	return n.storage.Update(secret)
+}

vendor/github.com/rancher/dynamiclistener/read.go

-package dynamiclistener
-
-import (
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-)
-
-func ReadTLSConfig(userConfig *UserConfig) error {
-	var err error
-
-	path := userConfig.CertPath
-
-	userConfig.CACerts, err = readPEM(filepath.Join(path, "cacerts.pem"))
-	if err != nil {
-		return err
-	}
-
-	userConfig.Key, err = readPEM(filepath.Join(path, "key.pem"))
-	if err != nil {
-		return err
-	}
-
-	userConfig.Cert, err = readPEM(filepath.Join(path, "cert.pem"))
-	if err != nil {
-		return err
-	}
-
-	valid := false
-	if userConfig.Key != "" && userConfig.Cert != "" {
-		valid = true
-	} else if userConfig.Key == "" && userConfig.Cert == "" {
-		valid = true
-	}
-
-	if !valid {
-		return fmt.Errorf("invalid SSL configuration found, please set cert/key, cert/key/cacerts, cacerts only, or none")
-	}
-
-	return nil
-}
-
-func readPEM(path string) (string, error) {
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", nil
-	}
-
-	return string(content), nil
-}

vendor/github.com/rancher/dynamiclistener/redirect.go

+package dynamiclistener
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+)
+
+// Approach taken from letsencrypt, except manglePort is specific to us
+func HTTPRedirect(next http.Handler) http.Handler {
+	return http.HandlerFunc(
+		func(rw http.ResponseWriter, r *http.Request) {
+			fmt.Println("!!!!!", r.URL.String(), r.Header)
+			if r.Header.Get("x-Forwarded-Proto") == "https" ||
+				r.Header.Get("x-Forwarded-Proto") == "wss" ||
+				strings.HasPrefix(r.URL.Path, "/ping") ||
+				strings.HasPrefix(r.URL.Path, "/health") {
+				next.ServeHTTP(rw, r)
+				return
+			}
+			if r.Method != "GET" && r.Method != "HEAD" {
+				http.Error(rw, "Use HTTPS", http.StatusBadRequest)
+				return
+			}
+			target := "https://" + manglePort(r.Host) + r.URL.RequestURI()
+			http.Redirect(rw, r, target, http.StatusFound)
+		})
+}
+
+func manglePort(hostport string) string {
+	host, port, err := net.SplitHostPort(hostport)
+	if err != nil {
+		return hostport
+	}
+
+	portInt, err := strconv.Atoi(port)
+	if err != nil {
+		return hostport
+	}
+
+	portInt = ((portInt / 1000) * 1000) + 443
+
+	return net.JoinHostPort(host, strconv.Itoa(portInt))
+}

vendor/github.com/rancher/dynamiclistener/storage/file/file.go

+package file
+
+import (
+	"encoding/json"
+	"github.com/rancher/dynamiclistener"
+	"k8s.io/api/core/v1"
+	"os"
+)
+
+func New(file string) dynamiclistener.TLSStorage {
+	return &storage{
+		file: file,
+	}
+}
+
+type storage struct {
+	file string
+}
+
+func (s *storage) Get() (*v1.Secret, error) {
+	f, err := os.Open(s.file)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	secret := v1.Secret{}
+	return &secret, json.NewDecoder(f).Decode(&secret)
+}
+
+func (s *storage) Update(secret *v1.Secret) error {
+	f, err := os.Create(s.file)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return json.NewEncoder(f).Encode(secret)
+}
+

vendor/github.com/rancher/dynamiclistener/storage/kubernetes/controller.go

+package kubernetes
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/rancher/dynamiclistener"
+	"github.com/rancher/wrangler-api/pkg/generated/controllers/core"
+	v1controller "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/rancher/wrangler/pkg/start"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+)
+
+type CoreGetter func() *core.Factory
+
+func New(ctx context.Context, core CoreGetter, namespace, name string, backing dynamiclistener.TLSStorage) dynamiclistener.TLSStorage {
+	storage := &storage{
+		name:      name,
+		namespace: namespace,
+		storage:   backing,
+		ctx:       ctx,
+	}
+
+	// lazy init
+	go func() {
+		for {
+			core := core()
+			if core != nil {
+				storage.init(core.Core().V1().Secret())
+				start.All(ctx, 5, core)
+				return
+			}
+
+			select {
+			case <-ctx.Done():
+				return
+			case <-time.After(time.Second):
+			}
+		}
+	}()
+
+	return storage
+}
+
+type storage struct {
+	sync.Mutex
+
+	namespace, name string
+	storage         dynamiclistener.TLSStorage
+	secrets         v1controller.SecretClient
+	ctx             context.Context
+}
+
+func (s *storage) init(secrets v1controller.SecretController) {
+	s.Lock()
+	defer s.Unlock()
+
+	secrets.OnChange(s.ctx, "tls-storage", func(key string, secret *v1.Secret) (*v1.Secret, error) {
+		if secret == nil {
+			return nil, nil
+		}
+		if secret.Namespace == s.namespace && secret.Name == s.name {
+			if err := s.Update(secret); err != nil {
+				return nil, err
+			}
+		}
+
+		return secret, nil
+	})
+	s.secrets = secrets
+}
+
+func (s *storage) Get() (*v1.Secret, error) {
+	s.Lock()
+	defer s.Unlock()
+
+	return s.storage.Get()
+}
+
+func (s *storage) Update(secret *v1.Secret) (err error) {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.secrets != nil {
+		if secret.UID == "" {
+			secret.Name = s.name
+			secret.Namespace = s.namespace
+			secret, err = s.secrets.Create(secret)
+			if err != nil {
+				return err
+			}
+		} else {
+			existingSecret, err := s.storage.Get()
+			if err != nil {
+				return err
+			}
+			if !equality.Semantic.DeepEqual(secret.Data, existingSecret.Data) {
+				secret, err = s.secrets.Update(secret)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return s.storage.Update(secret)
+}

vendor/github.com/rancher/dynamiclistener/storage/memory/memory.go

+package memory
+
+import (
+	"github.com/rancher/dynamiclistener"
+	v1 "k8s.io/api/core/v1"
+)
+
+func New() dynamiclistener.TLSStorage {
+	return &memory{}
+}
+
+func NewBacked(storage dynamiclistener.TLSStorage) dynamiclistener.TLSStorage {
+	return &memory{storage: storage}
+}
+
+type memory struct {
+	storage dynamiclistener.TLSStorage
+	secret *v1.Secret
+}
+
+func (m *memory) Get() (*v1.Secret, error) {
+	if m.secret == nil && m.storage != nil {
+		secret, err := m.storage.Get()
+		if err != nil {
+			return nil, err
+		}
+		m.secret = secret
+	}
+
+	return m.secret, nil
+}
+
+func (m *memory) Update(secret *v1.Secret) error {
+	if m.storage != nil {
+		if err := m.storage.Update(secret); err != nil {
+			return err
+		}
+	}
+
+	m.secret = secret
+	return nil
+}

vendor/github.com/rancher/dynamiclistener/tcp.go

+package dynamiclistener
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"time"
+)
+
+func NewTCPListener(ip string, port int) (net.Listener, error) {
+	l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
+	if err != nil {
+		return nil, err
+	}
+
+	tcpListener, ok := l.(*net.TCPListener)
+	if !ok {
+		return nil, fmt.Errorf("wrong listener type: %v", reflect.TypeOf(tcpListener))
+	}
+
+	return tcpKeepAliveListener{
+		TCPListener: tcpListener,
+	}, nil
+}
+
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(3 * time.Minute)
+	return tc, nil
+}

vendor/github.com/rancher/dynamiclistener/types.go

-package dynamiclistener
-
-import (
-	"net/http"
-)
-
-type ListenerConfigStorage interface {
-	Set(*ListenerStatus) (*ListenerStatus, error)
-	Get() (*ListenerStatus, error)
-}
-
-type ServerInterface interface {
-	Update(status *ListenerStatus) error
-	CACert() (string, error)
-}
-
-type UserConfig struct {
-	// Required fields
-
-	Handler   http.Handler
-	HTTPPort  int
-	HTTPSPort int
-	CertPath  string
-
-	// Optional fields
-
-	KnownIPs    []string
-	Domains     []string
-	Mode        string
-	NoCACerts   bool
-	CACerts     string
-	CAKey       string
-	Cert        string
-	Key         string
-	BindAddress string
-}
-
-type ListenerStatus struct {
-	Revision       string            `json:"revision,omitempty"`
-	CACert         string            `json:"caCert,omitempty"`
-	CAKey          string            `json:"caKey,omitempty"`
-	GeneratedCerts map[string]string `json:"generatedCerts" norman:"nocreate,noupdate"`
-	KnownIPs       map[string]bool   `json:"knownIps" norman:"nocreate,noupdate"`
-}
-
-func (l *ListenerStatus) DeepCopyInto(t *ListenerStatus) {
-	t.Revision = l.Revision
-	t.CACert = l.CACert
-	t.CAKey = l.CAKey
-	t.GeneratedCerts = copyMap(t.GeneratedCerts)
-	t.KnownIPs = map[string]bool{}
-	for k, v := range l.KnownIPs {
-		t.KnownIPs[k] = v
-	}
-}
-
-func copyMap(m map[string]string) map[string]string {
-	ret := map[string]string{}
-	for k, v := range m {
-		ret[k] = v
-	}
-	return ret
-}

vendor/github.com/rancher/kine/pkg/client/client.go

-package client
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/coreos/etcd/clientv3"
-	"github.com/rancher/kine/pkg/endpoint"
-)
-
-type Value struct {
-	Data     []byte
-	Modified int64
-}
-
-type Client interface {
-	Get(ctx context.Context, key string) (Value, error)
-	Put(ctx context.Context, key string, value []byte) error
-	Create(ctx context.Context, key string, value []byte) error
-	Update(ctx context.Context, key string, revision int64, value []byte) error
-	Close() error
-}
-
-type client struct {
-	c *clientv3.Client
-}
-
-func New(config endpoint.ETCDConfig) (Client, error) {
-	tlsConfig, err := config.TLSConfig.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-
-	c, err := clientv3.New(clientv3.Config{
-		Endpoints:   config.Endpoints,
-		DialTimeout: 5 * time.Second,
-		TLS:         tlsConfig,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return &client{
-		c: c,
-	}, nil
-}
-
-func (c *client) Get(ctx context.Context, key string) (Value, error) {
-	resp, err := c.c.Get(ctx, key)
-	if err != nil {
-		return Value{}, err
-	}
-
-	if len(resp.Kvs) == 1 {
-		return Value{
-			Data:     resp.Kvs[0].Value,
-			Modified: resp.Kvs[0].ModRevision,
-		}, nil
-	}
-
-	return Value{}, nil
-}
-
-func (c *client) Put(ctx context.Context, key string, value []byte) error {
-	val, err := c.Get(ctx, key)
-	if err != nil {
-		return err
-	}
-	if val.Modified == 0 {
-		return c.Create(ctx, key, value)
-	}
-	return c.Update(ctx, key, val.Modified, value)
-}
-
-func (c *client) Create(ctx context.Context, key string, value []byte) error {
-	resp, err := c.c.Txn(ctx).
-		If(clientv3.Compare(clientv3.ModRevision(key), "=", 0)).
-		Then(clientv3.OpPut(key, string(value))).
-		Commit()
-	if err != nil {
-		return err
-	}
-	if !resp.Succeeded {
-		return fmt.Errorf("key exists")
-	}
-	return nil
-}
-
-func (c *client) Update(ctx context.Context, key string, revision int64, value []byte) error {
-	resp, err := c.c.Txn(ctx).
-		If(clientv3.Compare(clientv3.ModRevision(key), "=", revision)).
-		Then(clientv3.OpPut(key, string(value))).
-		Else(clientv3.OpGet(key)).
-		Commit()
-	if err != nil {
-		return err
-	}
-	if !resp.Succeeded {
-		return fmt.Errorf("revision %d doesnt match", revision)
-	}
-	return nil
-}
-
-func (c *client) Close() error {
-	return c.c.Close()
-}