Merge pull request #359 from erikwilson/refactor-certs

manifests/rolebindings.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-apiserver-kubelet-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kubelet-api-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: kube-apiserver

pkg/agent/config/config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/net"
@@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
 		}
 		defer resp.Body.Close()
 
+		if resp.StatusCode == http.StatusForbidden {
+			return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
+		}
+
 		if resp.StatusCode != http.StatusOK {
 			return nil, fmt.Errorf("%s: %s", u, resp.Status)
 		}
@@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
 		return "", err
 	}
 	nodePassword := hex.EncodeToString(password)
-	return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600)
+	return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
 }
 
-func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
-	nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
+func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
+	servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil {
+	if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil {
 		return nil, errors.Wrapf(err, "failed to write node cert")
 	}
 
-	nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info)
+	servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info)
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil {
+	if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
 		return nil, errors.Wrapf(err, "failed to write node key")
 	}
 
-	cert, err := tls.X509KeyPair(nodeCert, nodeKey)
+	cert, err := tls.X509KeyPair(servingCert, servingKey)
 	if err != nil {
 		return nil, err
 	}
 	return &cert, nil
 }
 
-func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
-	clientCABytes := pem.EncodeToMemory(&pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: nodeCert.Certificate[1],
-	})
-
-	clientCA := filepath.Join(dataDir, "client-ca.pem")
-	if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil {
-		return "", errors.Wrapf(err, "failed to write client CA")
+func getHostFile(filename string, info *clientaccess.Info) error {
+	basename := filepath.Base(filename)
+	fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
 	}
+	return nil
+}
 
-	return clientCA, nil
+func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error {
+	basename := filepath.Base(filename)
+	fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile))
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
+	}
+	return nil
 }
 
 func getHostnameAndIP(info cmds.Agent) (string, string, error) {
@@ -169,17 +184,15 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
 }
 
 func localAddress(controlConfig *config.Control) string {
-	return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
+	return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort)
 }
 
-func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
+func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) {
 	os.MkdirAll(envInfo.DataDir, 0700)
 	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
-
-	info.URL = "https://" + localAddress(controlConfig)
 	info.CACerts = pem.EncodeToMemory(&pem.Block{
 		Type:  cert.CertificateBlockType,
-		Bytes: nodeCert.Certificate[1],
+		Bytes: tlsCert.Certificate[1],
 	})
 
 	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
@@ -253,36 +266,70 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
-	nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt")
-	nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key")
-	nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
-
-	nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info)
+	hostLocal, err := exec.LookPath("host-local")
 	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find host-local")
+	}
+
+	var flannelIface *sysnet.Interface
+	if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
+		flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to find interface")
+		}
+	}
+
+	clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt")
+	if err := getHostFile(clientCAFile, info); err != nil {
 		return nil, err
 	}
 
-	clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert)
-	if err != nil {
+	serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt")
+	if err := getHostFile(serverCAFile, info); err != nil {
 		return nil, err
 	}
 
-	kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert)
+	servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
+	servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
+	nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
+	servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
 	if err != nil {
 		return nil, err
 	}
 
-	hostLocal, err := exec.LookPath("host-local")
+	kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find host-local")
+		return nil, err
 	}
 
-	var flannelIface *sysnet.Interface
-	if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
-		flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
-		if err != nil {
-			return nil, errors.Wrapf(err, "unable to find interface")
-		}
+	clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
+	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
+		return nil, err
+	}
+
+	clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key")
+	if err := getHostFile(clientKubeletKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig")
+	if err := control.KubeConfig(kubeconfigKubelet, info.URL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
+		return nil, err
+	}
+
+	clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt")
+	if err := getHostFile(clientKubeProxyCert, info); err != nil {
+		return nil, err
+	}
+
+	clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key")
+	if err := getHostFile(clientKubeProxyKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig")
+	if err := control.KubeConfig(kubeconfigKubeproxy, info.URL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
+		return nil, err
 	}
 
 	nodeConfig := &config.Node{
@@ -295,14 +342,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
 	nodeConfig.AgentConfig.NodeIP = nodeIP
 	nodeConfig.AgentConfig.NodeName = nodeName
-	nodeConfig.AgentConfig.NodeCertFile = nodeCertFile
-	nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile
+	nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
+	nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
 	nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
 	nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
-	nodeConfig.AgentConfig.CACertPath = clientCA
+	nodeConfig.AgentConfig.ClientCA = clientCAFile
 	nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
-	nodeConfig.AgentConfig.KubeConfig = kubeConfig
+	nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
+	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
+	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
 	nodeConfig.CACerts = info.CACerts
@@ -316,7 +365,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
 	nodeConfig.ServerAddress = serverURLParsed.Host
-	nodeConfig.Certificate = nodeCert
+	nodeConfig.Certificate = servingCert
 	if !nodeConfig.NoFlannel {
 		nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
 		nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)

pkg/agent/flannel/setup.go

@@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error {
 func Run(ctx context.Context, config *config.Node) error {
 	nodeName := config.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}
@@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig)
+		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 

pkg/agent/proxy/proxy.go

-package proxy
-
-import (
-	"crypto/tls"
-	"net/http"
-
-	"github.com/pkg/errors"
-	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/k3s/pkg/proxy"
-	"github.com/sirupsen/logrus"
-)
-
-func Run(config *config.Node) error {
-	proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
-	if err != nil {
-		return err
-	}
-
-	listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{
-		Certificates: []tls.Certificate{
-			*config.Certificate,
-		},
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "Failed to start tls listener")
-	}
-
-	go func() {
-		err := http.Serve(listener, proxy)
-		logrus.Fatalf("TLS proxy stopped: %v", err)
-	}()
-
-	return nil
-}

pkg/agent/run.go

@@ -12,7 +12,6 @@ import (
 	"github.com/rancher/k3s/pkg/agent/config"
 	"github.com/rancher/k3s/pkg/agent/containerd"
 	"github.com/rancher/k3s/pkg/agent/flannel"
-	"github.com/rancher/k3s/pkg/agent/proxy"
 	"github.com/rancher/k3s/pkg/agent/syssetup"
 	"github.com/rancher/k3s/pkg/agent/tunnel"
 	"github.com/rancher/k3s/pkg/cli/cmds"
@@ -52,10 +51,6 @@ func run(ctx context.Context, cfg cmds.Agent) error {
 		return err
 	}
 
-	if err := proxy.Run(nodeConfig); err != nil {
-		return err
-	}
-
 	if err := agent.Agent(&nodeConfig.AgentConfig); err != nil {
 		return err
 	}

pkg/agent/tunnel/tunnel.go

@@ -26,7 +26,7 @@ var (
 )
 
 func Setup(config *config.Node) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}

pkg/cli/cmds/server.go

@@ -17,7 +17,7 @@ type Server struct {
 	DisableAgent        bool
 	KubeConfigOutput    string
 	KubeConfigMode      string
-	KnownIPs            cli.StringSlice
+	TLSSan              cli.StringSlice
 	BindAddress         string
 	ExtraAPIArgs        cli.StringSlice
 	ExtraSchedulerArgs  cli.StringSlice
@@ -28,6 +28,8 @@ type Server struct {
 	StorageCAFile       string
 	StorageCertFile     string
 	StorageKeyFile      string
+	AdvertiseIP         string
+	AdvertisePort       int
 }
 
 var ServerConfig Server
@@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			cli.StringSliceFlag{
 				Name:  "tls-san",
 				Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
-				Value: &ServerConfig.KnownIPs,
+				Value: &ServerConfig.TLSSan,
 			},
 			cli.StringSliceFlag{
 				Name:  "kube-apiserver-arg",
@@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.StorageKeyFile,
 				EnvVar:      "K3S_STORAGE_KEYFILE",
 			},
+			cli.StringFlag{
+				Name:        "advertise-address",
+				Usage:       "IP address that apiserver uses to advertise to members of the cluster",
+				Destination: &ServerConfig.AdvertiseIP,
+			},
+			cli.IntFlag{
+				Name:        "advertise-port",
+				Usage:       "Port that apiserver uses to advertise to members of the cluster",
+				Value:       0,
+				Destination: &ServerConfig.AdvertisePort,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,

pkg/cli/server/server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/pkg/volume/csi"
 
 	_ "github.com/go-sql-driver/mysql" // ensure we have mysql
@@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.Rootless = cfg.Rootless
 	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
-	serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)
+	for _, san := range knownIPs(cfg.TLSSan) {
+		addr := net2.ParseIP(san)
+		if addr != nil {
+			serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
+		} else {
+			serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
+		}
+	}
 	serverConfig.TLSConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
 	serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
@@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile
 	serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile
 	serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile
+	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
+	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
+
+	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
+		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
+	}
+	if serverConfig.ControlConfig.AdvertiseIP != "" {
+		serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
+	}
 
 	_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
 	if err != nil {
@@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err)
 	}
 
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+	serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
+
 	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
 	if cfg.ClusterDNS == "" {

pkg/daemons/agent/agent.go

@@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) {
 	argsMap := map[string]string{
 		"proxy-mode":           "iptables",
 		"healthz-bind-address": "127.0.0.1",
-		"kubeconfig":           cfg.KubeConfig,
+		"kubeconfig":           cfg.KubeConfigKubeProxy,
 		"cluster-cidr":         cfg.ClusterCIDR.String(),
 	}
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
@@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) {
 		"read-only-port":           "0",
 		"allow-privileged":         "true",
 		"cluster-domain":           cfg.ClusterDomain,
-		"kubeconfig":               cfg.KubeConfig,
+		"kubeconfig":               cfg.KubeConfigKubelet,
 		"eviction-hard":            "imagefs.available<5%,nodefs.available<5%",
 		"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
 		"fail-swap-on":             "false",
@@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) {
 	if cfg.ListenAddress != "" {
 		argsMap["address"] = cfg.ListenAddress
 	}
-	if cfg.CACertPath != "" {
+	if cfg.ClientCA != "" {
 		argsMap["anonymous-auth"] = "false"
-		argsMap["client-ca-file"] = cfg.CACertPath
+		argsMap["client-ca-file"] = cfg.ClientCA
 	}
-	if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" {
-		argsMap["tls-cert-file"] = cfg.NodeCertFile
-		argsMap["tls-private-key-file"] = cfg.NodeKeyFile
+	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
+		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
+		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
 	}
 	if cfg.NodeName != "" {
 		argsMap["hostname-override"] = cfg.NodeName

pkg/daemons/config/types.go

@@ -36,32 +36,41 @@ type Containerd struct {
 }
 
 type Agent struct {
-	NodeName           string
-	NodeCertFile       string
-	NodeKeyFile        string
-	ClusterCIDR        net.IPNet
-	ClusterDNS         net.IP
-	ClusterDomain      string
-	ResolvConf         string
-	RootDir            string
-	KubeConfig         string
-	NodeIP             string
-	RuntimeSocket      string
-	ListenAddress      string
-	CACertPath         string
-	CNIBinDir          string
-	CNIConfDir         string
-	ExtraKubeletArgs   []string
-	ExtraKubeProxyArgs []string
-	PauseImage         string
-	CNIPlugin          bool
-	NodeTaints         []string
-	NodeLabels         []string
+	NodeName            string
+	ClientKubeletCert   string
+	ClientKubeletKey    string
+	ClientKubeProxyCert string
+	ClientKubeProxyKey  string
+	ServingKubeletCert  string
+	ServingKubeletKey   string
+	ClusterCIDR         net.IPNet
+	ClusterDNS          net.IP
+	ClusterDomain       string
+	ResolvConf          string
+	RootDir             string
+	KubeConfigNode      string
+	KubeConfigKubelet   string
+	KubeConfigKubeProxy string
+	NodeIP              string
+	RuntimeSocket       string
+	ListenAddress       string
+	ClientCA            string
+	CNIBinDir           string
+	CNIConfDir          string
+	ExtraKubeletArgs    []string
+	ExtraKubeProxyArgs  []string
+	PauseImage          string
+	CNIPlugin           bool
+	NodeTaints          []string
+	NodeLabels          []string
 }
 
 type Control struct {
 	AdvertisePort         int
+	AdvertiseIP           string
 	ListenPort            int
+	HTTPSPort             int
+	ProxyPort             int
 	ClusterSecret         string
 	ClusterIPRange        *net.IPNet
 	ServiceIPRange        *net.IPNet
@@ -87,28 +96,45 @@ type Control struct {
 }
 
 type ControlRuntime struct {
-	TLSCert          string
-	TLSKey           string
-	TLSCA            string
-	TLSCAKey         string
-	TokenCA          string
-	TokenCAKey       string
-	ServiceKey       string
-	PasswdFile       string
-	KubeConfigSystem string
-
-	NodeCert      string
-	NodeKey       string
-	ClientToken   string
-	NodeToken     string
-	Handler       http.Handler
-	Tunnel        http.Handler
-	Authenticator authenticator.Request
+	ClientKubeAPICert string
+	ClientKubeAPIKey  string
+	ClientCA          string
+	ClientCAKey       string
+	ServerCA          string
+	ServerCAKey       string
+	ServiceKey        string
+	PasswdFile        string
+	NodePasswdFile    string
+
+	KubeConfigAdmin      string
+	KubeConfigController string
+	KubeConfigScheduler  string
+	KubeConfigAPIServer  string
+
+	ServingKubeAPICert string
+	ServingKubeAPIKey  string
+	ClientToken        string
+	NodeToken          string
+	Handler            http.Handler
+	Tunnel             http.Handler
+	Authenticator      authenticator.Request
 
 	RequestHeaderCA     string
 	RequestHeaderCAKey  string
 	ClientAuthProxyCert string
 	ClientAuthProxyKey  string
+
+	ClientAdminCert      string
+	ClientAdminKey       string
+	ClientControllerCert string
+	ClientControllerKey  string
+	ClientSchedulerCert  string
+	ClientSchedulerKey   string
+	ClientKubeProxyCert  string
+	ClientKubeProxyKey   string
+
+	ServingKubeletKey string
+	ClientKubeletKey  string
 }
 
 type ArgString []string

pkg/deploy/zz_generated_bindata.go

 // Code generated by go-bindata.
 // sources:
 // manifests/coredns.yaml
+// manifests/rolebindings.yaml
 // manifests/traefik.yaml
 // DO NOT EDIT!
 
@@ -89,6 +90,26 @@ func corednsYaml() (*asset, error) {
 	return a, nil
 }
 
+var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\xcf\xbd\x0a\xc2\x40\x10\x04\xe0\xfe\x9e\xe2\x5e\xe0\x22\x76\x72\xa5\x16\xf6\x01\xed\x37\xb9\x55\xd7\xdc\x1f\xbb\x7b\x01\x7d\x7a\x09\x48\x1a\x51\xb0\x1c\x18\xe6\x63\xa0\xd2\x19\x59\xa8\x64\x6f\x79\x80\xb1\x83\xa6\xb7\xc2\xf4\x04\xa5\x92\xbb\x69\x27\x1d\x95\xcd\xbc\x35\x13\xe5\xe0\xed\x21\x36\x51\xe4\xbe\x44\xdc\x53\x0e\x94\xaf\x26\xa1\x42\x00\x05\x6f\xac\xcd\x90\xd0\xdb\xa9\x0d\xe8\xa0\x92\x20\xcf\xc8\x6e\x89\x11\xd5\x41\x48\x94\x0d\x97\x88\x3d\x5e\x96\x36\x54\x3a\x72\x69\xf5\x87\x6c\xac\xfd\x80\x57\x47\x1e\xa2\x98\xfc\xba\x5f\xe9\x6d\x48\x1b\xee\x38\xaa\x78\xe3\xfe\x42\x4e\x82\xfc\xe5\x85\x79\x05\x00\x00\xff\xff\x54\xf2\x55\xe2\x29\x01\x00\x00")
+
+func rolebindingsYamlBytes() ([]byte, error) {
+	return bindataRead(
+		_rolebindingsYaml,
+		"rolebindings.yaml",
+	)
+}
+
+func rolebindingsYaml() (*asset, error) {
+	bytes, err := rolebindingsYamlBytes()
+	if err != nil {
+		return nil, err
+	}
+
+	info := bindataFileInfo{name: "rolebindings.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)}
+	a := &asset{bytes: bytes, info: info}
+	return a, nil
+}
+
 var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\xcf\x4f\x4b\xc3\x40\x10\x05\xf0\x7b\x3e\xc5\x50\xe8\xb1\xbb\x16\xc4\xc3\xde\xfc\x13\x50\x04\x29\x56\xbd\xca\x64\xf3\xda\x0c\xdd\x6c\xc2\xce\xa4\xa0\xe2\x77\x97\x94\x1c\x3d\xce\xcc\xe3\xc7\x3c\x1e\xe5\x03\x45\x65\xc8\x81\x3a\xa4\xde\x45\x36\x4b\x70\x32\xf8\xf3\xb6\x3a\x49\x6e\x03\x3d\x22\xf5\xf7\x1d\x17\xab\x7a\x18\xb7\x6c\x1c\x2a\xa2\xcc\x3d\x02\x59\x61\x1c\xe4\xb4\xcc\x3a\x72\x44\xa0\xd3\xd4\x60\xa3\x5f\x6a\xe8\x2b\x1d\x11\xe7\x78\x9c\x81\x40\x9d\xd9\xa8\xc1\xfb\xf5\xcf\xf3\xfb\x5d\xfd\xfa\x52\xbf\xd5\xfb\xcf\xdb\xdd\xd3\xef\xda\xab\xb1\x49\xf4\x97\xa0\xfa\x05\xde\x6c\xdd\xcd\xb5\xbb\x72\x76\xfc\xae\x88\x14\x36\x5b\x44\xa5\xe1\xe8\x90\xb9\x49\x68\x03\xad\xac\x4c\x58\x5d\x0e\xaa\xe9\xdf\xfd\xfc\x52\xc9\x30\xa8\x93\x7c\x2c\x50\xad\x73\x3b\x0e\x92\xcd\x4d\x8a\x07\x1c\x78\x4a\xb6\x9b\x9a\x24\xda\xa1\xdd\xa3\x9c\x65\x6e\xb2\x08\x7f\x01\x00\x00\xff\xff\x90\xbb\x64\x2c\x26\x01\x00\x00")
 
 func traefikYamlBytes() ([]byte, error) {
@@ -161,8 +182,9 @@ func AssetNames() []string {
 
 // _bindata is a table, holding each asset generator, mapped to its name.
 var _bindata = map[string]func() (*asset, error){
-	"coredns.yaml": corednsYaml,
-	"traefik.yaml": traefikYaml,
+	"coredns.yaml":      corednsYaml,
+	"rolebindings.yaml": rolebindingsYaml,
+	"traefik.yaml":      traefikYaml,
 }
 
 // AssetDir returns the file names below a certain
@@ -206,8 +228,9 @@ type bintree struct {
 }
 
 var _bintree = &bintree{nil, map[string]*bintree{
-	"coredns.yaml": &bintree{corednsYaml, map[string]*bintree{}},
-	"traefik.yaml": &bintree{traefikYaml, map[string]*bintree{}},
+	"coredns.yaml":      &bintree{corednsYaml, map[string]*bintree{}},
+	"rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}},
+	"traefik.yaml":      &bintree{traefikYaml, map[string]*bintree{}},
 }}
 
 // RestoreAsset restores an asset under the given directory

pkg/server/router.go

 package server
 
 import (
-	"bufio"
-	"crypto/rsa"
+	"crypto"
 	"crypto/x509"
+	"encoding/csv"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -17,10 +18,10 @@ import (
 	"github.com/gorilla/mux"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control"
 	"github.com/rancher/k3s/pkg/openapi"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
-	"k8s.io/kubernetes/pkg/master"
 )
 
 const (
@@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC
 	authed.Use(authMiddleware(serverConfig))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
 	authed.Path("/v1-k3s/connect").Handler(tunnel)
-	authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig))
-	authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig))
+	authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
+	authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
+	authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
+	authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
+	authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
+	authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
+	authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
+	authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
@@ -65,82 +72,122 @@ func cacerts(getter CACertsGetter) http.Handler {
 	})
 }
 
-func nodeCrt(server *config.Control) http.Handler {
+func getNodeInfo(req *http.Request) (string, string, error) {
+	nodeNames := req.Header["K3s-Node-Name"]
+	if len(nodeNames) != 1 || nodeNames[0] == "" {
+		return "", "", errors.New("node name not set")
+	}
+
+	nodePasswords := req.Header["K3s-Node-Password"]
+	if len(nodePasswords) != 1 || nodePasswords[0] == "" {
+		return "", "", errors.New("node password not set")
+	}
+
+	return strings.ToLower(nodeNames[0]), nodePasswords[0], nil
+}
+
+func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) {
+	keyBytes, err := ioutil.ReadFile(signingKeyFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	key, err := certutil.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(caKeyFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caBytes, err := ioutil.ReadFile(caCertFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
+}
+
+func servingKubeletCert(server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		nodeNames := req.Header["K3s-Node-Name"]
-		if len(nodeNames) != 1 || nodeNames[0] == "" {
-			sendError(errors.New("node name not set"), resp)
-			return
-		}
-
-		nodePasswords := req.Header["K3s-Node-Password"]
-		if len(nodePasswords) != 1 || nodePasswords[0] == "" {
-			sendError(errors.New("node password not set"), resp)
-			return
+		nodeName, nodePassword, err := getNodeInfo(req)
+		if err != nil {
+			sendError(err, resp)
 		}
 
-		if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil {
+		if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil {
 			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
-		nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey)
+		caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		key, err := certutil.ParsePrivateKeyPEM(nodeKey)
+		cert, err := certutil.NewSignedCert(certutil.Config{
+			CommonName: nodeName,
+			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+			AltNames: certutil.AltNames{
+				DNSNames: []string{nodeName, "localhost"},
+				IPs:      []net.IP{net.ParseIP("127.0.0.1")},
+			},
+		}, key, caCert[0], caKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey)
-		if err != nil {
-			sendError(err, resp)
-			return
-		}
+		resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+	})
+}
 
-		caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA)
-		if err != nil {
-			sendError(err, resp)
+func clientKubeletCert(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+		nodeName, nodePassword, err := getNodeInfo(req)
 		if err != nil {
 			sendError(err, resp)
-			return
 		}
 
-		caCert, err := certutil.ParseCertsPEM(caBytes)
-		if err != nil {
-			sendError(err, resp)
+		if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil {
+			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
-		_, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange)
+		caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		cfg := certutil.Config{
-			CommonName: "kubernetes",
-			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
-			AltNames: certutil.AltNames{
-				DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]},
-				IPs:      []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")},
-			},
-		}
-
-		cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey))
+		cert, err := certutil.NewSignedCert(certutil.Config{
+			CommonName:   "system:node:" + nodeName,
+			Organization: []string{"system:nodes"},
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		}, key, caCert[0], caKey)
 		if err != nil {
 			sendError(err, resp)
 			return
@@ -150,13 +197,13 @@ func nodeCrt(server *config.Control) http.Handler {
 	})
 }
 
-func nodeKey(server *config.Control) http.Handler {
+func fileHandler(fileName string) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
-		http.ServeFile(resp, req, server.Runtime.NodeKey)
+		http.ServeFile(resp, req, fileName)
 	})
 }
 
@@ -218,36 +265,37 @@ func sendError(err error, resp http.ResponseWriter, status ...int) {
 }
 
 func ensureNodePassword(passwdFile, nodeName, passwd string) error {
-	f, err := os.Open(passwdFile)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	user := strings.ToLower("node:" + nodeName)
-
-	buf := &strings.Builder{}
-	scan := bufio.NewScanner(f)
-	for scan.Scan() {
-		line := scan.Text()
-		parts := strings.Split(line, ",")
-		if len(parts) < 4 {
-			continue
-		}
-		if parts[1] == user {
-			if parts[0] == passwd {
-				return nil
+	records := [][]string{}
+
+	if _, err := os.Stat(passwdFile); !os.IsNotExist(err) {
+		f, err := os.Open(passwdFile)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		reader := csv.NewReader(f)
+		for {
+			record, err := reader.Read()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
 			}
-			return fmt.Errorf("Node password validation failed for [%s]", nodeName)
+			if len(record) < 2 {
+				return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record))
+			}
+			if record[1] == nodeName {
+				if record[0] == passwd {
+					return nil
+				}
+				return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
+			}
+			records = append(records, record)
 		}
-		buf.WriteString(line)
-		buf.WriteString("\n")
-	}
-	buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user))
-
-	if scan.Err() != nil {
-		return scan.Err()
+		f.Close()
 	}
 
-	f.Close()
-	return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600)
+	records = append(records, []string{passwd, nodeName})
+	return control.WritePasswords(passwdFile, records)
 }

pkg/server/server.go

@@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		controlConfig = &config.ControlConfig
 	)
 
+	caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
+	if err != nil {
+		return "", err
+	}
+	caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
+	if err != nil {
+		return "", err
+	}
+
+	tlsConfig.CACerts = string(caBytes)
+	tlsConfig.CAKey = string(caKeyBytes)
+
 	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
 		if tlsServer == nil {
 			return "", nil
@@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		return tlsServer.CACert()
 	})
 
-	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem)
+	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
 		return "", err
 	}

pkg/tls/storage.go

@@ -19,6 +19,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont
 	storage := &listenerConfigStorage{
 		client: listenerConfigs,
 		cache:  listenerConfigs.Cache(),
+		config: config,
 	}
 
 	server, err := dynamiclistener.NewServer(storage, config)
@@ -30,7 +31,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont
 		if obj == nil {
 			return nil, nil
 		}
-		return obj, server.Update(fromStorage(obj))
+		return obj, server.Update(storage.fromStorage(obj))
 	})
 
 	return server, err
@@ -39,6 +40,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont
 type listenerConfigStorage struct {
 	cache  k3sclient.ListenerConfigCache
 	client k3sclient.ListenerConfigClient
+	config dynamiclistener.UserConfig
 }
 
 func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dynamiclistener.ListenerStatus, error) {
@@ -53,7 +55,7 @@ func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dy
 		})
 
 		ls, err := l.client.Create(ls)
-		return fromStorage(ls), err
+		return l.fromStorage(ls), err
 	} else if err != nil {
 		return nil, err
 	}
@@ -63,8 +65,13 @@ func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dy
 	obj.Status = *config
 	obj.Status.Revision = ""
 
+	if l.config.CACerts != "" && l.config.CAKey != "" {
+		obj.Status.CACert = ""
+		obj.Status.CAKey = ""
+	}
+
 	obj, err = l.client.Update(obj)
-	return fromStorage(obj), err
+	return l.fromStorage(obj), err
 }
 
 func (l *listenerConfigStorage) Get() (*dynamiclistener.ListenerStatus, error) {
@@ -75,15 +82,21 @@ func (l *listenerConfigStorage) Get() (*dynamiclistener.ListenerStatus, error) {
 	if errors.IsNotFound(err) {
 		return &dynamiclistener.ListenerStatus{}, nil
 	}
-	return fromStorage(obj), err
+	return l.fromStorage(obj), err
 }
 
-func fromStorage(obj *v1.ListenerConfig) *dynamiclistener.ListenerStatus {
+func (l *listenerConfigStorage) fromStorage(obj *v1.ListenerConfig) *dynamiclistener.ListenerStatus {
 	if obj == nil {
 		return nil
 	}
 
 	copy := obj.DeepCopy()
 	copy.Status.Revision = obj.ResourceVersion
+
+	if l.config.CACerts != "" && l.config.CAKey != "" {
+		copy.Status.CACert = l.config.CACerts
+		copy.Status.CAKey = l.config.CAKey
+	}
+
 	return &copy.Status
 }

trash.lock

@@ -225,7 +225,8 @@ import:
 - package: github.com/prometheus/procfs
   version: 65c1f6f8f0fc1e2185eb9863a3bc751496404259
 - package: github.com/rancher/dynamiclistener
-  version: 077eb13a904f2c62496f31b158135d9743526f82
+  version: 03208cf106d553d58d3a73267aa473a45af63120
+  repo: https://github.com/erikwilson/rancher-dynamiclistener.git
 - package: github.com/rancher/helm-controller
   version: d5f5c830231110722f14d446d3b2038e5cdf1532
 - package: github.com/rancher/remotedialer
@@ -310,7 +311,7 @@ import:
 - package: k8s.io/klog
   version: v0.2.0-14-g8e90cee79f8237
 - package: k8s.io/kubernetes
-  version: v1.14.3-k3s.1
+  version: v1.14.3-k3s.2
   repo: https://github.com/rancher/k3s.git
   transitive: true
   staging: true

vendor.conf

@@ -9,7 +9,8 @@ package=github.com/opencontainers/runc/libcontainer/nsenter
 package=github.com/opencontainers/runc/libcontainer/specconv
 package=github.com/opencontainers/runc/contrib/cmd/recvtty
 
-k8s.io/kubernetes                   v1.14.3-k3s.1                             https://github.com/rancher/k3s.git                transitive=true,staging=true
+k8s.io/kubernetes                   v1.14.3-k3s.2                             https://github.com/rancher/k3s.git                transitive=true,staging=true
+github.com/rancher/dynamiclistener  03208cf106d553d58d3a73267aa473a45af63120  https://github.com/erikwilson/rancher-dynamiclistener.git
 
 github.com/rancher/wrangler         4202dbfa88013c19238bb004d82e013f0593493d
 github.com/rancher/wrangler-api     efe26ac6a9d720e1bfa5a8cc5f8dce5ad598ce26

vendor/github.com/rancher/dynamiclistener/server.go

@@ -3,6 +3,8 @@ package dynamiclistener
 import (
 	"bytes"
 	"context"
+	"crypto"
+	"crypto/ecdsa"
 	"crypto/md5"
 	"crypto/rsa"
 	"crypto/tls"
@@ -46,7 +48,7 @@ type server struct {
 	// dynamic config change on refresh
 	activeCert        *tls.Certificate
 	activeCA          *x509.Certificate
-	activeCAKey       *rsa.PrivateKey
+	activeCAKey       crypto.Signer
 	activeCAKeyString string
 	domains           map[string]bool
 }
@@ -91,6 +93,40 @@ func (s *server) CACert() (string, error) {
 	return status.CACert, nil
 }
 
+func marshalPrivateKey(privateKey crypto.Signer) (string, []byte, error) {
+	var (
+		keyType string
+		bytes   []byte
+		err     error
+	)
+	if key, ok := privateKey.(*ecdsa.PrivateKey); ok {
+		keyType = cert.ECPrivateKeyBlockType
+		bytes, err = x509.MarshalECPrivateKey(key)
+	} else if key, ok := privateKey.(*rsa.PrivateKey); ok {
+		keyType = cert.RSAPrivateKeyBlockType
+		bytes = x509.MarshalPKCS1PrivateKey(key)
+	} else {
+		keyType = cert.PrivateKeyBlockType
+		bytes, err = x509.MarshalPKCS8PrivateKey(privateKey)
+	}
+	if err != nil {
+		logrus.Errorf("Unable to marshal private key: %v", err)
+	}
+	return keyType, bytes, err
+}
+
+func newPrivateKey() (crypto.Signer, error) {
+	caKeyBytes, err := cert.MakeEllipticPrivateKeyPEM()
+	if err != nil {
+		return nil, err
+	}
+	caKeyIFace, err := cert.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return nil, err
+	}
+	return caKeyIFace.(crypto.Signer), nil
+}
+
 func (s *server) save() {
 	if s.activeCert != nil {
 		return
@@ -114,7 +150,10 @@ func (s *server) save() {
 	}
 
 	for key, cert := range s.certs {
-		certStr := certToString(cert)
+		certStr, err := certToString(cert)
+		if err != nil {
+			continue
+		}
 		if cfg.GeneratedCerts[key] != certStr {
 			cfg.GeneratedCerts[key] = certStr
 			changed = true
@@ -139,9 +178,14 @@ func (s *server) save() {
 		}
 
 		caKeyBuffer := bytes.Buffer{}
+		keyType, keyBytes, err := marshalPrivateKey(s.activeCAKey)
+		if err != nil {
+			return
+		}
+
 		if err := pem.Encode(&caKeyBuffer, &pem.Block{
-			Type:  cert.RSAPrivateKeyBlockType,
-			Bytes: x509.MarshalPKCS1PrivateKey(s.activeCAKey),
+			Type:  keyType,
+			Bytes: keyBytes,
 		}); err != nil {
 			return
 		}
@@ -198,8 +242,8 @@ func (s *server) userConfigure() error {
 	return nil
 }
 
-func genCA() (*x509.Certificate, *rsa.PrivateKey, error) {
-	caKey, err := cert.NewPrivateKey()
+func genCA() (*x509.Certificate, crypto.Signer, error) {
+	caKey, err := newPrivateKey()
 	if err != nil {
 		return nil, nil, err
 	}
@@ -225,7 +269,7 @@ func (s *server) Update(status *ListenerStatus) error {
 			s.Unlock()
 			return err
 		}
-		s.activeCAKey = cert.PrivateKey.(*rsa.PrivateKey)
+		s.activeCAKey = cert.PrivateKey.(crypto.Signer)
 		s.activeCAKeyString = status.CAKey
 
 		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
@@ -380,12 +424,25 @@ func (s *server) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, e
 	changed = true
 
 	if s.activeCA == nil {
-		ca, key, err := genCA()
-		if err != nil {
-			return nil, err
+		if s.userConfig.CACerts != "" && s.userConfig.CAKey != "" {
+			ca, err := cert.ParseCertsPEM([]byte(s.userConfig.CACerts))
+			if err != nil {
+				return nil, err
+			}
+			key, err := cert.ParsePrivateKeyPEM([]byte(s.userConfig.CAKey))
+			if err != nil {
+				return nil, err
+			}
+			s.activeCA = ca[0]
+			s.activeCAKey = key.(crypto.Signer)
+		} else {
+			ca, key, err := genCA()
+			if err != nil {
+				return nil, err
+			}
+			s.activeCA = ca
+			s.activeCAKey = key
 		}
-		s.activeCA = ca
-		s.activeCAKey = key
 	}
 
 	cfg := cert.Config{
@@ -398,7 +455,7 @@ func (s *server) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, e
 		},
 	}
 
-	key, err := cert.NewPrivateKey()
+	key, err := newPrivateKey()
 	if err != nil {
 		return nil, err
 	}
@@ -439,6 +496,7 @@ func (s *server) cacheIPHandler(handler http.Handler) http.Handler {
 
 func (s *server) serveHTTPS() error {
 	conf := &tls.Config{
+		ClientAuth:               tls.RequestClientCert,
 		GetCertificate:           s.getCertificate,
 		PreferServerCipherSuites: true,
 	}
@@ -602,32 +660,36 @@ func stringToCert(certString string) *tls.Certificate {
 		return nil
 	}
 
-	cert, key := parts[0], parts[1]
-	keyBytes, err := base64.StdEncoding.DecodeString(key)
+	certPart, keyPart := parts[0], parts[1]
+	keyBytes, err := base64.StdEncoding.DecodeString(keyPart)
 	if err != nil {
 		return nil
 	}
 
-	rsaKey, err := x509.ParsePKCS1PrivateKey(keyBytes)
+	key, err := cert.ParsePrivateKeyPEM(keyBytes)
 	if err != nil {
 		return nil
 	}
 
-	certBytes, err := base64.StdEncoding.DecodeString(cert)
+	certBytes, err := base64.StdEncoding.DecodeString(certPart)
 	if err != nil {
 		return nil
 	}
 
 	return &tls.Certificate{
 		Certificate: [][]byte{certBytes},
-		PrivateKey:  rsaKey,
+		PrivateKey:  key,
 	}
 }
 
-func certToString(cert *tls.Certificate) string {
+func certToString(cert *tls.Certificate) (string, error) {
+	_, keyBytes, err := marshalPrivateKey(cert.PrivateKey.(crypto.Signer))
+	if err != nil {
+		return "", err
+	}
 	certString := base64.StdEncoding.EncodeToString(cert.Certificate[0])
-	keyString := base64.StdEncoding.EncodeToString(x509.MarshalPKCS1PrivateKey(cert.PrivateKey.(*rsa.PrivateKey)))
-	return certString + "#" + keyString
+	keyString := base64.StdEncoding.EncodeToString(keyBytes)
+	return certString + "#" + keyString, nil
 }
 
 type tcpKeepAliveListener struct {

vendor/github.com/rancher/dynamiclistener/types.go

@@ -29,6 +29,7 @@ type UserConfig struct {
 	Mode        string
 	NoCACerts   bool
 	CACerts     string
+	CAKey       string
 	Cert        string
 	Key         string
 	BindAddress string

vendor/k8s.io/client-go/pkg/version/base.go

@@ -3,8 +3,8 @@ package version
 var (
 	gitMajor = "1"
 	gitMinor = "14"
-	gitVersion   = "v1.14.3-k3s.1"
-	gitCommit    = "8343999292c55c807be4406fcaa9f047e8751ffd"
+	gitVersion   = "v1.14.3-k3s.2"
+	gitCommit    = "6174f1fed28fd19300038f6578bf48e3920fa7ba"
 	gitTreeState = "clean"
-	buildDate = "2019-06-12T04:56+00:00Z"
+	buildDate = "2019-06-21T08:17+00:00Z"
 )

vendor/k8s.io/kubernetes/pkg/master/controller.go

@@ -142,6 +142,10 @@ func (c *Controller) Start() {
 		return
 	}
 
+	// Service definition is reconciled during first run to correct port and type per expectations.
+	if err := c.UpdateKubernetesService(true); err != nil {
+		klog.Errorf("Unable to perform initial Kubernetes service initialization: %v", err)
+	}
 	// Reconcile during first run removing itself until server is ready.
 	endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https", c.ExtraEndpointPorts)
 	if err := c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts); err != nil {

vendor/k8s.io/kubernetes/pkg/version/base.go

@@ -3,8 +3,8 @@ package version
 var (
 	gitMajor = "1"
 	gitMinor = "14"
-	gitVersion   = "v1.14.3-k3s.1"
-	gitCommit    = "8343999292c55c807be4406fcaa9f047e8751ffd"
+	gitVersion   = "v1.14.3-k3s.2"
+	gitCommit    = "6174f1fed28fd19300038f6578bf48e3920fa7ba"
 	gitTreeState = "clean"
-	buildDate = "2019-06-12T04:56+00:00Z"
+	buildDate = "2019-06-21T08:17+00:00Z"
 )