Refactor certs

manifests/rolebindings.yaml

+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-apiserver-kubelet-admin
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kubelet-api-admin
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: kube-apiserver

pkg/agent/config/config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/apimachinery/pkg/util/net"
@@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
 		}
 		defer resp.Body.Close()
 
+		if resp.StatusCode == http.StatusForbidden {
+			return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
+		}
+
 		if resp.StatusCode != http.StatusOK {
 			return nil, fmt.Errorf("%s: %s", u, resp.Status)
 		}
@@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
 		return "", err
 	}
 	nodePassword := hex.EncodeToString(password)
-	return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600)
+	return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
 }
 
-func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
-	nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
+func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
+	servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil {
+	if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil {
 		return nil, errors.Wrapf(err, "failed to write node cert")
 	}
 
-	nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info)
+	servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info)
 	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil {
+	if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
 		return nil, errors.Wrapf(err, "failed to write node key")
 	}
 
-	cert, err := tls.X509KeyPair(nodeCert, nodeKey)
+	cert, err := tls.X509KeyPair(servingCert, servingKey)
 	if err != nil {
 		return nil, err
 	}
 	return &cert, nil
 }
 
-func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
-	clientCABytes := pem.EncodeToMemory(&pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: nodeCert.Certificate[1],
-	})
-
-	clientCA := filepath.Join(dataDir, "client-ca.pem")
-	if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil {
-		return "", errors.Wrapf(err, "failed to write client CA")
+func getHostFile(filename string, info *clientaccess.Info) error {
+	basename := filepath.Base(filename)
+	fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info)
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
 	}
+	return nil
+}
 
-	return clientCA, nil
+func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error {
+	basename := filepath.Base(filename)
+	fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile))
+	if err != nil {
+		return err
+	}
+	if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
+		return errors.Wrapf(err, "failed to write cert %s", filename)
+	}
+	return nil
 }
 
 func getHostnameAndIP(info cmds.Agent) (string, string, error) {
@@ -169,17 +184,17 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
 }
 
 func localAddress(controlConfig *config.Control) string {
-	return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
+	return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort)
 }
 
-func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
+func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) {
 	os.MkdirAll(envInfo.DataDir, 0700)
 	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
 
 	info.URL = "https://" + localAddress(controlConfig)
 	info.CACerts = pem.EncodeToMemory(&pem.Block{
 		Type:  cert.CertificateBlockType,
-		Bytes: nodeCert.Certificate[1],
+		Bytes: tlsCert.Certificate[1],
 	})
 
 	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
@@ -253,36 +268,72 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 		return nil, err
 	}
 
-	nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt")
-	nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key")
-	nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
-
-	nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info)
+	hostLocal, err := exec.LookPath("host-local")
 	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find host-local")
+	}
+
+	var flannelIface *sysnet.Interface
+	if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
+		flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unable to find interface")
+		}
+	}
+
+	proxyURL := "https://" + localAddress(controlConfig)
+
+	clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt")
+	if err := getHostFile(clientCAFile, info); err != nil {
 		return nil, err
 	}
 
-	clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert)
-	if err != nil {
+	serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt")
+	if err := getHostFile(serverCAFile, info); err != nil {
 		return nil, err
 	}
 
-	kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert)
+	servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
+	servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
+	nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
+	servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
 	if err != nil {
 		return nil, err
 	}
 
-	hostLocal, err := exec.LookPath("host-local")
+	kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to find host-local")
+		return nil, err
 	}
 
-	var flannelIface *sysnet.Interface
-	if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
-		flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
-		if err != nil {
-			return nil, errors.Wrapf(err, "unable to find interface")
-		}
+	clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
+	if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
+		return nil, err
+	}
+
+	clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key")
+	if err := getHostFile(clientKubeletKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig")
+	if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
+		return nil, err
+	}
+
+	clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt")
+	if err := getHostFile(clientKubeProxyCert, info); err != nil {
+		return nil, err
+	}
+
+	clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key")
+	if err := getHostFile(clientKubeProxyKey, info); err != nil {
+		return nil, err
+	}
+
+	kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig")
+	if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
+		return nil, err
 	}
 
 	nodeConfig := &config.Node{
@@ -295,14 +346,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
 	nodeConfig.AgentConfig.NodeIP = nodeIP
 	nodeConfig.AgentConfig.NodeName = nodeName
-	nodeConfig.AgentConfig.NodeCertFile = nodeCertFile
-	nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile
+	nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
+	nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
 	nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
 	nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
-	nodeConfig.AgentConfig.CACertPath = clientCA
+	nodeConfig.AgentConfig.ClientCA = clientCAFile
 	nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
-	nodeConfig.AgentConfig.KubeConfig = kubeConfig
+	nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
+	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
+	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
 	nodeConfig.CACerts = info.CACerts
@@ -316,7 +369,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
 	nodeConfig.ServerAddress = serverURLParsed.Host
-	nodeConfig.Certificate = nodeCert
+	nodeConfig.Certificate = servingCert
 	if !nodeConfig.NoFlannel {
 		nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
 		nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)

pkg/agent/flannel/setup.go

@@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error {
 func Run(ctx context.Context, config *config.Node) error {
 	nodeName := config.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}
@@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig)
+		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 

pkg/agent/proxy/proxy.go

 package proxy
 
 import (
-	"crypto/tls"
-	"net/http"
-
-	"github.com/pkg/errors"
+	"github.com/google/tcpproxy"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/k3s/pkg/proxy"
 	"github.com/sirupsen/logrus"
 )
 
 func Run(config *config.Node) error {
-	proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
-	if err != nil {
-		return err
-	}
-
-	listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{
-		Certificates: []tls.Certificate{
-			*config.Certificate,
-		},
-	})
-
-	if err != nil {
-		return errors.Wrap(err, "Failed to start tls listener")
-	}
-
+	logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress)
+	var proxy tcpproxy.Proxy
+	proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress))
 	go func() {
-		err := http.Serve(listener, proxy)
+		err := proxy.Run()
 		logrus.Fatalf("TLS proxy stopped: %v", err)
 	}()
-
 	return nil
 }

pkg/agent/tunnel/tunnel.go

@@ -26,7 +26,7 @@ var (
 )
 
 func Setup(config *config.Node) error {
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}

pkg/cli/cmds/server.go

@@ -17,7 +17,7 @@ type Server struct {
 	DisableAgent        bool
 	KubeConfigOutput    string
 	KubeConfigMode      string
-	KnownIPs            cli.StringSlice
+	TLSSan              cli.StringSlice
 	BindAddress         string
 	ExtraAPIArgs        cli.StringSlice
 	ExtraSchedulerArgs  cli.StringSlice
@@ -28,6 +28,8 @@ type Server struct {
 	StorageCAFile       string
 	StorageCertFile     string
 	StorageKeyFile      string
+	AdvertiseIP         string
+	AdvertisePort       int
 }
 
 var ServerConfig Server
@@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 			cli.StringSliceFlag{
 				Name:  "tls-san",
 				Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
-				Value: &ServerConfig.KnownIPs,
+				Value: &ServerConfig.TLSSan,
 			},
 			cli.StringSliceFlag{
 				Name:  "kube-apiserver-arg",
@@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Destination: &ServerConfig.StorageKeyFile,
 				EnvVar:      "K3S_STORAGE_KEYFILE",
 			},
+			cli.StringFlag{
+				Name:        "advertise-address",
+				Usage:       "IP address that apiserver uses to advertise to members of the cluster",
+				Destination: &ServerConfig.AdvertiseIP,
+			},
+			cli.IntFlag{
+				Name:        "advertise-port",
+				Usage:       "Port that apiserver uses to advertise to members of the cluster",
+				Value:       0,
+				Destination: &ServerConfig.AdvertisePort,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,

pkg/cli/server/server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	"k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/pkg/volume/csi"
 
 	_ "github.com/go-sql-driver/mysql" // ensure we have mysql
@@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.Rootless = cfg.Rootless
 	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
-	serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)
+	for _, san := range knownIPs(cfg.TLSSan) {
+		addr := net2.ParseIP(san)
+		if addr != nil {
+			serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
+		} else {
+			serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
+		}
+	}
 	serverConfig.TLSConfig.BindAddress = cfg.BindAddress
+	serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
 	serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
@@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile
 	serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile
 	serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile
+	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
+	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
+
+	if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
+		serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
+	}
+	if serverConfig.ControlConfig.AdvertiseIP != "" {
+		serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
+	}
 
 	_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
 	if err != nil {
@@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 		return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err)
 	}
 
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+	serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
+
 	// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
 	// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
 	if cfg.ClusterDNS == "" {

pkg/daemons/agent/agent.go

@@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) {
 	argsMap := map[string]string{
 		"proxy-mode":           "iptables",
 		"healthz-bind-address": "127.0.0.1",
-		"kubeconfig":           cfg.KubeConfig,
+		"kubeconfig":           cfg.KubeConfigKubeProxy,
 		"cluster-cidr":         cfg.ClusterCIDR.String(),
 	}
 	args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
@@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) {
 		"read-only-port":           "0",
 		"allow-privileged":         "true",
 		"cluster-domain":           cfg.ClusterDomain,
-		"kubeconfig":               cfg.KubeConfig,
+		"kubeconfig":               cfg.KubeConfigKubelet,
 		"eviction-hard":            "imagefs.available<5%,nodefs.available<5%",
 		"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
 		"fail-swap-on":             "false",
@@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) {
 	if cfg.ListenAddress != "" {
 		argsMap["address"] = cfg.ListenAddress
 	}
-	if cfg.CACertPath != "" {
+	if cfg.ClientCA != "" {
 		argsMap["anonymous-auth"] = "false"
-		argsMap["client-ca-file"] = cfg.CACertPath
+		argsMap["client-ca-file"] = cfg.ClientCA
 	}
-	if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" {
-		argsMap["tls-cert-file"] = cfg.NodeCertFile
-		argsMap["tls-private-key-file"] = cfg.NodeKeyFile
+	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
+		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
+		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
 	}
 	if cfg.NodeName != "" {
 		argsMap["hostname-override"] = cfg.NodeName

pkg/daemons/config/types.go

@@ -36,32 +36,41 @@ type Containerd struct {
 }
 
 type Agent struct {
-	NodeName           string
-	NodeCertFile       string
-	NodeKeyFile        string
-	ClusterCIDR        net.IPNet
-	ClusterDNS         net.IP
-	ClusterDomain      string
-	ResolvConf         string
-	RootDir            string
-	KubeConfig         string
-	NodeIP             string
-	RuntimeSocket      string
-	ListenAddress      string
-	CACertPath         string
-	CNIBinDir          string
-	CNIConfDir         string
-	ExtraKubeletArgs   []string
-	ExtraKubeProxyArgs []string
-	PauseImage         string
-	CNIPlugin          bool
-	NodeTaints         []string
-	NodeLabels         []string
+	NodeName            string
+	ClientKubeletCert   string
+	ClientKubeletKey    string
+	ClientKubeProxyCert string
+	ClientKubeProxyKey  string
+	ServingKubeletCert  string
+	ServingKubeletKey   string
+	ClusterCIDR         net.IPNet
+	ClusterDNS          net.IP
+	ClusterDomain       string
+	ResolvConf          string
+	RootDir             string
+	KubeConfigNode      string
+	KubeConfigKubelet   string
+	KubeConfigKubeProxy string
+	NodeIP              string
+	RuntimeSocket       string
+	ListenAddress       string
+	ClientCA            string
+	CNIBinDir           string
+	CNIConfDir          string
+	ExtraKubeletArgs    []string
+	ExtraKubeProxyArgs  []string
+	PauseImage          string
+	CNIPlugin           bool
+	NodeTaints          []string
+	NodeLabels          []string
 }
 
 type Control struct {
 	AdvertisePort         int
+	AdvertiseIP           string
 	ListenPort            int
+	HTTPSPort             int
+	ProxyPort             int
 	ClusterSecret         string
 	ClusterIPRange        *net.IPNet
 	ServiceIPRange        *net.IPNet
@@ -87,28 +96,44 @@ type Control struct {
 }
 
 type ControlRuntime struct {
-	TLSCert          string
-	TLSKey           string
-	TLSCA            string
-	TLSCAKey         string
-	TokenCA          string
-	TokenCAKey       string
-	ServiceKey       string
-	PasswdFile       string
-	KubeConfigSystem string
-
-	NodeCert      string
-	NodeKey       string
-	ClientToken   string
-	NodeToken     string
-	Handler       http.Handler
-	Tunnel        http.Handler
-	Authenticator authenticator.Request
+	ClientKubeAPICert string
+	ClientKubeAPIKey  string
+	ClientCA          string
+	ClientCAKey       string
+	ServerCA          string
+	ServerCAKey       string
+	ServiceKey        string
+	PasswdFile        string
+
+	KubeConfigAdmin      string
+	KubeConfigController string
+	KubeConfigScheduler  string
+	KubeConfigAPIServer  string
+
+	ServingKubeAPICert string
+	ServingKubeAPIKey  string
+	ClientToken        string
+	NodeToken          string
+	Handler            http.Handler
+	Tunnel             http.Handler
+	Authenticator      authenticator.Request
 
 	RequestHeaderCA     string
 	RequestHeaderCAKey  string
 	ClientAuthProxyCert string
 	ClientAuthProxyKey  string
+
+	ClientAdminCert      string
+	ClientAdminKey       string
+	ClientControllerCert string
+	ClientControllerKey  string
+	ClientSchedulerCert  string
+	ClientSchedulerKey   string
+	ClientKubeProxyCert  string
+	ClientKubeProxyKey   string
+
+	ServingKubeletKey string
+	ClientKubeletKey  string
 }
 
 type ArgString []string

pkg/server/router.go

 package server
 
 import (
-	"bufio"
-	"crypto/rsa"
+	"crypto"
 	"crypto/x509"
+	"encoding/csv"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -17,10 +18,10 @@ import (
 	"github.com/gorilla/mux"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control"
 	"github.com/rancher/k3s/pkg/openapi"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
-	"k8s.io/kubernetes/pkg/master"
 )
 
 const (
@@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC
 	authed.Use(authMiddleware(serverConfig))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
 	authed.Path("/v1-k3s/connect").Handler(tunnel)
-	authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig))
-	authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig))
+	authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
+	authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
+	authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
+	authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
+	authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
+	authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
+	authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
+	authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
 	authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
@@ -65,82 +72,122 @@ func cacerts(getter CACertsGetter) http.Handler {
 	})
 }
 
-func nodeCrt(server *config.Control) http.Handler {
+func getNodeInfo(req *http.Request) (string, string, error) {
+	nodeNames := req.Header["K3s-Node-Name"]
+	if len(nodeNames) != 1 || nodeNames[0] == "" {
+		return "", "", errors.New("node name not set")
+	}
+
+	nodePasswords := req.Header["K3s-Node-Password"]
+	if len(nodePasswords) != 1 || nodePasswords[0] == "" {
+		return "", "", errors.New("node password not set")
+	}
+
+	return nodeNames[0], nodePasswords[0], nil
+}
+
+func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) {
+	keyBytes, err := ioutil.ReadFile(signingKeyFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	key, err := certutil.ParsePrivateKeyPEM(keyBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(caKeyFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caBytes, err := ioutil.ReadFile(caCertFile)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
+}
+
+func servingKubeletCert(server *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		nodeNames := req.Header["K3s-Node-Name"]
-		if len(nodeNames) != 1 || nodeNames[0] == "" {
-			sendError(errors.New("node name not set"), resp)
-			return
-		}
-
-		nodePasswords := req.Header["K3s-Node-Password"]
-		if len(nodePasswords) != 1 || nodePasswords[0] == "" {
-			sendError(errors.New("node password not set"), resp)
-			return
+		nodeName, nodePassword, err := getNodeInfo(req)
+		if err != nil {
+			sendError(err, resp)
 		}
 
-		if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil {
+		if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
 			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
-		nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey)
+		caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		key, err := certutil.ParsePrivateKeyPEM(nodeKey)
+		cert, err := certutil.NewSignedCert(certutil.Config{
+			CommonName: nodeName,
+			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+			AltNames: certutil.AltNames{
+				DNSNames: []string{nodeName, "localhost"},
+				IPs:      []net.IP{net.ParseIP("127.0.0.1")},
+			},
+		}, key, caCert[0], caKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey)
-		if err != nil {
-			sendError(err, resp)
-			return
-		}
+		resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+	})
+}
 
-		caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA)
-		if err != nil {
-			sendError(err, resp)
+func clientKubeletCert(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
 
-		caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+		nodeName, nodePassword, err := getNodeInfo(req)
 		if err != nil {
 			sendError(err, resp)
-			return
 		}
 
-		caCert, err := certutil.ParseCertsPEM(caBytes)
-		if err != nil {
-			sendError(err, resp)
+		if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
+			sendError(err, resp, http.StatusForbidden)
 			return
 		}
 
-		_, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange)
+		caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
 		if err != nil {
 			sendError(err, resp)
 			return
 		}
 
-		cfg := certutil.Config{
-			CommonName: "kubernetes",
-			Usages:     []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
-			AltNames: certutil.AltNames{
-				DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]},
-				IPs:      []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")},
-			},
-		}
-
-		cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey))
+		cert, err := certutil.NewSignedCert(certutil.Config{
+			CommonName:   "system:node:" + nodeName,
+			Organization: []string{"system:nodes"},
+			Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		}, key, caCert[0], caKey)
 		if err != nil {
 			sendError(err, resp)
 			return
@@ -150,13 +197,13 @@ func nodeCrt(server *config.Control) http.Handler {
 	})
 }
 
-func nodeKey(server *config.Control) http.Handler {
+func fileHandler(fileName string) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
 			return
 		}
-		http.ServeFile(resp, req, server.Runtime.NodeKey)
+		http.ServeFile(resp, req, fileName)
 	})
 }
 
@@ -225,29 +272,29 @@ func ensureNodePassword(passwdFile, nodeName, passwd string) error {
 	defer f.Close()
 	user := strings.ToLower("node:" + nodeName)
 
-	buf := &strings.Builder{}
-	scan := bufio.NewScanner(f)
-	for scan.Scan() {
-		line := scan.Text()
-		parts := strings.Split(line, ",")
-		if len(parts) < 4 {
-			continue
+	records := [][]string{}
+	reader := csv.NewReader(f)
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
 		}
-		if parts[1] == user {
-			if parts[0] == passwd {
+		if len(record) < 3 {
+			return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record))
+		}
+		if record[1] == user {
+			if record[0] == passwd {
 				return nil
 			}
-			return fmt.Errorf("Node password validation failed for [%s]", nodeName)
+			return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
 		}
-		buf.WriteString(line)
-		buf.WriteString("\n")
-	}
-	buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user))
-
-	if scan.Err() != nil {
-		return scan.Err()
+		records = append(records, record)
 	}
+	records = append(records, []string{passwd, user, user, "system:node:" + nodeName})
 
 	f.Close()
-	return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600)
+	return control.WritePasswords(passwdFile, records)
 }

pkg/server/server.go

@@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		controlConfig = &config.ControlConfig
 	)
 
+	caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
+	if err != nil {
+		return "", err
+	}
+	caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
+	if err != nil {
+		return "", err
+	}
+
+	tlsConfig.CACerts = string(caBytes)
+	tlsConfig.CAKey = string(caKeyBytes)
+
 	tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
 		if tlsServer == nil {
 			return "", nil
@@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
 		return tlsServer.CACert()
 	})
 
-	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem)
+	sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
 	if err != nil {
 		return "", err
 	}