Merge pull request #2407 from erikwilson/node-passwd-cleanup

pkg/agent/loadbalancer/loadbalancer_test.go

@@ -152,7 +152,7 @@ func TestFailFast(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 
 	cfg := cmds.Agent{
-		ServerURL: "http://127.0.0.1:-1/",
+		ServerURL: "http://127.0.0.1:0/",
 		DataDir:   tmpDir,
 	}
 

pkg/authenticator/hash/hash.go

+package hash
+
+// Hasher is a generic interface for hashing algorithms
+type Hasher interface {
+	// CreateHash will return a hashed version of the secretKey, or an error
+	CreateHash(secretKey string) (string, error)
+	// VerifyHash will compare a secretKey and a hash, and return nil if they match
+	VerifyHash(hash, secretKey string) error
+}

pkg/authenticator/hash/scrypt.go

+package hash
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"encoding/base64"
+	"errors"
+	"fmt"
+
+	"golang.org/x/crypto/scrypt"
+)
+
+// Version is the hashing format version
+const Version = 1
+const hashFormat = "$%d:%x:%d:%d:%d:%s"
+
+// SCrypt contains all of the variables needed for scrypt hashing
+type SCrypt struct {
+	N       int
+	R       int
+	P       int
+	KeyLen  int
+	SaltLen int
+}
+
+// NewSCrypt returns a scrypt hasher with recommended default values
+func NewSCrypt() Hasher {
+	return SCrypt{
+		N:       15,
+		R:       8,
+		P:       1,
+		KeyLen:  64,
+		SaltLen: 8,
+	}
+}
+
+// CreateHash will return a hashed version of the secretKey, or an error
+func (s SCrypt) CreateHash(secretKey string) (string, error) {
+	salt := make([]byte, s.SaltLen)
+
+	_, err := rand.Read(salt)
+	if err != nil {
+		return "", err
+	}
+
+	dk, err := scrypt.Key([]byte(secretKey), salt, 1<<s.N, s.R, s.P, s.KeyLen)
+	if err != nil {
+		return "", err
+	}
+
+	enc := base64.RawStdEncoding.EncodeToString(dk)
+	hash := fmt.Sprintf(hashFormat, Version, salt, s.N, s.R, s.P, enc)
+
+	return hash, nil
+}
+
+// VerifyHash will compare a secretKey and a hash, and return nil if they match
+func (s SCrypt) VerifyHash(hash, secretKey string) error {
+	var (
+		version, n uint
+		r, p       int
+		enc        string
+		salt       []byte
+	)
+	_, err := fmt.Sscanf(hash, hashFormat, &version, &salt, &n, &r, &p, &enc)
+	if err != nil {
+		return err
+	}
+	if version != Version {
+		return fmt.Errorf("hash version %d does not match package version %d", version, Version)
+	}
+
+	dk, err := base64.RawStdEncoding.DecodeString(enc)
+	if err != nil {
+		return err
+	}
+
+	verify, err := scrypt.Key([]byte(secretKey), salt, 1<<n, r, p, len(dk))
+	if err != nil {
+		return err
+	}
+
+	if subtle.ConstantTimeCompare(dk, verify) != 1 {
+		return errors.New("hash does not match")
+	}
+
+	return nil
+}

pkg/authenticator/hash/scrypt_test.go

+package hash
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var hasher = NewSCrypt()
+
+func TestBasicHash(t *testing.T) {
+	secretKey := "hello world"
+	hash, err := hasher.CreateHash(secretKey)
+	assert.Nil(t, err)
+	assert.NotNil(t, hash)
+
+	assert.Nil(t, hasher.VerifyHash(hash, secretKey))
+	assert.NotNil(t, hasher.VerifyHash(hash, "goodbye"))
+}
+
+func TestLongKey(t *testing.T) {
+	secretKey := strings.Repeat("A", 720)
+	hash, err := hasher.CreateHash(secretKey)
+	assert.Nil(t, err)
+	assert.NotNil(t, hash)
+
+	assert.Nil(t, hasher.VerifyHash(hash, secretKey))
+	assert.NotNil(t, hasher.VerifyHash(hash, secretKey+":wrong!"))
+}

pkg/node/controller.go

@@ -5,13 +5,21 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/nodepassword"
 	coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	core "k8s.io/api/core/v1"
 )
 
-func Register(ctx context.Context, configMap coreclient.ConfigMapController, nodes coreclient.NodeController) error {
+func Register(ctx context.Context,
+	modCoreDNS bool,
+	secretClient coreclient.SecretClient,
+	configMap coreclient.ConfigMapController,
+	nodes coreclient.NodeController,
+) error {
 	h := &handler{
+		modCoreDNS:   modCoreDNS,
+		secretClient: secretClient,
 		configCache:  configMap.Cache(),
 		configClient: configMap,
 	}
@@ -22,6 +30,8 @@ func Register(ctx context.Context, configMap coreclient.ConfigMapController, nod
 }
 
 type handler struct {
+	modCoreDNS   bool
+	secretClient coreclient.SecretClient
 	configCache  coreclient.ConfigMapCache
 	configClient coreclient.ConfigMapClient
 }
@@ -39,31 +49,45 @@ func (h *handler) onRemove(key string, node *core.Node) (*core.Node, error) {
 
 func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error) {
 	var (
-		newHosts    string
+		nodeName    string
 		nodeAddress string
-		hostsMap    map[string]string
 	)
-	hostsMap = make(map[string]string)
-
+	nodeName = node.Name
 	for _, address := range node.Status.Addresses {
 		if address.Type == "InternalIP" {
 			nodeAddress = address.Address
 			break
 		}
 	}
-	if nodeAddress == "" {
-		logrus.Errorf("No InternalIP found for node " + node.Name)
-		return nil, nil
+	if removed {
+		if err := h.removeNodePassword(nodeName); err != nil {
+			logrus.Warn(errors.Wrap(err, "Unable to remove node password"))
+		}
+	}
+	if h.modCoreDNS {
+		if err := h.updateCoreDNSConfigMap(nodeName, nodeAddress, removed); err != nil {
+			return nil, err
+		}
+	}
+	return nil, nil
+}
+
+func (h *handler) updateCoreDNSConfigMap(nodeName, nodeAddress string, removed bool) error {
+	if nodeAddress == "" && !removed {
+		logrus.Errorf("No InternalIP found for node " + nodeName)
+		return nil
 	}
 
 	configMapCache, err := h.configCache.Get("kube-system", "coredns")
 	if err != nil || configMapCache == nil {
 		logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map"))
-		return nil, nil
+		return nil
 	}
-	configMap := configMapCache.DeepCopy()
 
+	configMap := configMapCache.DeepCopy()
 	hosts := configMap.Data["NodeHosts"]
+	hostsMap := map[string]string{}
+
 	for _, line := range strings.Split(hosts, "\n") {
 		if line == "" {
 			continue
@@ -75,27 +99,29 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error)
 		}
 		ip := fields[0]
 		host := fields[1]
-		if host == node.Name {
+		if host == nodeName {
 			if removed {
 				continue
 			}
 			if ip == nodeAddress {
-				return nil, nil
+				return nil
 			}
 		}
 		hostsMap[host] = ip
 	}
 
 	if !removed {
-		hostsMap[node.Name] = nodeAddress
+		hostsMap[nodeName] = nodeAddress
 	}
+
+	var newHosts string
 	for host, ip := range hostsMap {
 		newHosts += ip + " " + host + "\n"
 	}
 	configMap.Data["NodeHosts"] = newHosts
 
 	if _, err := h.configClient.Update(configMap); err != nil {
-		return nil, err
+		return err
 	}
 
 	var actionType string
@@ -104,7 +130,10 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error)
 	} else {
 		actionType = "Updated"
 	}
-	logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeAddress+" "+node.Name)
+	logrus.Infof("%s coredns node hosts entry [%s]", actionType, nodeAddress+" "+nodeName)
+	return nil
+}
 
-	return nil, nil
+func (h *handler) removeNodePassword(nodeName string) error {
+	return nodepassword.Delete(h.secretClient, nodeName)
 }

pkg/nodepassword/nodepassword.go

+package nodepassword
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/authenticator/hash"
+	"github.com/rancher/k3s/pkg/passwd"
+	"github.com/rancher/k3s/pkg/version"
+	coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	// hasher provides the algorithm for generating and verifying hashes
+	hasher = hash.NewSCrypt()
+)
+
+func getSecretName(nodeName string) string {
+	return strings.ToLower(nodeName + ".node-password." + version.Program)
+}
+
+func verifyHash(secretClient coreclient.SecretClient, nodeName, pass string) error {
+	name := getSecretName(nodeName)
+	secret, err := secretClient.Get(metav1.NamespaceSystem, name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if hash, ok := secret.Data["hash"]; ok {
+		if err := hasher.VerifyHash(string(hash), pass); err != nil {
+			return errors.Wrapf(err, "unable to verify hash for node '%s'", nodeName)
+		}
+		return nil
+	}
+	return fmt.Errorf("unable to locate hash data for node secret '%s'", name)
+}
+
+// Ensure will verify a node-password secret if it exists, otherwise it will create one
+func Ensure(secretClient coreclient.SecretClient, nodeName, pass string) error {
+	if err := verifyHash(secretClient, nodeName, pass); !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	hash, err := hasher.CreateHash(pass)
+	if err != nil {
+		return errors.Wrapf(err, "unable to create hash for node '%s'", nodeName)
+	}
+
+	immutable := true
+	_, err = secretClient.Create(&v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      getSecretName(nodeName),
+			Namespace: metav1.NamespaceSystem,
+		},
+		Immutable: &immutable,
+		Data:      map[string][]byte{"hash": []byte(hash)},
+	})
+	if apierrors.IsAlreadyExists(err) {
+		return verifyHash(secretClient, nodeName, pass)
+	}
+	return err
+}
+
+// Delete will remove a node-password secret
+func Delete(secretClient coreclient.SecretClient, nodeName string) error {
+	return secretClient.Delete(metav1.NamespaceSystem, getSecretName(nodeName), &metav1.DeleteOptions{})
+}
+
+// MigrateFile moves password file entries to secrets
+func MigrateFile(secretClient coreclient.SecretClient, nodeClient coreclient.NodeClient, passwordFile string) error {
+	_, err := os.Stat(passwordFile)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	passwd, err := passwd.Read(passwordFile)
+	if err != nil {
+		return err
+	}
+
+	nodeNames := []string{}
+	nodeList, _ := nodeClient.List(metav1.ListOptions{})
+	if nodeList != nil {
+		for _, node := range nodeList.Items {
+			nodeNames = append(nodeNames, node.Name)
+		}
+	}
+	if len(nodeNames) == 0 {
+		nodeNames = append(nodeNames, passwd.Users()...)
+	}
+
+	logrus.Infof("Migrating node password entries from '%s'", passwordFile)
+	ensured := int64(0)
+	start := time.Now()
+	for _, nodeName := range nodeNames {
+		if pass, ok := passwd.Pass(nodeName); ok {
+			if err := Ensure(secretClient, nodeName, pass); err != nil {
+				logrus.Warn(errors.Wrapf(err, "error migrating node password entry for node '%s'", nodeName))
+			} else {
+				ensured++
+			}
+		}
+	}
+	ms := time.Since(start).Milliseconds()
+	logrus.Infof("Migrated %d node password entries in %d milliseconds, average %d ms", ensured, ms, ms/ensured)
+	return os.Remove(passwordFile)
+}

pkg/nodepassword/nodepassword_test.go

+package nodepassword
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"runtime"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+)
+
+const migrateNumNodes = 10
+const createNumNodes = 3
+
+func TestAsserts(t *testing.T) {
+	assertEqual(t, 1, 1)
+	assertNotEqual(t, 1, 0)
+}
+
+func TestEnsureDelete(t *testing.T) {
+	logMemUsage(t)
+
+	secretClient := &mockSecretClient{}
+	assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
+	assertEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
+	assertNotEqual(t, Ensure(secretClient, "node1", "Goodbye World"), nil)
+	assertEqual(t, secretClient.created, 1)
+
+	assertEqual(t, Delete(secretClient, "node1"), nil)
+	assertNotEqual(t, Delete(secretClient, "node1"), nil)
+	assertEqual(t, secretClient.deleted, 1)
+
+	assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil)
+	assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
+	assertEqual(t, Ensure(secretClient, "node1", "Hello Universe"), nil)
+	assertEqual(t, secretClient.created, 2)
+
+	logMemUsage(t)
+}
+
+func TestMigrateFile(t *testing.T) {
+	nodePasswordFile := generateNodePasswordFile(migrateNumNodes)
+	defer os.Remove(nodePasswordFile)
+
+	secretClient := &mockSecretClient{}
+	nodeClient := &mockNodeClient{}
+
+	logMemUsage(t)
+	if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil {
+		log.Fatal(err)
+	}
+	logMemUsage(t)
+
+	assertEqual(t, secretClient.created, migrateNumNodes)
+	assertNotEqual(t, Ensure(secretClient, "node1", "Hello World"), nil)
+	assertEqual(t, Ensure(secretClient, "node1", "node1"), nil)
+}
+
+func TestMigrateFileNodes(t *testing.T) {
+	nodePasswordFile := generateNodePasswordFile(migrateNumNodes)
+	defer os.Remove(nodePasswordFile)
+
+	secretClient := &mockSecretClient{}
+	nodeClient := &mockNodeClient{}
+	nodeClient.nodes = make([]v1.Node, createNumNodes, createNumNodes)
+	for i := range nodeClient.nodes {
+		nodeClient.nodes[i].Name = fmt.Sprintf("node%d", i+1)
+	}
+
+	logMemUsage(t)
+	if err := MigrateFile(secretClient, nodeClient, nodePasswordFile); err != nil {
+		log.Fatal(err)
+	}
+	logMemUsage(t)
+
+	assertEqual(t, secretClient.created, createNumNodes)
+	for _, node := range nodeClient.nodes {
+		assertNotEqual(t, Ensure(secretClient, node.Name, "wrong-password"), nil)
+		assertEqual(t, Ensure(secretClient, node.Name, node.Name), nil)
+	}
+	newNode := fmt.Sprintf("node%d", createNumNodes+1)
+	assertEqual(t, Ensure(secretClient, newNode, "new-password"), nil)
+	assertNotEqual(t, Ensure(secretClient, newNode, "wrong-password"), nil)
+}
+
+// --------------------------
+
+// mock secret client interface
+
+type mockSecretClient struct {
+	entries map[string]map[string]v1.Secret
+	created int
+	deleted int
+}
+
+func (m *mockSecretClient) Create(secret *v1.Secret) (*v1.Secret, error) {
+	if m.entries == nil {
+		m.entries = map[string]map[string]v1.Secret{}
+	}
+	if _, ok := m.entries[secret.Namespace]; !ok {
+		m.entries[secret.Namespace] = map[string]v1.Secret{}
+	}
+	if _, ok := m.entries[secret.Namespace][secret.Name]; ok {
+		return nil, errorAlreadyExists()
+	}
+	m.created++
+	m.entries[secret.Namespace][secret.Name] = *secret
+	return secret, nil
+}
+
+func (m *mockSecretClient) Update(secret *v1.Secret) (*v1.Secret, error) {
+	return nil, errorNotImplemented()
+}
+
+func (m *mockSecretClient) Delete(namespace, name string, options *metav1.DeleteOptions) error {
+	if m.entries == nil {
+		return errorNotFound()
+	}
+	if _, ok := m.entries[namespace]; !ok {
+		return errorNotFound()
+	}
+	if _, ok := m.entries[namespace][name]; !ok {
+		return errorNotFound()
+	}
+	m.deleted++
+	delete(m.entries[namespace], name)
+	return nil
+}
+
+func (m *mockSecretClient) Get(namespace, name string, options metav1.GetOptions) (*v1.Secret, error) {
+	if m.entries == nil {
+		return nil, errorNotFound()
+	}
+	if _, ok := m.entries[namespace]; !ok {
+		return nil, errorNotFound()
+	}
+	if secret, ok := m.entries[namespace][name]; ok {
+		return &secret, nil
+	}
+	return nil, errorNotFound()
+}
+
+func (m *mockSecretClient) List(namespace string, opts metav1.ListOptions) (*v1.SecretList, error) {
+	return nil, errorNotImplemented()
+}
+
+func (m *mockSecretClient) Watch(namespace string, opts metav1.ListOptions) (watch.Interface, error) {
+	return nil, errorNotImplemented()
+}
+
+func (m *mockSecretClient) Patch(namespace, name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Secret, err error) {
+	return nil, errorNotImplemented()
+}
+
+// --------------------------
+
+// mock node client interface
+
+type mockNodeClient struct {
+	nodes []v1.Node
+}
+
+func (m *mockNodeClient) Create(node *v1.Node) (*v1.Node, error) {
+	return nil, errorNotImplemented()
+}
+func (m *mockNodeClient) Update(node *v1.Node) (*v1.Node, error) {
+	return nil, errorNotImplemented()
+}
+func (m *mockNodeClient) UpdateStatus(node *v1.Node) (*v1.Node, error) {
+	return nil, errorNotImplemented()
+}
+func (m *mockNodeClient) Delete(name string, options *metav1.DeleteOptions) error {
+	return errorNotImplemented()
+}
+func (m *mockNodeClient) Get(name string, options metav1.GetOptions) (*v1.Node, error) {
+	return nil, errorNotImplemented()
+}
+func (m *mockNodeClient) List(opts metav1.ListOptions) (*v1.NodeList, error) {
+	return &v1.NodeList{Items: m.nodes}, nil
+}
+func (m *mockNodeClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return nil, errorNotImplemented()
+}
+func (m *mockNodeClient) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.Node, err error) {
+	return nil, errorNotImplemented()
+}
+
+// --------------------------
+
+// utility functions
+
+func assertEqual(t *testing.T, a interface{}, b interface{}) {
+	if a != b {
+		t.Fatalf("[ %v != %v ]", a, b)
+	}
+}
+
+func assertNotEqual(t *testing.T, a interface{}, b interface{}) {
+	if a == b {
+		t.Fatalf("[ %v == %v ]", a, b)
+	}
+}
+
+func generateNodePasswordFile(migrateNumNodes int) string {
+	tempFile, err := ioutil.TempFile("", "node-password-test.*")
+	if err != nil {
+		log.Fatal(err)
+	}
+	tempFile.Close()
+
+	var passwordEntries string
+	for i := 1; i <= migrateNumNodes; i++ {
+		passwordEntries += fmt.Sprintf("node%d,node%d\n", i, i)
+	}
+	if err := ioutil.WriteFile(tempFile.Name(), []byte(passwordEntries), 0600); err != nil {
+		log.Fatal(err)
+	}
+
+	return tempFile.Name()
+}
+
+func errorNotFound() error {
+	return apierrors.NewNotFound(schema.GroupResource{}, "not-found")
+}
+
+func errorAlreadyExists() error {
+	return apierrors.NewAlreadyExists(schema.GroupResource{}, "already-exists")
+}
+
+func errorNotImplemented() error {
+	log.Fatal("not implemented")
+	return apierrors.NewMethodNotSupported(schema.GroupResource{}, "not-implemented")
+}
+
+func logMemUsage(t *testing.T) {
+	var stats runtime.MemStats
+	runtime.ReadMemStats(&stats)
+	t.Logf("Memory Usage:  Alloc=%d MB,  Sys=%d MB,  NumGC=%d",
+		toMB(stats.Alloc), toMB(stats.Sys), stats.NumGC)
+}
+
+func toMB(bytes uint64) uint64 {
+	return bytes / (1024 * 1024)
+}

pkg/passwd/passwd.go

@@ -107,6 +107,14 @@ func (p *Passwd) EnsureUser(name, role, passwd string) error {
 	return nil
 }
 
+func (p *Passwd) Users() []string {
+	users := []string{}
+	for user := range p.names {
+		users = append(users, user)
+	}
+	return users
+}
+
 func (p *Passwd) Pass(name string) (string, bool) {
 	e, ok := p.names[name]
 	if !ok {

pkg/server/router.go

@@ -4,7 +4,6 @@ import (
 	"crypto"
 	"crypto/x509"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -16,8 +15,9 @@ import (
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/k3s/pkg/bootstrap"
 	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/k3s/pkg/passwd"
+	"github.com/rancher/k3s/pkg/nodepassword"
 	"github.com/rancher/k3s/pkg/version"
+	coreclient "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
 )
@@ -26,13 +26,18 @@ const (
 	staticURL = "/static/"
 )
 
-func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.Handler {
+func router(serverConfig *config.Control, tunnel http.Handler, secretClient coreclient.SecretClient) (http.Handler, error) {
+	ca, err := ioutil.ReadFile(serverConfig.Runtime.ServerCA)
+	if err != nil {
+		return nil, err
+	}
+
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter()
 	authed.Use(authMiddleware(serverConfig, version.Program+":agent"))
 	authed.NotFoundHandler = serverConfig.Runtime.Handler
-	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey))
-	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey))
+	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, secretClient))
+	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, secretClient))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
 	authed.Path(prefix + "/client-" + version.Program + "-controller.crt").Handler(fileHandler(serverConfig.Runtime.ClientK3sControllerCert, serverConfig.Runtime.ClientK3sControllerKey))
 	authed.Path(prefix + "/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
@@ -59,7 +64,7 @@ func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.H
 	router.Path("/cacerts").Handler(cacerts(ca))
 	router.Path("/ping").Handler(ping())
 
-	return router
+	return router, nil
 }
 
 func cacerts(ca []byte) http.Handler {
@@ -117,7 +122,7 @@ func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Cer
 	return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
 }
 
-func servingKubeletCert(server *config.Control, keyFile string) http.Handler {
+func servingKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
@@ -130,7 +135,7 @@ func servingKubeletCert(server *config.Control, keyFile string) http.Handler {
 			return
 		}
 
-		if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil {
+		if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
 			sendError(err, resp, http.StatusForbidden)
 			return
 		}
@@ -170,7 +175,7 @@ func servingKubeletCert(server *config.Control, keyFile string) http.Handler {
 	})
 }
 
-func clientKubeletCert(server *config.Control, keyFile string) http.Handler {
+func clientKubeletCert(server *config.Control, keyFile string, secretClient coreclient.SecretClient) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.TLS == nil {
 			resp.WriteHeader(http.StatusNotFound)
@@ -183,7 +188,7 @@ func clientKubeletCert(server *config.Control, keyFile string) http.Handler {
 			return
 		}
 
-		if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil {
+		if err := nodepassword.Ensure(secretClient, nodeName, nodePassword); err != nil {
 			sendError(err, resp, http.StatusForbidden)
 			return
 		}
@@ -274,20 +279,3 @@ func sendError(err error, resp http.ResponseWriter, status ...int) {
 	resp.WriteHeader(code)
 	resp.Write([]byte(err.Error()))
 }
-
-func ensureNodePassword(passwdFile, nodeName, pass string) error {
-	passwd, err := passwd.Read(passwdFile)
-	if err != nil {
-		return err
-	}
-	match, exists := passwd.Check(nodeName, pass)
-	if exists {
-		if !match {
-			return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
-		}
-		return nil
-	}
-	// If user doesn't exist we save this password for future validation
-	passwd.EnsureUser(nodeName, "", pass)
-	return passwd.Write(passwdFile)
-}

pkg/server/server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rancher/k3s/pkg/datadir"
 	"github.com/rancher/k3s/pkg/deploy"
 	"github.com/rancher/k3s/pkg/node"
+	"github.com/rancher/k3s/pkg/nodepassword"
 	"github.com/rancher/k3s/pkg/rootlessports"
 	"github.com/rancher/k3s/pkg/servicelb"
 	"github.com/rancher/k3s/pkg/static"
@@ -56,9 +57,7 @@ func StartServer(ctx context.Context, config *Config) error {
 		return errors.Wrap(err, "starting kubernetes")
 	}
 
-	if err := startWrangler(ctx, config); err != nil {
-		return errors.Wrap(err, "starting tls server")
-	}
+	go startOnAPIServerReady(ctx, config)
 
 	for _, hook := range config.StartupHooks {
 		if err := hook(ctx, config.ControlConfig.Runtime.APIServerReady, config.ControlConfig.Runtime.KubeConfigAdmin); err != nil {
@@ -83,32 +82,15 @@ func StartServer(ctx context.Context, config *Config) error {
 	return writeKubeConfig(config.ControlConfig.Runtime.ServerCA, config)
 }
 
-func startWrangler(ctx context.Context, config *Config) error {
-	var (
-		err           error
-		controlConfig = &config.ControlConfig
-	)
-
-	ca, err := ioutil.ReadFile(config.ControlConfig.Runtime.ServerCA)
-	if err != nil {
-		return err
-	}
-
-	controlConfig.Runtime.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, ca)
-
-	// Start in background
-	go func() {
-		select {
-		case <-ctx.Done():
-			return
-		case <-config.ControlConfig.Runtime.APIServerReady:
-			if err := runControllers(ctx, config); err != nil {
-				logrus.Fatalf("failed to start controllers: %v", err)
-			}
+func startOnAPIServerReady(ctx context.Context, config *Config) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-config.ControlConfig.Runtime.APIServerReady:
+		if err := runControllers(ctx, config); err != nil {
+			logrus.Fatalf("failed to start controllers: %v", err)
 		}
-	}()
-
-	return nil
+	}
 }
 
 func runControllers(ctx context.Context, config *Config) error {
@@ -141,6 +123,11 @@ func runControllers(ctx context.Context, config *Config) error {
 		if err := sc.Start(ctx); err != nil {
 			panic(err)
 		}
+		handler, err := router(controlConfig, controlConfig.Runtime.Tunnel, sc.Core.Core().V1().Secret())
+		if err != nil {
+			panic(errors.Wrap(err, "starting router"))
+		}
+		controlConfig.Runtime.Handler = handler
 	}
 	if !config.DisableAgent {
 		go setMasterRoleLabel(ctx, sc.Core.Core().V1().Node())
@@ -162,10 +149,19 @@ func runControllers(ctx context.Context, config *Config) error {
 }
 
 func masterControllers(ctx context.Context, sc *Context, config *Config) error {
-	if !config.ControlConfig.Skips["coredns"] {
-		if err := node.Register(ctx, sc.Core.Core().V1().ConfigMap(), sc.Core.Core().V1().Node()); err != nil {
-			return err
-		}
+	if err := nodepassword.MigrateFile(
+		sc.Core.Core().V1().Secret(),
+		sc.Core.Core().V1().Node(),
+		config.ControlConfig.Runtime.NodePasswdFile); err != nil {
+		logrus.Warn(errors.Wrapf(err, "error migrating node-password file"))
+	}
+
+	if err := node.Register(ctx,
+		!config.ControlConfig.Skips["coredns"],
+		sc.Core.Core().V1().Secret(),
+		sc.Core.Core().V1().ConfigMap(),
+		sc.Core.Core().V1().Node()); err != nil {
+		return err
 	}
 
 	helm.Register(ctx, sc.Apply,

scripts/generate

+#!/bin/bash
+
+cd $(dirname $0)/..
+
+rm -rf build/data
+mkdir -p build/data
+
+GO=${GO-go}
+
+echo Running: "${GO}" generate
+"${GO}" generate

scripts/validate

@@ -31,3 +31,5 @@ if [ -n "$DIRTY" ]; then
     git status --porcelain --untracked-files=no
     exit 1
 fi
+
+"${GO}" test -v ./...

vendor/golang.org/x/crypto/scrypt/scrypt.go

+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package scrypt implements the scrypt key derivation function as defined in
+// Colin Percival's paper "Stronger Key Derivation via Sequential Memory-Hard
+// Functions" (https://www.tarsnap.com/scrypt/scrypt.pdf).
+package scrypt // import "golang.org/x/crypto/scrypt"
+
+import (
+	"crypto/sha256"
+	"errors"
+
+	"golang.org/x/crypto/pbkdf2"
+)
+
+const maxInt = int(^uint(0) >> 1)
+
+// blockCopy copies n numbers from src into dst.
+func blockCopy(dst, src []uint32, n int) {
+	copy(dst, src[:n])
+}
+
+// blockXOR XORs numbers from dst with n numbers from src.
+func blockXOR(dst, src []uint32, n int) {
+	for i, v := range src[:n] {
+		dst[i] ^= v
+	}
+}
+
+// salsaXOR applies Salsa20/8 to the XOR of 16 numbers from tmp and in,
+// and puts the result into both tmp and out.
+func salsaXOR(tmp *[16]uint32, in, out []uint32) {
+	w0 := tmp[0] ^ in[0]
+	w1 := tmp[1] ^ in[1]
+	w2 := tmp[2] ^ in[2]
+	w3 := tmp[3] ^ in[3]
+	w4 := tmp[4] ^ in[4]
+	w5 := tmp[5] ^ in[5]
+	w6 := tmp[6] ^ in[6]
+	w7 := tmp[7] ^ in[7]
+	w8 := tmp[8] ^ in[8]
+	w9 := tmp[9] ^ in[9]
+	w10 := tmp[10] ^ in[10]
+	w11 := tmp[11] ^ in[11]
+	w12 := tmp[12] ^ in[12]
+	w13 := tmp[13] ^ in[13]
+	w14 := tmp[14] ^ in[14]
+	w15 := tmp[15] ^ in[15]
+
+	x0, x1, x2, x3, x4, x5, x6, x7, x8 := w0, w1, w2, w3, w4, w5, w6, w7, w8
+	x9, x10, x11, x12, x13, x14, x15 := w9, w10, w11, w12, w13, w14, w15
+
+	for i := 0; i < 8; i += 2 {
+		u := x0 + x12
+		x4 ^= u<<7 | u>>(32-7)
+		u = x4 + x0
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x4
+		x12 ^= u<<13 | u>>(32-13)
+		u = x12 + x8
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x1
+		x9 ^= u<<7 | u>>(32-7)
+		u = x9 + x5
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x9
+		x1 ^= u<<13 | u>>(32-13)
+		u = x1 + x13
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x6
+		x14 ^= u<<7 | u>>(32-7)
+		u = x14 + x10
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x14
+		x6 ^= u<<13 | u>>(32-13)
+		u = x6 + x2
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x11
+		x3 ^= u<<7 | u>>(32-7)
+		u = x3 + x15
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x3
+		x11 ^= u<<13 | u>>(32-13)
+		u = x11 + x7
+		x15 ^= u<<18 | u>>(32-18)
+
+		u = x0 + x3
+		x1 ^= u<<7 | u>>(32-7)
+		u = x1 + x0
+		x2 ^= u<<9 | u>>(32-9)
+		u = x2 + x1
+		x3 ^= u<<13 | u>>(32-13)
+		u = x3 + x2
+		x0 ^= u<<18 | u>>(32-18)
+
+		u = x5 + x4
+		x6 ^= u<<7 | u>>(32-7)
+		u = x6 + x5
+		x7 ^= u<<9 | u>>(32-9)
+		u = x7 + x6
+		x4 ^= u<<13 | u>>(32-13)
+		u = x4 + x7
+		x5 ^= u<<18 | u>>(32-18)
+
+		u = x10 + x9
+		x11 ^= u<<7 | u>>(32-7)
+		u = x11 + x10
+		x8 ^= u<<9 | u>>(32-9)
+		u = x8 + x11
+		x9 ^= u<<13 | u>>(32-13)
+		u = x9 + x8
+		x10 ^= u<<18 | u>>(32-18)
+
+		u = x15 + x14
+		x12 ^= u<<7 | u>>(32-7)
+		u = x12 + x15
+		x13 ^= u<<9 | u>>(32-9)
+		u = x13 + x12
+		x14 ^= u<<13 | u>>(32-13)
+		u = x14 + x13
+		x15 ^= u<<18 | u>>(32-18)
+	}
+	x0 += w0
+	x1 += w1
+	x2 += w2
+	x3 += w3
+	x4 += w4
+	x5 += w5
+	x6 += w6
+	x7 += w7
+	x8 += w8
+	x9 += w9
+	x10 += w10
+	x11 += w11
+	x12 += w12
+	x13 += w13
+	x14 += w14
+	x15 += w15
+
+	out[0], tmp[0] = x0, x0
+	out[1], tmp[1] = x1, x1
+	out[2], tmp[2] = x2, x2
+	out[3], tmp[3] = x3, x3
+	out[4], tmp[4] = x4, x4
+	out[5], tmp[5] = x5, x5
+	out[6], tmp[6] = x6, x6
+	out[7], tmp[7] = x7, x7
+	out[8], tmp[8] = x8, x8
+	out[9], tmp[9] = x9, x9
+	out[10], tmp[10] = x10, x10
+	out[11], tmp[11] = x11, x11
+	out[12], tmp[12] = x12, x12
+	out[13], tmp[13] = x13, x13
+	out[14], tmp[14] = x14, x14
+	out[15], tmp[15] = x15, x15
+}
+
+func blockMix(tmp *[16]uint32, in, out []uint32, r int) {
+	blockCopy(tmp[:], in[(2*r-1)*16:], 16)
+	for i := 0; i < 2*r; i += 2 {
+		salsaXOR(tmp, in[i*16:], out[i*8:])
+		salsaXOR(tmp, in[i*16+16:], out[i*8+r*16:])
+	}
+}
+
+func integer(b []uint32, r int) uint64 {
+	j := (2*r - 1) * 16
+	return uint64(b[j]) | uint64(b[j+1])<<32
+}
+
+func smix(b []byte, r, N int, v, xy []uint32) {
+	var tmp [16]uint32
+	x := xy
+	y := xy[32*r:]
+
+	j := 0
+	for i := 0; i < 32*r; i++ {
+		x[i] = uint32(b[j]) | uint32(b[j+1])<<8 | uint32(b[j+2])<<16 | uint32(b[j+3])<<24
+		j += 4
+	}
+	for i := 0; i < N; i += 2 {
+		blockCopy(v[i*(32*r):], x, 32*r)
+		blockMix(&tmp, x, y, r)
+
+		blockCopy(v[(i+1)*(32*r):], y, 32*r)
+		blockMix(&tmp, y, x, r)
+	}
+	for i := 0; i < N; i += 2 {
+		j := int(integer(x, r) & uint64(N-1))
+		blockXOR(x, v[j*(32*r):], 32*r)
+		blockMix(&tmp, x, y, r)
+
+		j = int(integer(y, r) & uint64(N-1))
+		blockXOR(y, v[j*(32*r):], 32*r)
+		blockMix(&tmp, y, x, r)
+	}
+	j = 0
+	for _, v := range x[:32*r] {
+		b[j+0] = byte(v >> 0)
+		b[j+1] = byte(v >> 8)
+		b[j+2] = byte(v >> 16)
+		b[j+3] = byte(v >> 24)
+		j += 4
+	}
+}
+
+// Key derives a key from the password, salt, and cost parameters, returning
+// a byte slice of length keyLen that can be used as cryptographic key.
+//
+// N is a CPU/memory cost parameter, which must be a power of two greater than 1.
+// r and p must satisfy r * p < 2³⁰. If the parameters do not satisfy the
+// limits, the function returns a nil byte slice and an error.
+//
+// For example, you can get a derived key for e.g. AES-256 (which needs a
+// 32-byte key) by doing:
+//
+//      dk, err := scrypt.Key([]byte("some password"), salt, 32768, 8, 1, 32)
+//
+// The recommended parameters for interactive logins as of 2017 are N=32768, r=8
+// and p=1. The parameters N, r, and p should be increased as memory latency and
+// CPU parallelism increases; consider setting N to the highest power of 2 you
+// can derive within 100 milliseconds. Remember to get a good random salt.
+func Key(password, salt []byte, N, r, p, keyLen int) ([]byte, error) {
+	if N <= 1 || N&(N-1) != 0 {
+		return nil, errors.New("scrypt: N must be > 1 and a power of 2")
+	}
+	if uint64(r)*uint64(p) >= 1<<30 || r > maxInt/128/p || r > maxInt/256 || N > maxInt/128/r {
+		return nil, errors.New("scrypt: parameters are too large")
+	}
+
+	xy := make([]uint32, 64*r)
+	v := make([]uint32, 32*N*r)
+	b := pbkdf2.Key(password, salt, 1, p*128*r, sha256.New)
+
+	for i := 0; i < p; i++ {
+		smix(b[i*128*r:], r, N, v, xy)
+	}
+
+	return pbkdf2.Key(password, b, 1, keyLen, sha256.New), nil
+}

vendor/modules.txt

@@ -1146,6 +1146,7 @@ golang.org/x/crypto/pkcs12
 golang.org/x/crypto/pkcs12/internal/rc2
 golang.org/x/crypto/poly1305
 golang.org/x/crypto/salsa20/salsa
+golang.org/x/crypto/scrypt
 golang.org/x/crypto/ssh
 golang.org/x/crypto/ssh/terminal
 # golang.org/x/mod v0.3.0