Skip to content

K
k3s
Commit 2c944439 authored by Erik Wilson's avatar Erik Wilson
Browse files
Options

Refactor certs

parent c745be58
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 347 additions and 178 deletions
manifests/rolebindings.yaml 0 → 100644
View file @ 2c944439
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kube-apiserver-kubelet-admin
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kubelet-api-admin
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kube-apiserver
pkg/agent/config/config.go
View file @ 2c944439
......@@ -23,6 +23,7 @@ import (
"github.com/rancher/k3s/pkg/cli/cmds"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/net"
......@@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester {
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusForbidden {
return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("%s: %s", u, resp.Status)
}
......@@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) {
return "", err
}
nodePassword := hex.EncodeToString(password)
return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600)
return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600)
}
func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) {
servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile))
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil {
if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil {
return nil, errors.Wrapf(err, "failed to write node cert")
}
nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info)
servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info)
if err != nil {
return nil, err
}
if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil {
if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil {
return nil, errors.Wrapf(err, "failed to write node key")
}
cert, err := tls.X509KeyPair(nodeCert, nodeKey)
cert, err := tls.X509KeyPair(servingCert, servingKey)
if err != nil {
return nil, err
}
return &cert, nil
}
func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
clientCABytes := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: nodeCert.Certificate[1],
})
clientCA := filepath.Join(dataDir, "client-ca.pem")
if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil {
return "", errors.Wrapf(err, "failed to write client CA")
func getHostFile(filename string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info)
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}
return clientCA, nil
func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error {
basename := filepath.Base(filename)
fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile))
if err != nil {
return err
}
if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil {
return errors.Wrapf(err, "failed to write cert %s", filename)
}
return nil
}
func getHostnameAndIP(info cmds.Agent) (string, string, error) {
......@@ -169,17 +184,17 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) {
}
func localAddress(controlConfig *config.Control) string {
return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort)
}
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) {
os.MkdirAll(envInfo.DataDir, 0700)
kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
info.URL = "https://" + localAddress(controlConfig)
info.CACerts = pem.EncodeToMemory(&pem.Block{
Type: cert.CertificateBlockType,
Bytes: nodeCert.Certificate[1],
Bytes: tlsCert.Certificate[1],
})
return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
......@@ -253,36 +268,72 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
return nil, err
}
nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt")
nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key")
nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info)
hostLocal, err := exec.LookPath("host-local")
if err != nil {
return nil, errors.Wrapf(err, "failed to find host-local")
}
var flannelIface *sysnet.Interface
if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
if err != nil {
return nil, errors.Wrapf(err, "unable to find interface")
}
}
proxyURL := "https://" + localAddress(controlConfig)
clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt")
if err := getHostFile(clientCAFile, info); err != nil {
return nil, err
}
clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert)
if err != nil {
serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt")
if err := getHostFile(serverCAFile, info); err != nil {
return nil, err
}
kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert)
servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt")
servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key")
nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt")
servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info)
if err != nil {
return nil, err
}
hostLocal, err := exec.LookPath("host-local")
kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert)
if err != nil {
return nil, errors.Wrapf(err, "failed to find host-local")
return nil, err
}
var flannelIface *sysnet.Interface
if !envInfo.NoFlannel && len(envInfo.FlannelIface) > 0 {
flannelIface, err = sysnet.InterfaceByName(envInfo.FlannelIface)
if err != nil {
return nil, errors.Wrapf(err, "unable to find interface")
}
clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt")
if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil {
return nil, err
}
clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key")
if err := getHostFile(clientKubeletKey, info); err != nil {
return nil, err
}
kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig")
if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil {
return nil, err
}
clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt")
if err := getHostFile(clientKubeProxyCert, info); err != nil {
return nil, err
}
clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key")
if err := getHostFile(clientKubeProxyKey, info); err != nil {
return nil, err
}
kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig")
if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil {
return nil, err
}
nodeConfig := &config.Node{
......@@ -295,14 +346,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
nodeConfig.AgentConfig.NodeIP = nodeIP
nodeConfig.AgentConfig.NodeName = nodeName
nodeConfig.AgentConfig.NodeCertFile = nodeCertFile
nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile
nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert
nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey
nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
nodeConfig.AgentConfig.CACertPath = clientCA
nodeConfig.AgentConfig.ClientCA = clientCAFile
nodeConfig.AgentConfig.ListenAddress = "0.0.0.0"
nodeConfig.AgentConfig.KubeConfig = kubeConfig
nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode
nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
nodeConfig.CACerts = info.CACerts
......@@ -316,7 +369,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
nodeConfig.ServerAddress = serverURLParsed.Host
nodeConfig.Certificate = nodeCert
nodeConfig.Certificate = servingCert
if !nodeConfig.NoFlannel {
nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
......
pkg/agent/flannel/setup.go
View file @ 2c944439
......@@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error {
func Run(ctx context.Context, config *config.Node) error {
nodeName := config.AgentConfig.NodeName
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
......@@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error {
}
go func() {
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig)
err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
logrus.Fatalf("flannel exited: %v", err)
}()
......
pkg/agent/proxy/proxy.go
View file @ 2c944439
package proxy
import (
"crypto/tls"
"net/http"
"github.com/pkg/errors"
"github.com/google/tcpproxy"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/proxy"
"github.com/sirupsen/logrus"
)
func Run(config *config.Node) error {
proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
if err != nil {
return err
}
listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{
Certificates: []tls.Certificate{
*config.Certificate,
},
})
if err != nil {
return errors.Wrap(err, "Failed to start tls listener")
}
logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress)
var proxy tcpproxy.Proxy
proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress))
go func() {
err := http.Serve(listener, proxy)
err := proxy.Run()
logrus.Fatalf("TLS proxy stopped: %v", err)
}()
return nil
}
pkg/agent/tunnel/tunnel.go
View file @ 2c944439
......@@ -26,7 +26,7 @@ var (
)
func Setup(config *config.Node) error {
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
if err != nil {
return err
}
......
pkg/cli/cmds/server.go
View file @ 2c944439
......@@ -17,7 +17,7 @@ type Server struct {
DisableAgent bool
KubeConfigOutput string
KubeConfigMode string
KnownIPs cli.StringSlice
TLSSan cli.StringSlice
BindAddress string
ExtraAPIArgs cli.StringSlice
ExtraSchedulerArgs cli.StringSlice
......@@ -28,6 +28,8 @@ type Server struct {
StorageCAFile string
StorageCertFile string
StorageKeyFile string
AdvertiseIP string
AdvertisePort int
}
var ServerConfig Server
......@@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
cli.StringSliceFlag{
Name: "tls-san",
Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert",
Value: &ServerConfig.KnownIPs,
Value: &ServerConfig.TLSSan,
},
cli.StringSliceFlag{
Name: "kube-apiserver-arg",
......@@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
Destination: &ServerConfig.StorageKeyFile,
EnvVar: "K3S_STORAGE_KEYFILE",
},
cli.StringFlag{
Name: "advertise-address",
Usage: "IP address that apiserver uses to advertise to members of the cluster",
Destination: &ServerConfig.AdvertiseIP,
},
cli.IntFlag{
Name: "advertise-port",
Usage: "Port that apiserver uses to advertise to members of the cluster",
Value: 0,
Destination: &ServerConfig.AdvertisePort,
},
NodeIPFlag,
NodeNameFlag,
DockerFlag,
......
pkg/cli/server/server.go
View file @ 2c944439
......@@ -23,6 +23,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
"k8s.io/apimachinery/pkg/util/net"
"k8s.io/kubernetes/pkg/master"
"k8s.io/kubernetes/pkg/volume/csi"
_ "github.com/go-sql-driver/mysql" // ensure we have mysql
......@@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.Rootless = cfg.Rootless
serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs)
for _, san := range knownIPs(cfg.TLSSan) {
addr := net2.ParseIP(san)
if addr != nil {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san)
} else {
serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san)
}
}
serverConfig.TLSConfig.BindAddress = cfg.BindAddress
serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort
serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
......@@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile
serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile
serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile
serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" {
serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP
}
if serverConfig.ControlConfig.AdvertiseIP != "" {
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP)
}
_, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR)
if err != nil {
......@@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error {
return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err)
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange)
if err != nil {
return err
}
serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String())
// If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10,
// i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10
if cfg.ClusterDNS == "" {
......
pkg/daemons/agent/agent.go
View file @ 2c944439
......@@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) {
argsMap := map[string]string{
"proxy-mode": "iptables",
"healthz-bind-address": "127.0.0.1",
"kubeconfig": cfg.KubeConfig,
"kubeconfig": cfg.KubeConfigKubeProxy,
"cluster-cidr": cfg.ClusterCIDR.String(),
}
args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs)
......@@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) {
"read-only-port": "0",
"allow-privileged": "true",
"cluster-domain": cfg.ClusterDomain,
"kubeconfig": cfg.KubeConfig,
"kubeconfig": cfg.KubeConfigKubelet,
"eviction-hard": "imagefs.available<5%,nodefs.available<5%",
"eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%",
"fail-swap-on": "false",
......@@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) {
if cfg.ListenAddress != "" {
argsMap["address"] = cfg.ListenAddress
}
if cfg.CACertPath != "" {
if cfg.ClientCA != "" {
argsMap["anonymous-auth"] = "false"
argsMap["client-ca-file"] = cfg.CACertPath
argsMap["client-ca-file"] = cfg.ClientCA
}
if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" {
argsMap["tls-cert-file"] = cfg.NodeCertFile
argsMap["tls-private-key-file"] = cfg.NodeKeyFile
if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
argsMap["tls-cert-file"] = cfg.ServingKubeletCert
argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
}
if cfg.NodeName != "" {
argsMap["hostname-override"] = cfg.NodeName
......
pkg/daemons/config/types.go
View file @ 2c944439
......@@ -36,32 +36,41 @@ type Containerd struct {
}
type Agent struct {
NodeName string
NodeCertFile string
NodeKeyFile string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfig string
NodeIP string
RuntimeSocket string
ListenAddress string
CACertPath string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
NodeName string
ClientKubeletCert string
ClientKubeletKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ServingKubeletCert string
ServingKubeletKey string
ClusterCIDR net.IPNet
ClusterDNS net.IP
ClusterDomain string
ResolvConf string
RootDir string
KubeConfigNode string
KubeConfigKubelet string
KubeConfigKubeProxy string
NodeIP string
RuntimeSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
CNIPlugin bool
NodeTaints []string
NodeLabels []string
}
type Control struct {
AdvertisePort int
AdvertiseIP string
ListenPort int
HTTPSPort int
ProxyPort int
ClusterSecret string
ClusterIPRange *net.IPNet
ServiceIPRange *net.IPNet
......@@ -87,28 +96,44 @@ type Control struct {
}
type ControlRuntime struct {
TLSCert string
TLSKey string
TLSCA string
TLSCAKey string
TokenCA string
TokenCAKey string
ServiceKey string
PasswdFile string
KubeConfigSystem string
NodeCert string
NodeKey string
ClientToken string
NodeToken string
Handler http.Handler
Tunnel http.Handler
Authenticator authenticator.Request
ClientKubeAPICert string
ClientKubeAPIKey string
ClientCA string
ClientCAKey string
ServerCA string
ServerCAKey string
ServiceKey string
PasswdFile string
KubeConfigAdmin string
KubeConfigController string
KubeConfigScheduler string
KubeConfigAPIServer string
ServingKubeAPICert string
ServingKubeAPIKey string
ClientToken string
NodeToken string
Handler http.Handler
Tunnel http.Handler
Authenticator authenticator.Request
RequestHeaderCA string
RequestHeaderCAKey string
ClientAuthProxyCert string
ClientAuthProxyKey string
ClientAdminCert string
ClientAdminKey string
ClientControllerCert string
ClientControllerKey string
ClientSchedulerCert string
ClientSchedulerKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ServingKubeletKey string
ClientKubeletKey string
}
type ArgString []string
......
pkg/server/router.go
View file @ 2c944439
package server
import (
"bufio"
"crypto/rsa"
"crypto"
"crypto/x509"
"encoding/csv"
"errors"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
......@@ -17,10 +18,10 @@ import (
"github.com/gorilla/mux"
certutil "github.com/rancher/dynamiclistener/cert"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/daemons/control"
"github.com/rancher/k3s/pkg/openapi"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/kubernetes/pkg/master"
)
const (
......@@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC
authed.Use(authMiddleware(serverConfig))
authed.NotFoundHandler = serverConfig.Runtime.Handler
authed.Path("/v1-k3s/connect").Handler(tunnel)
authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig))
authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig))
authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig))
authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey))
authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig))
authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey))
authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert))
authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey))
authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA))
authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA))
authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
staticDir := filepath.Join(serverConfig.DataDir, "static")
......@@ -65,82 +72,122 @@ func cacerts(getter CACertsGetter) http.Handler {
})
}
func nodeCrt(server *config.Control) http.Handler {
func getNodeInfo(req *http.Request) (string, string, error) {
nodeNames := req.Header["K3s-Node-Name"]
if len(nodeNames) != 1 || nodeNames[0] == "" {
return "", "", errors.New("node name not set")
}
nodePasswords := req.Header["K3s-Node-Password"]
if len(nodePasswords) != 1 || nodePasswords[0] == "" {
return "", "", errors.New("node password not set")
}
return nodeNames[0], nodePasswords[0], nil
}
func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) {
keyBytes, err := ioutil.ReadFile(signingKeyFile)
if err != nil {
return nil, nil, nil, err
}
key, err := certutil.ParsePrivateKeyPEM(keyBytes)
if err != nil {
return nil, nil, nil, err
}
caKeyBytes, err := ioutil.ReadFile(caKeyFile)
if err != nil {
return nil, nil, nil, err
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
if err != nil {
return nil, nil, nil, err
}
caBytes, err := ioutil.ReadFile(caCertFile)
if err != nil {
return nil, nil, nil, err
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
return nil, nil, nil, err
}
return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil
}
func servingKubeletCert(server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
nodeNames := req.Header["K3s-Node-Name"]
if len(nodeNames) != 1 || nodeNames[0] == "" {
sendError(errors.New("node name not set"), resp)
return
}
nodePasswords := req.Header["K3s-Node-Password"]
if len(nodePasswords) != 1 || nodePasswords[0] == "" {
sendError(errors.New("node password not set"), resp)
return
nodeName, nodePassword, err := getNodeInfo(req)
if err != nil {
sendError(err, resp)
}
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil {
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
sendError(err, resp, http.StatusForbidden)
return
}
nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey)
caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey)
if err != nil {
sendError(err, resp)
return
}
key, err := certutil.ParsePrivateKeyPEM(nodeKey)
cert, err := certutil.NewSignedCert(certutil.Config{
CommonName: nodeName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
AltNames: certutil.AltNames{
DNSNames: []string{nodeName, "localhost"},
IPs: []net.IP{net.ParseIP("127.0.0.1")},
},
}, key, caCert[0], caKey)
if err != nil {
sendError(err, resp)
return
}
caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey)
if err != nil {
sendError(err, resp)
return
}
resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
})
}
caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA)
if err != nil {
sendError(err, resp)
func clientKubeletCert(server *config.Control) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
nodeName, nodePassword, err := getNodeInfo(req)
if err != nil {
sendError(err, resp)
return
}
caCert, err := certutil.ParseCertsPEM(caBytes)
if err != nil {
sendError(err, resp)
if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil {
sendError(err, resp, http.StatusForbidden)
return
}
_, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange)
caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey)
if err != nil {
sendError(err, resp)
return
}
cfg := certutil.Config{
CommonName: "kubernetes",
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
AltNames: certutil.AltNames{
DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]},
IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")},
},
}
cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey))
cert, err := certutil.NewSignedCert(certutil.Config{
CommonName: "system:node:" + nodeName,
Organization: []string{"system:nodes"},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
}, key, caCert[0], caKey)
if err != nil {
sendError(err, resp)
return
......@@ -150,13 +197,13 @@ func nodeCrt(server *config.Control) http.Handler {
})
}
func nodeKey(server *config.Control) http.Handler {
func fileHandler(fileName string) http.Handler {
return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
if req.TLS == nil {
resp.WriteHeader(http.StatusNotFound)
return
}
http.ServeFile(resp, req, server.Runtime.NodeKey)
http.ServeFile(resp, req, fileName)
})
}
......@@ -225,29 +272,29 @@ func ensureNodePassword(passwdFile, nodeName, passwd string) error {
defer f.Close()
user := strings.ToLower("node:" + nodeName)
buf := &strings.Builder{}
scan := bufio.NewScanner(f)
for scan.Scan() {
line := scan.Text()
parts := strings.Split(line, ",")
if len(parts) < 4 {
continue
records := [][]string{}
reader := csv.NewReader(f)
for {
record, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
return err
}
if parts[1] == user {
if parts[0] == passwd {
if len(record) < 3 {
return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record))
}
if record[1] == user {
if record[0] == passwd {
return nil
}
return fmt.Errorf("Node password validation failed for [%s]", nodeName)
return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile)
}
buf.WriteString(line)
buf.WriteString("\n")
}
buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user))
if scan.Err() != nil {
return scan.Err()
records = append(records, record)
}
records = append(records, []string{passwd, user, user, "system:node:" + nodeName})
f.Close()
return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600)
return control.WritePasswords(passwdFile, records)
}
pkg/server/server.go
View file @ 2c944439
......@@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
controlConfig = &config.ControlConfig
)
caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA)
if err != nil {
return "", err
}
caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey)
if err != nil {
return "", err
}
tlsConfig.CACerts = string(caBytes)
tlsConfig.CAKey = string(caKeyBytes)
tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) {
if tlsServer == nil {
return "", nil
......@@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) {
return tlsServer.CACert()
})
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem)
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return "", err
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment