Kubeadm supports for Kubelet Dynamic Configuration.

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -18,6 +18,7 @@ package kubeadm
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -142,6 +143,13 @@ type NodeConfiguration struct {
 	// without CA verification via DiscoveryTokenCACertHashes. This can weaken
 	// the security of kubeadm since other nodes can impersonate the master.
 	DiscoveryTokenUnsafeSkipCAVerification bool
+
+	// FeatureGates enabled by the user
+	FeatureGates map[string]bool
+}
+
+type KubeletConfiguration struct {
+	BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration
 }
 
 // GetControlPlaneImageRepository returns name of image repository

cmd/kubeadm/app/apis/kubeadm/v1alpha1/types.go

@@ -18,6 +18,7 @@ package v1alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -136,6 +137,13 @@ type NodeConfiguration struct {
 	// without CA verification via DiscoveryTokenCACertHashes. This can weaken
 	// the security of kubeadm since other nodes can impersonate the master.
 	DiscoveryTokenUnsafeSkipCAVerification bool `json:"discoveryTokenUnsafeSkipCAVerification"`
+
+	// FeatureGates enabled by the user
+	FeatureGates map[string]bool `json:"featureGates,omitempty"`
+}
+
+type KubeletConfiguration struct {
+	BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration
 }
 
 // HostPathMount contains elements describing volumes that are mounted from the

cmd/kubeadm/app/cmd/init.go

@@ -26,15 +26,20 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/ghodss/yaml"
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
 	flag "github.com/spf13/pflag"
 
+	"k8s.io/api/core/v1"
+	rbac "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/images"
@@ -58,9 +63,17 @@ import (
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
+	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 	utilsexec "k8s.io/utils/exec"
 )
 
+const (
+	// KubeletBaseConfigMapRoleName sets the name for the ClusterRole that allows access to ConfigMaps in the kube-system ns
+	KubeletBaseConfigMapRoleName = "kubeadm:kubelet-base-configmap"
+)
+
 var (
 	initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(`
 		Your Kubernetes master has initialized successfully!
@@ -217,7 +230,6 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, cfgPath *string, skipPreFlight, sk
 
 // NewInit validates given arguments and instantiates Init struct with provided information.
 func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint, dryRun bool, criSocket string) (*Init, error) {
-
 	fmt.Println("[kubeadm] WARNING: kubeadm is currently in beta")
 
 	if cfgPath != "" {
@@ -255,9 +267,6 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight,
 		if err := preflight.RunInitMasterChecks(utilsexec.New(), cfg, criSocket); err != nil {
 			return nil, err
 		}
-
-		// Try to start the kubelet service in case it's inactive
-		preflight.TryStartKubelet()
 	} else {
 		fmt.Println("[preflight] Skipping pre-flight checks.")
 	}
@@ -338,6 +347,57 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("error creating client: %v", err)
 	}
 
+	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		// TODO: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+		kubeletCfg := &kubeadmapi.KubeletConfiguration{
+			BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
+				PodManifestPath: "/etc/kubernetes/manifests",
+				AllowPrivileged: utilpointer.BoolPtr(true),
+				ClusterDNS:      []string{"10.96.0.10"},
+				ClusterDomain:   "cluster.local",
+				Authorization: kubeletconfigv1alpha1.KubeletAuthorization{
+					Mode: "Webhook",
+				},
+				Authentication: kubeletconfigv1alpha1.KubeletAuthentication{
+					X509: kubeletconfigv1alpha1.KubeletX509Authentication{
+						ClientCAFile: "/etc/kubernetes/pki/ca.crt",
+					},
+				},
+				CAdvisorPort: utilpointer.Int32Ptr(0),
+			},
+		}
+
+		// Convert cfg to the external version as that's the only version of the API that can be deserialized later
+		externalKubeletCfg := &kubeadmapiext.KubeletConfiguration{}
+		legacyscheme.Scheme.Convert(kubeletCfg, externalKubeletCfg, nil)
+
+		kubeletCfgYaml, err := yaml.Marshal(*externalKubeletCfg)
+		if err != nil {
+			return err
+		}
+
+		err = apiclient.CreateOrUpdateConfigMap(client, &v1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      kubeadmconstants.KubeletBaseConfigurationConfigMap,
+				Namespace: metav1.NamespaceSystem,
+			},
+			Data: map[string]string{
+				kubeadmconstants.KubeletBaseConfigurationConfigMapKey: string(kubeletCfgYaml),
+			},
+		})
+		if err != nil {
+			return err
+		}
+
+		if err := CreateKubeletBaseConfigMapRBACRules(client, i.cfg.NodeName); err != nil {
+			return fmt.Errorf("error creating kubelet base configmap RBAC rules: %v", err)
+		}
+
+	}
+
+	// Try to start the kubelet service in case it's inactive
+	preflight.TryStartKubelet()
+
 	// waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled
 	waiter := getWaiter(i.dryRun, client)
 
@@ -354,6 +414,13 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("couldn't initialize a Kubernetes cluster")
 	}
 
+	if features.Enabled(i.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		err = cmdutil.UpdateNodeWithConfigMap(client, i.cfg.NodeName)
+		if err != nil {
+			return nil
+		}
+	}
+
 	// Upload currently used configuration to the cluster
 	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
 	// depend on centralized information from this source in the future
@@ -535,3 +602,37 @@ func waitForAPIAndKubelet(waiter apiclient.Waiter) error {
 	// This call is blocking until one of the goroutines sends to errorChan
 	return <-errorChan
 }
+
+// CreateKubeletBaseConfigMapRBACRules creates the RBAC rules for exposing the kubelet base ConfigMap in the kube-system namespace to unauthenticated users
+func CreateKubeletBaseConfigMapRBACRules(client clientset.Interface, nodeName string) error {
+	err := apiclient.CreateOrUpdateRole(client, &rbac.Role{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      KubeletBaseConfigMapRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		Rules: []rbac.PolicyRule{
+			rbachelper.NewRule("get").Groups("").Resources("configmaps").Names(kubeadmconstants.KubeletBaseConfigurationConfigMap).RuleOrDie(),
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	return apiclient.CreateOrUpdateRoleBinding(client, &rbac.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      KubeletBaseConfigMapRoleName,
+			Namespace: metav1.NamespaceSystem,
+		},
+		RoleRef: rbac.RoleRef{
+			APIGroup: rbac.GroupName,
+			Kind:     "Role",
+			Name:     KubeletBaseConfigMapRoleName,
+		},
+		Subjects: []rbac.Subject{
+			{
+				Kind: "Group",
+				Name: kubeadmconstants.NodesGroup,
+			},
+		},
+	})
+}

cmd/kubeadm/app/cmd/join.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 
 	"github.com/renstrom/dedent"
 	"github.com/spf13/cobra"
@@ -31,8 +32,10 @@ import (
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
+	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
+	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
@@ -103,6 +106,7 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 	var skipPreFlight bool
 	var cfgPath string
 	var criSocket string
+	var featureGatesString string
 
 	cmd := &cobra.Command{
 		Use:   "join [flags]",
@@ -111,6 +115,11 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			cfg.DiscoveryTokenAPIServers = args
 
+			var err error
+			if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil {
+				kubeadmutil.CheckErr(err)
+			}
+
 			legacyscheme.Scheme.Default(cfg)
 			internalcfg := &kubeadmapi.NodeConfiguration{}
 			legacyscheme.Scheme.Convert(cfg, internalcfg, nil)
@@ -122,14 +131,14 @@ func NewCmdJoin(out io.Writer) *cobra.Command {
 		},
 	}
 
-	AddJoinConfigFlags(cmd.PersistentFlags(), cfg)
+	AddJoinConfigFlags(cmd.PersistentFlags(), cfg, &featureGatesString)
 	AddJoinOtherFlags(cmd.PersistentFlags(), &cfgPath, &skipPreFlight, &criSocket)
 
 	return cmd
 }
 
 // AddJoinConfigFlags adds join flags bound to the config to the specified flagset
-func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfiguration) {
+func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfiguration, featureGatesString *string) {
 	flagSet.StringVar(
 		&cfg.DiscoveryFile, "discovery-file", "",
 		"A file or url from which to load cluster information.")
@@ -151,6 +160,10 @@ func AddJoinConfigFlags(flagSet *flag.FlagSet, cfg *kubeadmapiext.NodeConfigurat
 	flagSet.StringVar(
 		&cfg.Token, "token", "",
 		"Use this token for both discovery-token and tls-bootstrap-token.")
+	flagSet.StringVar(
+		featureGatesString, "feature-gates", *featureGatesString,
+		"A set of key=value pairs that describe feature gates for various features. "+
+			"Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n"))
 }
 
 // AddJoinOtherFlags adds join flags that are not bound to a configuration file to the given flagset
@@ -199,9 +212,6 @@ func NewJoin(cfgPath string, args []string, cfg *kubeadmapi.NodeConfiguration, s
 		if err := preflight.RunJoinNodeChecks(utilsexec.New(), cfg, criSocket); err != nil {
 			return nil, err
 		}
-
-		// Try to start the kubelet service in case it's inactive
-		preflight.TryStartKubelet()
 	} else {
 		fmt.Println("[preflight] Skipping pre-flight checks.")
 	}
@@ -224,6 +234,22 @@ func (j *Join) Run(out io.Writer) error {
 		return err
 	}
 
+	// Try to start the kubelet service in case it's inactive
+	preflight.TryStartKubelet()
+
+	if features.Enabled(j.cfg.FeatureGates, features.DynamicKubeletConfig) {
+		// TODO: flag "--dynamic-config-dir" should be specified in /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+		client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
+		if err != nil {
+			return err
+		}
+
+		err = cmdutil.UpdateNodeWithConfigMap(client, j.cfg.NodeName)
+		if err != nil {
+			return nil
+		}
+	}
+
 	kubeconfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletBootstrapKubeConfigFileName)
 
 	// Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk

cmd/kubeadm/app/cmd/util/cmdutil.go

@@ -17,9 +17,18 @@ limitations under the License.
 package util
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/spf13/cobra"
+
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 )
 
 // SubCmdRunE returns a function that handles a case where a subcommand must be specified
@@ -57,3 +66,42 @@ func ValidateExactArgNumber(args []string, supportedArgs []string) error {
 	}
 	return nil
 }
+
+// UpdateNodeWithConfigMap updates node ConfigSource with KubeletBaseConfigurationConfigMap
+func UpdateNodeWithConfigMap(client clientset.Interface, nodeName string) error {
+	node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	oldData, err := json.Marshal(node)
+	if err != nil {
+		return err
+	}
+
+	kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeletBaseConfigurationConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	node.Spec.ConfigSource.ConfigMapRef.UID = kubeletCfg.UID
+
+	newData, err := json.Marshal(node)
+	if err != nil {
+		return err
+	}
+
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Node{})
+	if err != nil {
+		return err
+	}
+
+	if _, err := client.CoreV1().Nodes().Patch(node.Name, types.StrategicMergePatchType, patchBytes); err != nil {
+		if apierrs.IsConflict(err) {
+			fmt.Println("Temporarily unable to update node metadata due to conflict (will retry)")
+		}
+		return err
+	}
+
+	return nil
+}

cmd/kubeadm/app/constants/constants.go

@@ -140,6 +140,12 @@ const (
 	// MasterConfigurationConfigMapKey specifies in what ConfigMap key the master configuration should be stored
 	MasterConfigurationConfigMapKey = "MasterConfiguration"
 
+	// KubeletBaseConfigurationConfigMap specifies in what ConfigMap in the kube-system namespace the init kubelet configuration should be stored
+	KubeletBaseConfigurationConfigMap = "kubelet-base-config-1.9"
+
+	// KubeletBaseConfigurationConfigMapKey specifies in what ConfigMap key the init kubelet configuration should be stored
+	KubeletBaseConfigurationConfigMapKey = "kubelet"
+
 	// MinExternalEtcdVersion indicates minimum external etcd version which kubeadm supports
 	MinExternalEtcdVersion = "3.0.14"
 

cmd/kubeadm/app/features/features.go

@@ -41,6 +41,9 @@ const (
 
 	// SupportIPVSProxyMode is alpha in v1.8
 	SupportIPVSProxyMode = "SupportIPVSProxyMode"
+
+	// DynamicKubeletConfig is alpha in v1.9
+	DynamicKubeletConfig = "DynamicKubeletConfig"
 )
 
 var v190 = version.MustParseSemantic("v1.9.0-alpha.1")
@@ -52,6 +55,7 @@ var InitFeatureGates = FeatureList{
 	HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
 	SupportIPVSProxyMode: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
 	CoreDNS:              {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
+	DynamicKubeletConfig: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190},
 }
 
 // Feature represents a feature being gated

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -70,7 +70,6 @@ func CreateSchedulerStaticPodManifestFile(manifestDir string, cfg *kubeadmapi.Ma
 // GetStaticPodSpecs returns all staticPodSpecs actualized to the context of the current MasterConfiguration
 // NB. this methods holds the information about how kubeadm creates static pod mainfests.
 func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) map[string]v1.Pod {
-
 	// Get the required hostpath mounts
 	mounts := getHostPathVolumesForTheControlPlane(cfg)
 
@@ -110,7 +109,6 @@ func GetStaticPodSpecs(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.
 
 // createStaticPodFiles creates all the requested static pod files.
 func createStaticPodFiles(manifestDir string, cfg *kubeadmapi.MasterConfiguration, componentNames ...string) error {
-
 	// TODO: Move the "pkg/util/version".Version object into the internal API instead of always parsing the string
 	k8sVersion, err := version.ParseSemantic(cfg.KubernetesVersion)
 	if err != nil {
@@ -210,12 +208,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *versio
 		command = append(command, "--endpoint-reconciler-type="+reconcilers.LeaseEndpointReconcilerType)
 	}
 
+	if features.Enabled(cfg.FeatureGates, features.DynamicKubeletConfig) {
+		command = append(command, "--feature-gates=DynamicKubeletConfig=true")
+	}
+
 	return command
 }
 
 // getControllerManagerCommand builds the right controller manager command from the given config object and version
 func getControllerManagerCommand(cfg *kubeadmapi.MasterConfiguration, k8sVersion *version.Version) []string {
-
 	defaultArguments := map[string]string{
 		"address":                          "127.0.0.1",
 		"leader-elect":                     "true",

pkg/util/pointer/pointer.go

@@ -60,3 +60,9 @@ func Int32PtrDerefOr(ptr *int32, def int32) int32 {
 	}
 	return def
 }
+
+// BoolPtr returns a pointer to a bool
+func BoolPtr(b bool) *bool {
+	o := b
+	return &o
+}