kubeadm: graduate control plane prepare phase

cmd/kubeadm/app/cmd/BUILD

@@ -38,7 +38,6 @@ go_library(
         "//cmd/kubeadm/app/images:go_default_library",
         "//cmd/kubeadm/app/phases/bootstraptoken/node:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
-        "//cmd/kubeadm/app/phases/controlplane:go_default_library",
         "//cmd/kubeadm/app/phases/etcd:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",

cmd/kubeadm/app/cmd/join.go

@@ -29,6 +29,7 @@ import (
 	flag "github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/klog"
@@ -42,10 +43,7 @@ import (
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
-	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
-	controlplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
 	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
-	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane"
 	patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
@@ -161,6 +159,7 @@ type joinData struct {
 	cfg                   *kubeadmapi.JoinConfiguration
 	initCfg               *kubeadmapi.InitConfiguration
 	tlsBootstrapCfg       *clientcmdapi.Config
+	clientSets            map[string]*clientset.Clientset
 	ignorePreflightErrors sets.String
 	outputWriter          io.Writer
 }
@@ -200,6 +199,8 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
 	addJoinOtherFlags(cmd.Flags(), &joinOptions.cfgPath, &joinOptions.ignorePreflightErrors, &joinOptions.controlPlane, &joinOptions.token)
 
 	joinRunner.AppendPhase(phases.NewPreflightJoinPhase())
+	joinRunner.AppendPhase(phases.NewControlPlanePreparePhase())
+	joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
 
 	// sets the data builder function, that will be used by the runner
 	// both when running the entire workflow or single phases
@@ -373,6 +374,7 @@ func newJoinData(cmd *cobra.Command, args []string, options *joinOptions, out io
 
 	return &joinData{
 		cfg:                   cfg,
+		clientSets:            map[string]*clientset.Clientset{},
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		outputWriter:          out,
 	}, nil
@@ -408,6 +410,18 @@ func (j *joinData) InitCfg() (*kubeadmapi.InitConfiguration, error) {
 	return initCfg, err
 }
 
+func (j *joinData) ClientSetFromFile(path string) (*clientset.Clientset, error) {
+	if client, ok := j.clientSets[path]; ok {
+		return client, nil
+	}
+	client, err := kubeconfigutil.ClientSetFromFile(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "[join] couldn't create Kubernetes client")
+	}
+	j.clientSets[path] = client
+	return client, nil
+}
+
 // IgnorePreflightErrors returns the list of preflight errors to ignore.
 func (j *joinData) IgnorePreflightErrors() sets.String {
 	return j.ignorePreflightErrors
@@ -432,13 +446,6 @@ func (j *joinData) Run() error {
 	if err != nil {
 		return err
 	}
-	if j.cfg.ControlPlane != nil {
-		// Prepares the node for hosting a new control plane instance by writing necessary
-		// kubeconfig files, and static pod manifests
-		if err := j.PrepareForHostingControlPlane(initCfg); err != nil {
-			return err
-		}
-	}
 
 	// Executes the kubelet TLS bootstrap process, that completes with the node
 	// joining the cluster with a dedicates set of credentials as required by
@@ -478,47 +485,6 @@ func (j *joinData) Run() error {
 	return nil
 }
 
-// PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance
-func (j *joinData) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
-
-	// Generate missing certificates (if any)
-	if err := certsphase.CreatePKIAssets(initConfiguration); err != nil {
-		return err
-	}
-
-	// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
-	// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
-	// following steps of the join --experimental-control plane workflow
-	if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil {
-		return errors.Wrap(err, "error generating kubeconfig files")
-	}
-
-	// Creates static pod manifests file for the control plane components to be deployed on this node
-	// Static pods will be created and managed by the kubelet as soon as it starts
-	if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil {
-		return errors.Wrap(err, "error creating static pod manifest files for the control plane components")
-	}
-
-	// in case of local etcd
-	if initConfiguration.Etcd.External == nil {
-		// Checks that the etcd cluster is healthy
-		// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
-		//     for connecting to etcd already in place
-		kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
-
-		client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
-		if err != nil {
-			return errors.Wrap(err, "couldn't create Kubernetes client")
-		}
-
-		if err := etcdphase.CheckLocalEtcdClusterStatus(client, &initConfiguration.ClusterConfiguration); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // BootstrapKubelet executes the kubelet TLS bootstrap process.
 // This process is executed by the kubelet and completes with the node joining the cluster
 // with a dedicates set of credentials as required by the node authorizer

cmd/kubeadm/app/cmd/phases/BUILD

@@ -6,6 +6,7 @@ go_library(
         "addons.go",
         "bootstraptoken.go",
         "certs.go",
+        "checketcd.go",
         "controlplane.go",
         "etcd.go",
         "kubeconfig.go",

cmd/kubeadm/app/cmd/phases/checketcd.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package phases
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
+	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
+)
+
+type checkEtcdData interface {
+	Cfg() *kubeadmapi.JoinConfiguration
+	ClientSetFromFile(string) (*clientset.Clientset, error)
+	InitCfg() (*kubeadmapi.InitConfiguration, error)
+}
+
+// NewCheckEtcdPhase is a hidden phase that runs after the control-plane-prepare and
+// before the bootstrap-kubelet phase that ensures etcd is healthy
+func NewCheckEtcdPhase() workflow.Phase {
+	return workflow.Phase{
+		Name:   "check-etcd",
+		Run:    runCheckEtcdPhase,
+		Hidden: true,
+	}
+}
+
+func runCheckEtcdPhase(c workflow.RunData) error {
+	data, ok := c.(checkEtcdData)
+	if !ok {
+		return errors.New("check-etcd phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	if cfg.Etcd.External != nil {
+		fmt.Println("[check-etcd] Skipping etcd check in external mode")
+		return nil
+	}
+
+	fmt.Println("[check-etcd] Checking that the etcd cluster is healthy")
+
+	// Checks that the etcd cluster is healthy
+	// NB. this check cannot be implemented before because it requires the admin.conf and all the certificates
+	//     for connecting to etcd already in place
+	client, err := data.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
+	if err != nil {
+		return err
+	}
+
+	return etcdphase.CheckLocalEtcdClusterStatus(client, &cfg.ClusterConfiguration)
+}

cmd/kubeadm/app/cmd/phases/controlplane.go

@@ -21,18 +21,21 @@ import (
 
 	"github.com/pkg/errors"
 
+	clientset "k8s.io/client-go/kubernetes"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	certsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
+	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	"k8s.io/kubernetes/pkg/util/normalizer"
 )
 
 var (
 	controlPlaneExample = normalizer.Examples(`
-		# Generates all static Pod manifest files for control plane components, 
+		# Generates all static Pod manifest files for control plane components,
 		# functionally equivalent to what is generated by kubeadm init.
 		kubeadm init phase control-plane all
 
@@ -65,6 +68,12 @@ type controlPlaneData interface {
 	ManifestDir() string
 }
 
+type controlPlanePrepareData interface {
+	Cfg() *kubeadmapi.JoinConfiguration
+	ClientSetFromFile(string) (*clientset.Clientset, error)
+	InitCfg() (*kubeadmapi.InitConfiguration, error)
+}
+
 func getPhaseDescription(component string) string {
 	return fmt.Sprintf("Generates the %s static Pod manifest", component)
 }
@@ -83,20 +92,34 @@ func NewControlPlanePhase() workflow.Phase {
 				Example:        controlPlaneExample,
 				RunAllSiblings: true,
 			},
-			newControlPlaneSubPhase(kubeadmconstants.KubeAPIServer),
-			newControlPlaneSubPhase(kubeadmconstants.KubeControllerManager),
-			newControlPlaneSubPhase(kubeadmconstants.KubeScheduler),
+			newControlPlaneSubphase(kubeadmconstants.KubeAPIServer),
+			newControlPlaneSubphase(kubeadmconstants.KubeControllerManager),
+			newControlPlaneSubphase(kubeadmconstants.KubeScheduler),
 		},
 		Run: runControlPlanePhase,
 	}
 	return phase
 }
 
-func newControlPlaneSubPhase(component string) workflow.Phase {
+// NewControlPlanePreparePhase creates a kubeadm workflow phase that implements the preparation of the node to serve a control plane
+func NewControlPlanePreparePhase() workflow.Phase {
+	return workflow.Phase{
+		Name:  "control-plane-prepare",
+		Short: "Prepares the machine for serving a control plane.",
+		Long:  cmdutil.MacroCommandLongDescription,
+		Phases: []workflow.Phase{
+			newControlPlanePrepareCertsSubphase(),
+			newControlPlanePrepareKubeconfigSubphase(),
+			newControlPlanePrepareManifestsSubphases(),
+		},
+	}
+}
+
+func newControlPlaneSubphase(component string) workflow.Phase {
 	phase := workflow.Phase{
 		Name:         controlPlanePhaseProperties[component].name,
 		Short:        controlPlanePhaseProperties[component].short,
-		Run:          runControlPlaneSubPhase(component),
+		Run:          runControlPlaneSubphase(component),
 		InheritFlags: getControlPlanePhaseFlags(component),
 	}
 	return phase
@@ -132,6 +155,65 @@ func getControlPlanePhaseFlags(name string) []string {
 	return flags
 }
 
+func getControlPlanePreparePhaseFlags() []string {
+	return []string{
+		options.APIServerAdvertiseAddress,
+		options.APIServerBindPort,
+		options.CfgPath,
+		options.ControlPlane,
+		options.NodeName,
+		options.TokenDiscovery,
+		options.TokenDiscoveryCAHash,
+		options.TokenDiscoverySkipCAHash,
+	}
+}
+
+func newControlPlanePrepareCertsSubphase() workflow.Phase {
+	return workflow.Phase{
+		Name:         "certs",
+		Short:        "Generates the certificates for the new control plane components",
+		Run:          runControlPlanePrepareCertsPhaseLocal,
+		InheritFlags: getControlPlanePreparePhaseFlags(),
+	}
+}
+
+func newControlPlanePrepareKubeconfigSubphase() workflow.Phase {
+	return workflow.Phase{
+		Name:         "kubeconfig",
+		Short:        "Generates the kubeconfig for the new control plane components",
+		Run:          runControlPlanePrepareKubeconfigPhaseLocal,
+		InheritFlags: getControlPlanePreparePhaseFlags(),
+	}
+}
+
+func newControlPlanePrepareManifestsSubphases() workflow.Phase {
+	return workflow.Phase{
+		Name:  "manifests",
+		Short: "Generates the manifests for the new control plane components",
+		Phases: []workflow.Phase{
+			{
+				Name:           "all",
+				Short:          "Generates all static Pod manifest files",
+				InheritFlags:   getControlPlanePreparePhaseFlags(),
+				RunAllSiblings: true,
+			},
+			newControlPlanePrepareSubphase(kubeadmconstants.KubeAPIServer),
+			newControlPlanePrepareSubphase(kubeadmconstants.KubeControllerManager),
+			newControlPlanePrepareSubphase(kubeadmconstants.KubeScheduler),
+		},
+		InheritFlags: getControlPlanePreparePhaseFlags(),
+	}
+}
+
+func newControlPlanePrepareSubphase(component string) workflow.Phase {
+	return workflow.Phase{
+		Name:         controlPlanePhaseProperties[component].name,
+		Short:        controlPlanePhaseProperties[component].short,
+		Run:          runControlPlanePrepareJoinSubphase(component),
+		InheritFlags: getControlPlanePreparePhaseFlags(),
+	}
+}
+
 func runControlPlanePhase(c workflow.RunData) error {
 	data, ok := c.(controlPlaneData)
 	if !ok {
@@ -142,7 +224,7 @@ func runControlPlanePhase(c workflow.RunData) error {
 	return nil
 }
 
-func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
+func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
 	return func(c workflow.RunData) error {
 		data, ok := c.(controlPlaneData)
 		if !ok {
@@ -154,3 +236,75 @@ func runControlPlaneSubPhase(component string) func(c workflow.RunData) error {
 		return controlplane.CreateStaticPodFiles(data.ManifestDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
 	}
 }
+
+func runControlPlanePrepareCertsPhaseLocal(c workflow.RunData) error {
+	data, ok := c.(controlPlanePrepareData)
+	if !ok {
+		return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	// Generate missing certificates (if any)
+	return certsphase.CreatePKIAssets(cfg)
+}
+
+func runControlPlanePrepareKubeconfigPhaseLocal(c workflow.RunData) error {
+	data, ok := c.(controlPlanePrepareData)
+	if !ok {
+		return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+	}
+
+	// Skip if this is not a control plane
+	if data.Cfg().ControlPlane == nil {
+		return nil
+	}
+
+	cfg, err := data.InitCfg()
+	if err != nil {
+		return err
+	}
+
+	fmt.Println("[control-plane-prepare] Generating kubeconfig files")
+
+	// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
+	// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
+	// following steps of the join --experimental-control plane workflow
+	if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, cfg); err != nil {
+		return errors.Wrap(err, "error generating kubeconfig files")
+	}
+
+	return nil
+}
+
+func runControlPlanePrepareJoinSubphase(component string) func(c workflow.RunData) error {
+	return func(c workflow.RunData) error {
+		data, ok := c.(controlPlanePrepareData)
+		if !ok {
+			return errors.New("control-plane-prepare phase invoked with an invalid data struct")
+		}
+
+		// Skip if this is not a control plane
+		if data.Cfg().ControlPlane == nil {
+			return nil
+		}
+
+		cfg, err := data.InitCfg()
+		if err != nil {
+			return err
+		}
+
+		// Creates static pod manifests file for the control plane components to be deployed on this node
+		// Static pods will be created and managed by the kubelet as soon as it starts
+		fmt.Printf("[control-plane-prepare] Creating static Pod manifest for %q\n", component)
+		return controlplane.CreateStaticPodFiles(kubeadmconstants.GetStaticPodDirectory(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
+	}
+}