controller-manager: switch to config/option struct pattern

cmd/cloud-controller-manager/app/config/config.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"time"
+
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+)
+
+// ExtraConfig are part of Config, also can place your custom config here.
+type ExtraConfig struct {
+	NodeStatusUpdateFrequency time.Duration
+}
+
+// Config is the main context object for the cloud controller manager.
+type Config struct {
+	Generic genericcontrollermanager.Config
+	Extra   ExtraConfig
+}
+
+type completedConfig struct {
+	Generic genericcontrollermanager.CompletedConfig
+	Extra   *ExtraConfig
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{
+		c.Generic.Complete(),
+		&c.Extra,
+	}
+
+	return &CompletedConfig{&cc}
+}

cmd/cloud-controller-manager/app/options/options.go

@@ -17,10 +17,13 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
 	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/master/ports"
@@ -31,39 +34,78 @@ import (
 	"github.com/spf13/pflag"
 )
 
-// CloudControllerManagerServer is the main context object for the controller manager.
-type CloudControllerManagerServer struct {
-	cmoptions.ControllerManagerServer
+// CloudControllerManagerOptions is the main context object for the controller manager.
+type CloudControllerManagerOptions struct {
+	Generic cmoptions.GenericControllerManagerOptions
 
 	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
 	NodeStatusUpdateFrequency metav1.Duration
 }
 
-// NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
-func NewCloudControllerManagerServer() *CloudControllerManagerServer {
-	s := CloudControllerManagerServer{
+// NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config.
+func NewCloudControllerManagerOptions() *CloudControllerManagerOptions {
+	componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureCloudControllerManagerPort)
+
+	s := CloudControllerManagerOptions{
 		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
 		// Please make common changes there and put anything cloud specific here.
-		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			KubeControllerManagerConfiguration: cmoptions.GetDefaultControllerOptions(ports.CloudControllerManagerPort),
-		},
+		Generic:                   cmoptions.NewGenericControllerManagerOptions(componentConfig),
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
 	}
-	s.LeaderElection.LeaderElect = true
+	s.Generic.ComponentConfig.LeaderElection.LeaderElect = true
+
 	return &s
 }
 
 // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
-func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
-	cmoptions.AddDefaultControllerFlags(&s.ControllerManagerServer, fs)
-	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
-	fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
+func (o *CloudControllerManagerOptions) AddFlags(fs *pflag.FlagSet) {
+	o.Generic.AddFlags(fs)
+
+	fs.StringVar(&o.Generic.ComponentConfig.CloudProvider, "cloud-provider", o.Generic.ComponentConfig.CloudProvider, "The provider of cloud services. Cannot be empty.")
+	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
-	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
+	fs.StringVar(&o.Generic.ComponentConfig.ServiceAccountKeyFile, "service-account-private-key-file", o.Generic.ComponentConfig.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
 	fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
-	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
+	fs.Int32Var(&o.Generic.ComponentConfig.ConcurrentServiceSyncs, "concurrent-service-syncs", o.Generic.ComponentConfig.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
 
-	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
+	leaderelectionconfig.BindFlags(&o.Generic.ComponentConfig.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }
+
+// ApplyTo fills up cloud controller manager config with options.
+func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config) error {
+	if err := o.Generic.ApplyTo(&c.Generic, "cloud-controller-manager"); err != nil {
+		return err
+	}
+
+	c.Extra.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency.Duration
+
+	return nil
+}
+
+// Validate is used to validate config before launching the cloud controller manager
+func (o *CloudControllerManagerOptions) Validate() error {
+	errors := []error{}
+	errors = append(errors, o.Generic.Validate()...)
+
+	if len(o.Generic.ComponentConfig.CloudProvider) == 0 {
+		errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty"))
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Config return a cloud controller manager config objective
+func (o CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, error) {
+	if err := o.Validate(); err != nil {
+		return nil, err
+	}
+
+	c := &cloudcontrollerconfig.Config{}
+	if err := o.ApplyTo(c); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}

cmd/cloud-controller-manager/app/options/options_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"net"
 	"reflect"
 	"testing"
 	"time"
@@ -31,7 +32,7 @@ import (
 
 func TestAddFlags(t *testing.T) {
 	f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
-	s := NewCloudControllerManagerServer()
+	s := NewCloudControllerManagerOptions()
 	s.AddFlags(f)
 
 	args := []string{
@@ -65,13 +66,13 @@ func TestAddFlags(t *testing.T) {
 	}
 	f.Parse(args)
 
-	expected := &CloudControllerManagerServer{
-		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
+	expected := &CloudControllerManagerOptions{
+		Generic: cmoptions.GenericControllerManagerOptions{
+			ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
 				CloudProvider:                                   "gce",
 				CloudConfigFile:                                 "/cloud-config",
-				Port:                                            10000,
-				Address:                                         "192.168.4.10",
+				Port:                                            10253,     // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				Address:                                         "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
 				ConcurrentEndpointSyncs:                         5,
 				ConcurrentRSSyncs:                               5,
 				ConcurrentResourceQuotaSyncs:                    5,
@@ -138,6 +139,11 @@ func TestAddFlags(t *testing.T) {
 				CIDRAllocatorType:         "RangeAllocator",
 				Controllers:               []string{"*"},
 			},
+			InsecureServing: &cmoptions.InsecureServingOptions{
+				BindAddress: net.ParseIP("192.168.4.10"),
+				BindPort:    int(10000),
+				BindNetwork: "tcp",
+			},
 			Kubeconfig: "/kubeconfig",
 			Master:     "192.168.4.20",
 		},

cmd/controller-manager/app/config.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// Config is the main context object for the controller manager.
+type Config struct {
+	// TODO: split up the component config. This is not generic.
+	ComponentConfig componentconfig.KubeControllerManagerConfiguration
+
+	InsecureServing *InsecureServingInfo
+
+	// the general kube client
+	Client *clientset.Clientset
+
+	// the client only used for leader election
+	LeaderElectionClient *clientset.Clientset
+
+	// the rest config for the master
+	Kubeconfig *restclient.Config
+
+	// the event sink
+	EventRecorder record.EventRecorder
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() CompletedConfig {
+	cc := completedConfig{c}
+	return CompletedConfig{&cc}
+}

cmd/controller-manager/app/insecure_serving.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apiserver/pkg/server"
+)
+
+// InsecureServingInfo is the main context object for the insecure http server.
+type InsecureServingInfo struct {
+	// Listener is the secure server network listener.
+	Listener net.Listener
+}
+
+// Serve starts an insecure http server with the given handler. It fails only if
+// the initial listen call fails. It does not block.
+func (s *InsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error {
+	insecureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	glog.Infof("Serving insecurely on %s", s.Listener.Addr())
+	return server.RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
+}

cmd/controller-manager/app/options/insecure_serving.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/server/options"
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// InsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
+// No one should be using these anymore.
+type InsecureServingOptions struct {
+	BindAddress net.IP
+	BindPort    int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
+}
+
+// Validate ensures that the insecure port values within the range of the port.
+func (s *InsecureServingOptions) Validate() []error {
+	errors := []error{}
+
+	if s == nil {
+		return nil
+	}
+
+	if s.BindPort < 0 || s.BindPort > 32767 {
+		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 32767, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
+	}
+
+	return errors
+}
+
+// AddFlags adds flags related to insecure serving for controller manager to the specified FlagSet.
+func (s *InsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+}
+
+// AddDeprecatedFlags adds deprecated flags related to insecure serving for controller manager to the specified FlagSet.
+// TODO: remove it until kops stop using `--address`
+func (s *InsecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
+		"DEPRECATED: the IP address on which to listen for the --port port. See --bind-address instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("address", "see --bind-address instead.")
+
+	fs.IntVar(&s.BindPort, "port", s.BindPort, "DEPRECATED: the port on which to serve HTTP insecurely without authentication and authorization. If 0, don't serve HTTPS at all. See --secure-port instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("port", "see --secure-port instead.")
+}
+
+// ApplyTo adds InsecureServingOptions to the insecureserverinfo amd kube-controller manager configuration.
+// Note: the double pointer allows to set the *InsecureServingInfo to nil without referencing the struct hosting this pointer.
+func (s *InsecureServingOptions) ApplyTo(c **genericcontrollermanager.InsecureServingInfo, cfg *componentconfig.KubeControllerManagerConfiguration) error {
+	if s == nil {
+		return nil
+	}
+	if s.BindPort <= 0 {
+		return nil
+	}
+
+	if s.Listener == nil {
+		var err error
+		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
+		s.Listener, s.BindPort, err = options.CreateListener(s.BindNetwork, addr)
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	}
+
+	*c = &genericcontrollermanager.InsecureServingInfo{
+		Listener: s.Listener,
+	}
+
+	// sync back to component config
+	// TODO: find more elegant way than synching back the values.
+	cfg.Port = int32(s.BindPort)
+	cfg.Address = s.BindAddress.String()
+
+	return nil
+}

cmd/controller-manager/app/options/utils.go

-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package options
-
-import (
-	"github.com/cloudflare/cfssl/helpers"
-	"time"
-
-	"github.com/spf13/pflag"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
-	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
-)
-
-// ControllerManagerServer is the common structure for a controller manager. It works with GetDefaultControllerOptions
-// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
-type ControllerManagerServer struct {
-	componentconfig.KubeControllerManagerConfiguration
-
-	Master     string
-	Kubeconfig string
-}
-
-const (
-	// These defaults are deprecated and exported so that we can warn if
-	// they are being used.
-
-	// DefaultClusterSigningCertFile is deprecated. Do not use.
-	DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
-	// DefaultClusterSigningKeyFile is deprecated. Do not use.
-	DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
-)
-
-// GetDefaultControllerOptions returns common/default configuration values for both
-// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
-// be made here. Any individual changes should be made in that controller.
-func GetDefaultControllerOptions(port int32) componentconfig.KubeControllerManagerConfiguration {
-	return componentconfig.KubeControllerManagerConfiguration{
-		Controllers:                                     []string{"*"},
-		Port:                                            port,
-		Address:                                         "0.0.0.0",
-		ConcurrentEndpointSyncs:                         5,
-		ConcurrentServiceSyncs:                          1,
-		ConcurrentRCSyncs:                               5,
-		ConcurrentRSSyncs:                               5,
-		ConcurrentDaemonSetSyncs:                        2,
-		ConcurrentJobSyncs:                              5,
-		ConcurrentResourceQuotaSyncs:                    5,
-		ConcurrentDeploymentSyncs:                       5,
-		ConcurrentNamespaceSyncs:                        10,
-		ConcurrentSATokenSyncs:                          5,
-		RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
-		ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
-		NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
-		PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
-		HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
-		HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
-		HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
-		HorizontalPodAutoscalerTolerance:                0.1,
-		DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
-		MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
-		RegisterRetryCount:                              10,
-		PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
-		NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
-		NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
-		NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
-		ClusterName:                                     "kubernetes",
-		NodeCIDRMaskSize:                                24,
-		ConfigureCloudRoutes:                            true,
-		TerminatedPodGCThreshold:                        12500,
-		VolumeConfiguration: componentconfig.VolumeConfiguration{
-			EnableHostPathProvisioning: false,
-			EnableDynamicProvisioning:  true,
-			PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
-				MaximumRetry:             3,
-				MinimumTimeoutNFS:        300,
-				IncrementTimeoutNFS:      30,
-				MinimumTimeoutHostPath:   60,
-				IncrementTimeoutHostPath: 30,
-			},
-			FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
-		},
-		ContentType:                           "application/vnd.kubernetes.protobuf",
-		KubeAPIQPS:                            20.0,
-		KubeAPIBurst:                          30,
-		LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-		ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
-		EnableGarbageCollector:                true,
-		ConcurrentGCSyncs:                     20,
-		ClusterSigningCertFile:                DefaultClusterSigningCertFile,
-		ClusterSigningKeyFile:                 DefaultClusterSigningKeyFile,
-		ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
-		ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
-		EnableTaintManager:                    true,
-		HorizontalPodAutoscalerUseRESTClients: true,
-	}
-}
-
-// AddDefaultControllerFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
-// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
-func AddDefaultControllerFlags(s *ControllerManagerServer, fs *pflag.FlagSet) {
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on.")
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
-}

cmd/controller-manager/app/serve.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net/http"
+	"net/http/pprof"
+	goruntime "runtime"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/kubernetes/pkg/util/configz"
+)
+
+type serveFunc func(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error
+
+// Serve creates a base handler chain for a controller manager. It runs the
+// the chain with the given serveFunc.
+func Serve(c *CompletedConfig, serveFunc serveFunc, stopCh <-chan struct{}) error {
+	mux := http.NewServeMux()
+	healthz.InstallHandler(mux)
+	if c.ComponentConfig.EnableProfiling {
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+		if c.ComponentConfig.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+	configz.InstallHandler(mux)
+	mux.Handle("/metrics", prometheus.Handler())
+
+	return serveFunc(mux, 0, stopCh)
+}

cmd/kube-controller-manager/app/autoscaling.go

@@ -38,7 +38,7 @@ func startHPAController(ctx ControllerContext) (bool, error) {
 		return false, nil
 	}
 
-	if ctx.Options.HorizontalPodAutoscalerUseRESTClients {
+	if ctx.ComponentConfig.HorizontalPodAutoscalerUseRESTClients {
 		// use the new-style clients if support for custom metrics is enabled
 		return startHPAControllerWithRESTClient(ctx)
 	}
@@ -88,7 +88,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 	replicaCalc := podautoscaler.NewReplicaCalculator(
 		metricsClient,
 		hpaClient.CoreV1(),
-		ctx.Options.HorizontalPodAutoscalerTolerance,
+		ctx.ComponentConfig.HorizontalPodAutoscalerTolerance,
 	)
 	go podautoscaler.NewHorizontalController(
 		hpaClientGoClient.CoreV1(),
@@ -97,9 +97,9 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 		restMapper,
 		replicaCalc,
 		ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
-		ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
-		ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
-		ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
 	).Run(ctx.Stop)
 	return true, nil
 }

cmd/kube-controller-manager/app/batch.go

@@ -36,7 +36,7 @@ func startJobController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.InformerFactory.Batch().V1().Jobs(),
 		ctx.ClientBuilder.ClientOrDie("job-controller"),
-	).Run(int(ctx.Options.ConcurrentJobSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentJobSyncs), ctx.Stop)
 	return true, nil
 }
 

cmd/kube-controller-manager/app/certificates.go

@@ -37,7 +37,7 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"}] {
 		return false, nil
 	}
-	if ctx.Options.ClusterSigningCertFile == "" || ctx.Options.ClusterSigningKeyFile == "" {
+	if ctx.ComponentConfig.ClusterSigningCertFile == "" || ctx.ComponentConfig.ClusterSigningKeyFile == "" {
 		return false, nil
 	}
 
@@ -52,15 +52,15 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	// bail out of startController without logging.
 	var keyFileExists, keyUsesDefault, certFileExists, certUsesDefault bool
 
-	_, err := os.Stat(ctx.Options.ClusterSigningCertFile)
+	_, err := os.Stat(ctx.ComponentConfig.ClusterSigningCertFile)
 	certFileExists = !os.IsNotExist(err)
 
-	certUsesDefault = (ctx.Options.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
+	certUsesDefault = (ctx.ComponentConfig.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
 
-	_, err = os.Stat(ctx.Options.ClusterSigningKeyFile)
+	_, err = os.Stat(ctx.ComponentConfig.ClusterSigningKeyFile)
 	keyFileExists = !os.IsNotExist(err)
 
-	keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
+	keyUsesDefault = (ctx.ComponentConfig.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
 
 	switch {
 	case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault):
@@ -84,9 +84,9 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	signer, err := signer.NewCSRSigningController(
 		c,
 		ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(),
-		ctx.Options.ClusterSigningCertFile,
-		ctx.Options.ClusterSigningKeyFile,
-		ctx.Options.ClusterSigningDuration.Duration,
+		ctx.ComponentConfig.ClusterSigningCertFile,
+		ctx.ComponentConfig.ClusterSigningKeyFile,
+		ctx.ComponentConfig.ClusterSigningDuration.Duration,
 	)
 	if err != nil {
 		return false, fmt.Errorf("failed to start certificate controller: %v", err)

cmd/kube-controller-manager/app/config/config.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"time"
+
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+)
+
+// ExtraConfig are part of Config, also can place your custom config here.
+type ExtraConfig struct {
+	NodeStatusUpdateFrequency time.Duration
+}
+
+// Config is the main context object for the controller manager.
+type Config struct {
+	Generic genericcontrollermanager.Config
+	Extra   ExtraConfig
+}
+
+type completedConfig struct {
+	Generic genericcontrollermanager.CompletedConfig
+	Extra   *ExtraConfig
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{
+		c.Generic.Complete(),
+		&c.Extra,
+	}
+
+	return &CompletedConfig{&cc}
+}

cmd/kube-controller-manager/app/extensions.go

@@ -43,7 +43,7 @@ func startDaemonSetController(ctx ControllerContext) (bool, error) {
 	if err != nil {
 		return true, fmt.Errorf("error creating DaemonSets controller: %v", err)
 	}
-	go dsc.Run(int(ctx.Options.ConcurrentDaemonSetSyncs), ctx.Stop)
+	go dsc.Run(int(ctx.ComponentConfig.ConcurrentDaemonSetSyncs), ctx.Stop)
 	return true, nil
 }
 
@@ -60,7 +60,7 @@ func startDeploymentController(ctx ControllerContext) (bool, error) {
 	if err != nil {
 		return true, fmt.Errorf("error creating Deployment controller: %v", err)
 	}
-	go dc.Run(int(ctx.Options.ConcurrentDeploymentSyncs), ctx.Stop)
+	go dc.Run(int(ctx.ComponentConfig.ConcurrentDeploymentSyncs), ctx.Stop)
 	return true, nil
 }
 
@@ -73,6 +73,6 @@ func startReplicaSetController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.ClientBuilder.ClientOrDie("replicaset-controller"),
 		replicaset.BurstReplicas,
-	).Run(int(ctx.Options.ConcurrentRSSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentRSSyncs), ctx.Stop)
 	return true, nil
 }

cmd/kube-controller-manager/app/options/options_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"net"
 	"reflect"
 	"sort"
 	"testing"
@@ -32,7 +33,7 @@ import (
 
 func TestAddFlags(t *testing.T) {
 	f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
-	s := NewCMServer()
+	s := NewKubeControllerManagerOptions()
 	s.AddFlags(f, []string{""}, []string{""})
 
 	args := []string{
@@ -107,13 +108,13 @@ func TestAddFlags(t *testing.T) {
 	f.Parse(args)
 	// Sort GCIgnoredResources because it's built from a map, which means the
 	// insertion order is random.
-	sort.Sort(sortedGCIgnoredResources(s.GCIgnoredResources))
+	sort.Sort(sortedGCIgnoredResources(s.Generic.ComponentConfig.GCIgnoredResources))
 
-	expected := &CMServer{
-		ControllerManagerServer: cmoptions.ControllerManagerServer{
-			KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-				Port:                                            10000,
-				Address:                                         "192.168.4.10",
+	expected := &KubeControllerManagerOptions{
+		Generic: cmoptions.GenericControllerManagerOptions{
+			ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
+				Port:                                            10252,     // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				Address:                                         "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
 				AllocateNodeCIDRs:                               true,
 				CloudConfigFile:                                 "/cloud-config",
 				CloudProvider:                                   "gce",
@@ -204,6 +205,11 @@ func TestAddFlags(t *testing.T) {
 				HorizontalPodAutoscalerUseRESTClients: true,
 				UseServiceAccountCredentials:          true,
 			},
+			InsecureServing: &cmoptions.InsecureServingOptions{
+				BindAddress: net.ParseIP("192.168.4.10"),
+				BindPort:    int(10000),
+				BindNetwork: "tcp",
+			},
 			Kubeconfig: "/kubeconfig",
 			Master:     "192.168.4.20",
 		},
@@ -211,7 +217,7 @@ func TestAddFlags(t *testing.T) {
 
 	// Sort GCIgnoredResources because it's built from a map, which means the
 	// insertion order is random.
-	sort.Sort(sortedGCIgnoredResources(expected.GCIgnoredResources))
+	sort.Sort(sortedGCIgnoredResources(expected.Generic.ComponentConfig.GCIgnoredResources))
 
 	if !reflect.DeepEqual(expected, s) {
 		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s))

cmd/kube-controller-manager/controller-manager.go

@@ -22,14 +22,13 @@ package main
 
 import (
 	goflag "flag"
+	"fmt"
 	"math/rand"
 	"os"
 	"time"
 
 	"github.com/spf13/pflag"
 
-	"fmt"
-
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app"

cmd/kubeadm/app/preflight/checks.go

@@ -530,7 +530,7 @@ func (eac ExtraArgsCheck) Check() (warnings, errors []error) {
 	}
 	if len(eac.ControllerManagerExtraArgs) > 0 {
 		flags := pflag.NewFlagSet("", pflag.ContinueOnError)
-		s := cmoptions.NewCMServer()
+		s := cmoptions.NewKubeControllerManagerOptions()
 		s.AddFlags(flags, []string{}, []string{})
 		warnings = append(warnings, argsCheck("kube-controller-manager", eac.ControllerManagerExtraArgs, flags)...)
 	}

pkg/master/ports/ports.go

@@ -26,12 +26,12 @@ const (
 	// SchedulerPort is the default port for the scheduler status server.
 	// May be overridden by a flag at startup.
 	SchedulerPort = 10251
-	// ControllerManagerPort is the default port for the controller manager status server.
+	// InsecureKubeControllerManagerPort is the default port for the controller manager status server.
 	// May be overridden by a flag at startup.
-	ControllerManagerPort = 10252
-	// CloudControllerManagerPort is the default port for the cloud controller manager server.
+	InsecureControllerManagerPort = 10252
+	// InsecureCloudControllerManagerPort is the default port for the cloud controller manager server.
 	// This value may be overridden by a flag at startup.
-	CloudControllerManagerPort = 10253
+	InsecureCloudControllerManagerPort = 10253
 	// KubeletReadOnlyPort exposes basic read-only services from the kubelet.
 	// May be overridden by a flag at startup.
 	// This is necessary for heapster to collect monitoring stats from the kubelet

pkg/registry/core/rest/storage_core.go

@@ -239,7 +239,7 @@ type componentStatusStorage struct {
 
 func (s componentStatusStorage) serversToValidate() map[string]*componentstatus.Server {
 	serversToValidate := map[string]*componentstatus.Server{
-		"controller-manager": {Addr: "127.0.0.1", Port: ports.ControllerManagerPort, Path: "/healthz"},
+		"controller-manager": {Addr: "127.0.0.1", Port: ports.InsecureKubeControllerManagerPort, Path: "/healthz"},
 		"scheduler":          {Addr: "127.0.0.1", Port: ports.SchedulerPort, Path: "/healthz"},
 	}
 

test/e2e/apps/daemon_restart.go

@@ -249,7 +249,7 @@ var _ = SIGDescribe("DaemonRestart [Disruptive]", func() {
 		// Requires master ssh access.
 		framework.SkipUnlessProviderIs("gce", "aws")
 		restarter := NewRestartConfig(
-			framework.GetMasterHost(), "kube-controller", ports.ControllerManagerPort, restartPollInterval, restartTimeout)
+			framework.GetMasterHost(), "kube-controller", ports.InsecureKubeControllerManagerPort, restartPollInterval, restartTimeout)
 		restarter.restart()
 
 		// The intent is to ensure the replication controller manager has observed and reported status of

test/e2e/framework/metrics/metrics_grabber.go

@@ -158,7 +158,7 @@ func (g *MetricsGrabber) GrabFromControllerManager() (ControllerManagerMetrics,
 	if !g.registeredMaster {
 		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering.")
 	}
-	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.ControllerManagerPort)
+	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
 	if err != nil {
 		return ControllerManagerMetrics{}, err
 	}

test/e2e/framework/util.go

@@ -3967,7 +3967,7 @@ func RestartControllerManager() error {
 }
 
 func WaitForControllerManagerUp() error {
-	cmd := "curl http://localhost:" + strconv.Itoa(ports.ControllerManagerPort) + "/healthz"
+	cmd := "curl http://localhost:" + strconv.Itoa(ports.InsecureKubeControllerManagerPort) + "/healthz"
 	for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(5 * time.Second) {
 		result, err := SSH(cmd, net.JoinHostPort(GetMasterHost(), sshPort), TestContext.Provider)
 		if err != nil || result.Code != 0 {

test/e2e/network/firewall.go

@@ -170,7 +170,7 @@ var _ = SIGDescribe("Firewall rule", func() {
 		nodeAddrs := framework.NodeAddresses(nodes, v1.NodeExternalIP)
 		Expect(len(nodeAddrs)).NotTo(BeZero())
 		masterAddr := framework.GetMasterAddress(cs)
-		flag, _ := framework.TestNotReachableHTTPTimeout(masterAddr, ports.ControllerManagerPort, framework.FirewallTestTcpTimeout)
+		flag, _ := framework.TestNotReachableHTTPTimeout(masterAddr, ports.InsecureKubeControllerManagerPort, framework.FirewallTestTcpTimeout)
 		Expect(flag).To(BeTrue())
 		flag, _ = framework.TestNotReachableHTTPTimeout(masterAddr, ports.SchedulerPort, framework.FirewallTestTcpTimeout)
 		Expect(flag).To(BeTrue())