genericapiserver: move MasterCount and service options into master

cmd/kube-apiserver/app/options/options.go

@@ -18,6 +18,7 @@ limitations under the License.
 package options
 
 import (
+	"net"
 	"time"
 
 	"k8s.io/kubernetes/pkg/api"
@@ -25,10 +26,14 @@ import (
 	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
 	"k8s.io/kubernetes/pkg/master/ports"
+	utilnet "k8s.io/kubernetes/pkg/util/net"
 
 	"github.com/spf13/pflag"
 )
 
+// DefaultServiceNodePortRange is the default port range for NodePort services.
+var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768}
+
 // ServerRunOptions runs a kubernetes api server.
 type ServerRunOptions struct {
 	GenericServerRunOptions *genericoptions.ServerRunOptions
@@ -38,12 +43,16 @@ type ServerRunOptions struct {
 	Authentication          *genericoptions.BuiltInAuthenticationOptions
 	Authorization           *genericoptions.BuiltInAuthorizationOptions
 
-	AllowPrivileged          bool
-	EventTTL                 time.Duration
-	KubeletConfig            kubeletclient.KubeletClientConfig
-	MaxConnectionBytesPerSec int64
-	SSHKeyfile               string
-	SSHUser                  string
+	AllowPrivileged           bool
+	EventTTL                  time.Duration
+	KubeletConfig             kubeletclient.KubeletClientConfig
+	KubernetesServiceNodePort int
+	MasterCount               int
+	MaxConnectionBytesPerSec  int64
+	ServiceClusterIPRange     net.IPNet // TODO: make this a list
+	ServiceNodePortRange      utilnet.PortRange
+	SSHKeyfile                string
+	SSHUser                   string
 }
 
 // NewServerRunOptions creates a new ServerRunOptions object with default parameters
@@ -56,7 +65,8 @@ func NewServerRunOptions() *ServerRunOptions {
 		Authentication:  genericoptions.NewBuiltInAuthenticationOptions().WithAll(),
 		Authorization:   genericoptions.NewBuiltInAuthorizationOptions(),
 
-		EventTTL: 1 * time.Hour,
+		EventTTL:    1 * time.Hour,
+		MasterCount: 1,
 		KubeletConfig: kubeletclient.KubeletClientConfig{
 			Port: ports.KubeletPort,
 			PreferredAddressTypes: []string{
@@ -68,6 +78,7 @@ func NewServerRunOptions() *ServerRunOptions {
 			EnableHttps: true,
 			HTTPTimeout: time.Duration(5) * time.Second,
 		},
+		ServiceNodePortRange: DefaultServiceNodePortRange,
 	}
 	return &s
 }
@@ -104,6 +115,30 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"If non-zero, throttle each user connection to this number of bytes/sec. "+
 		"Currently only applies to long-running requests.")
 
+	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
+		"The number of apiservers running in the cluster.")
+
+	// See #14282 for details on how to test/try this option out.
+	// TODO: remove this comment once this option is tested in CI.
+	fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+
+		"If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+
+		"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
+		"service will be of type ClusterIP.")
+
+	fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
+		"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
+		"overlap with any IP ranges assigned to nodes for pods.")
+
+	fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange,
+		"DEPRECATED: see --service-cluster-ip-range instead.")
+	fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead")
+
+	fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+
+		"A port range to reserve for services with NodePort visibility. "+
+		"Example: '30000-32767'. Inclusive at both ends of the range.")
+	fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead")
+	fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead")
+
 	// Kubelet related flags:
 	fs.BoolVar(&s.KubeletConfig.EnableHttps, "kubelet-https", s.KubeletConfig.EnableHttps,
 		"Use https for kubelet connections.")

pkg/genericapiserver/validation/universal_validation.go → cmd/kube-apiserver/app/options/validation.go

@@ -14,18 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package validation
+package options
 
 import (
 	"fmt"
 
-	"github.com/golang/glog"
-	"k8s.io/kubernetes/pkg/genericapiserver/options"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
 )
 
 // TODO: Longer term we should read this from some config store, rather than a flag.
-func verifyClusterIPFlags(options *options.ServerRunOptions) []error {
+func verifyClusterIPFlags(options *ServerRunOptions) []error {
 	errors := []error{}
 	if options.ServiceClusterIPRange.IP == nil {
 		errors = append(errors, fmt.Errorf("No --service-cluster-ip-range specified"))
@@ -37,7 +35,7 @@ func verifyClusterIPFlags(options *options.ServerRunOptions) []error {
 	return errors
 }
 
-func verifyServiceNodePort(options *options.ServerRunOptions) []error {
+func verifyServiceNodePort(options *ServerRunOptions) []error {
 	errors := []error{}
 	if options.KubernetesServiceNodePort < 0 || options.KubernetesServiceNodePort > 65535 {
 		errors = append(errors, fmt.Errorf("--kubernetes-service-node-port %v must be between 0 and 65535, inclusive. If 0, the Kubernetes master service will be of type ClusterIP.", options.KubernetesServiceNodePort))
@@ -49,7 +47,7 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error {
 	return errors
 }
 
-func ValidateRunOptions(options *options.ServerRunOptions) {
+func ValidateRunOptions(options *ServerRunOptions) error {
 	errors := []error{}
 	if errs := verifyClusterIPFlags(options); len(errs) > 0 {
 		errors = append(errors, errs...)
@@ -58,6 +56,7 @@ func ValidateRunOptions(options *options.ServerRunOptions) {
 		errors = append(errors, errs...)
 	}
 	if err := utilerrors.NewAggregate(errors); err != nil {
-		glog.Fatalf("Validate server run options failed: %v", err)
+		return fmt.Errorf("validate server run options failed: %v", err)
 	}
+	return nil
 }

cmd/kube-apiserver/app/server.go

@@ -88,7 +88,7 @@ func Run(s *options.ServerRunOptions) error {
 		return err
 	}
 
-	serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
+	serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
 	if err != nil {
 		return fmt.Errorf("error determining service IP ranges: %v", err)
 	}
@@ -97,8 +97,14 @@ func Run(s *options.ServerRunOptions) error {
 		return fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
 
+	// TODO(sttts): change signature of DefaultAndValidateRunOptions to aggregate errors
 	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
 
+	// TODO(sttts): move all defaulting and validation above into cmd/kube-apiserver/app/options.DefaultAndValidateRunOptions()
+	if err != options.ValidateRunOptions(s) {
+		return err
+	}
+
 	genericConfig := genericapiserver.NewConfig(). // create the new config
 							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
 							ApplyInsecureServingOptions(s.InsecureServing)
@@ -313,10 +319,10 @@ func Run(s *options.ServerRunOptions) error {
 		APIServerServiceIP:   apiServerServiceIP,
 		APIServerServicePort: 443,
 
-		ServiceNodePortRange:      s.GenericServerRunOptions.ServiceNodePortRange,
-		KubernetesServiceNodePort: s.GenericServerRunOptions.KubernetesServiceNodePort,
+		ServiceNodePortRange:      s.ServiceNodePortRange,
+		KubernetesServiceNodePort: s.KubernetesServiceNodePort,
 
-		MasterCount: s.GenericServerRunOptions.MasterCount,
+		MasterCount: s.MasterCount,
 	}
 
 	if s.GenericServerRunOptions.EnableWatchCache {

examples/apiserver/apiserver.go

@@ -80,12 +80,10 @@ func NewServerRunOptions() *ServerRunOptions {
 }
 
 func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
-	// Set ServiceClusterIPRange
-	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
-	serverOptions.GenericServerRunOptions.ServiceClusterIPRange = *serviceClusterIPRange
 	serverOptions.Etcd.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"}
 
-	genericvalidation.ValidateRunOptions(serverOptions.GenericServerRunOptions)
+	// TODO(sttts): unify signature of DefaultAndValidateRunOptions with the others
+	genericapiserver.DefaultAndValidateRunOptions(serverOptions.GenericServerRunOptions)
 	if errs := serverOptions.Etcd.Validate(); len(errs) > 0 {
 		return utilerrors.NewAggregate(errs)
 	}

hack/update-federation-openapi-spec.sh

@@ -61,8 +61,7 @@ kube::log::status "Starting federation-apiserver"
   --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}" \
   --advertise-address="10.10.10.10" \
   --cert-dir="${TMP_DIR}/certs" \
-  --token-auth-file=$TMP_DIR/tokenauth.csv \
-  --service-cluster-ip-range="10.0.0.0/24" >/tmp/openapi-federation-api-server.log 2>&1 &
+  --token-auth-file=$TMP_DIR/tokenauth.csv >/tmp/openapi-federation-api-server.log 2>&1 &
 APISERVER_PID=$!
 kube::util::wait_for_url "${API_HOST}:${API_PORT}/" "apiserver: "
 

pkg/genericapiserver/config.go

@@ -59,7 +59,6 @@ import (
 	openapicommon "k8s.io/kubernetes/pkg/genericapiserver/openapi/common"
 	"k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/genericapiserver/routes"
-	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
 	"k8s.io/kubernetes/pkg/healthz"
 	"k8s.io/kubernetes/pkg/runtime"
 	certutil "k8s.io/kubernetes/pkg/util/cert"
@@ -631,8 +630,6 @@ func (s *GenericAPIServer) installAPI(c *Config) {
 }
 
 func DefaultAndValidateRunOptions(options *options.ServerRunOptions) {
-	genericvalidation.ValidateRunOptions(options)
-
 	glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress)
 
 	// Set default value for ExternalAddress if not specified.

pkg/genericapiserver/options/server_run_options.go

@@ -26,13 +26,10 @@ import (
 	"k8s.io/kubernetes/pkg/apimachinery/registered"
 	"k8s.io/kubernetes/pkg/runtime/schema"
 	"k8s.io/kubernetes/pkg/util/config"
-	utilnet "k8s.io/kubernetes/pkg/util/net"
 
 	"github.com/spf13/pflag"
 )
 
-var DefaultServiceNodePortRange = utilnet.PortRange{Base: 30000, Size: 2768}
-
 // ServerRunOptions contains the options while running a generic api server.
 type ServerRunOptions struct {
 	AdmissionControl           string
@@ -54,14 +51,10 @@ type ServerRunOptions struct {
 	EnableSwaggerUI             bool
 	EnableWatchCache            bool
 	ExternalHost                string
-	KubernetesServiceNodePort   int
-	MasterCount                 int
 	MaxRequestsInFlight         int
 	MaxMutatingRequestsInFlight int
 	MinRequestTimeout           int
 	RuntimeConfig               config.ConfigurationMap
-	ServiceClusterIPRange       net.IPNet // TODO: make this a list
-	ServiceNodePortRange        utilnet.PortRange
 	StorageVersions             string
 	// The default values for StorageVersions. StorageVersions overrides
 	// these; you can change this if you want to change the defaults (e.g.,
@@ -81,12 +74,10 @@ func NewServerRunOptions() *ServerRunOptions {
 		EnableProfiling:             true,
 		EnableContentionProfiling:   false,
 		EnableWatchCache:            true,
-		MasterCount:                 1,
 		MaxRequestsInFlight:         400,
 		MaxMutatingRequestsInFlight: 200,
 		MinRequestTimeout:           1800,
 		RuntimeConfig:               make(config.ConfigurationMap),
-		ServiceNodePortRange:        DefaultServiceNodePortRange,
 		StorageVersions:             registered.AllPreferredGroupVersions(),
 	}
 }
@@ -227,22 +218,12 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
 		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs).")
 
-	// See #14282 for details on how to test/try this option out.
-	// TODO: remove this comment once this option is tested in CI.
-	fs.IntVar(&s.KubernetesServiceNodePort, "kubernetes-service-node-port", s.KubernetesServiceNodePort, ""+
-		"If non-zero, the Kubernetes master service (which apiserver creates/maintains) will be "+
-		"of type NodePort, using this as the value of the port. If zero, the Kubernetes master "+
-		"service will be of type ClusterIP.")
-
 	// TODO: remove post-1.6
 	fs.String("long-running-request-regexp", "", ""+
 		"A regular expression matching long running requests which should "+
 		"be excluded from maximum inflight request handling.")
 	fs.MarkDeprecated("long-running-request-regexp", "regular expression matching of long-running requests is no longer supported")
 
-	fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
-		"The number of apiservers running in the cluster.")
-
 	deprecatedMasterServiceNamespace := api.NamespaceDefault
 	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
 		"DEPRECATED: the namespace from which the kubernetes master services should be injected into pods.")
@@ -267,20 +248,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and "+
 		"api/legacy are special keys to control all and legacy api versions respectively.")
 
-	fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
-		"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
-		"overlap with any IP ranges assigned to nodes for pods.")
-
-	fs.IPNetVar(&s.ServiceClusterIPRange, "portal-net", s.ServiceClusterIPRange,
-		"DEPRECATED: see --service-cluster-ip-range instead.")
-	fs.MarkDeprecated("portal-net", "see --service-cluster-ip-range instead")
-
-	fs.Var(&s.ServiceNodePortRange, "service-node-port-range", ""+
-		"A port range to reserve for services with NodePort visibility. "+
-		"Example: '30000-32767'. Inclusive at both ends of the range.")
-	fs.Var(&s.ServiceNodePortRange, "service-node-ports", "DEPRECATED: see --service-node-port-range instead")
-	fs.MarkDeprecated("service-node-ports", "see --service-node-port-range instead")
-
 	deprecatedStorageVersion := ""
 	fs.StringVar(&deprecatedStorageVersion, "storage-version", deprecatedStorageVersion,
 		"DEPRECATED: the version to store the legacy v1 resources with. Defaults to server preferred.")

pkg/master/master.go

@@ -24,6 +24,7 @@ import (
 	"strconv"
 	"time"
 
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api"
 	apiv1 "k8s.io/kubernetes/pkg/api/v1"
 	appsapi "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
@@ -39,7 +40,6 @@ import (
 	corev1client "k8s.io/kubernetes/pkg/client/clientset_generated/clientset/typed/core/v1"
 	coreclient "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/internalversion"
 	"k8s.io/kubernetes/pkg/genericapiserver"
-	"k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/healthz"
 	kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
 	"k8s.io/kubernetes/pkg/master/thirdparty"
@@ -116,7 +116,6 @@ type Config struct {
 	// Port names should align with ports defined in ExtraServicePorts
 	ExtraEndpointPorts []api.EndpointPort
 	// If non-zero, the "kubernetes" services uses this port as NodePort.
-	// TODO(sttts): move into master
 	KubernetesServiceNodePort int
 
 	// Number of masters running; all masters must be started with the

test/e2e_node/services/apiserver.go

@@ -46,7 +46,7 @@ func (a *APIServer) Start() error {
 	if err != nil {
 		return err
 	}
-	config.GenericServerRunOptions.ServiceClusterIPRange = *ipnet
+	config.ServiceClusterIPRange = *ipnet
 	config.AllowPrivileged = true
 	errCh := make(chan error)
 	go func() {

test/integration/federation/server_test.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"testing"
 	"time"
@@ -48,8 +47,6 @@ func TestRun(t *testing.T) {
 	s := options.NewServerRunOptions()
 	s.SecureServing.ServingOptions.BindPort = securePort
 	s.InsecureServing.BindPort = insecurePort
-	_, ipNet, _ := net.ParseCIDR("10.10.10.0/24")
-	s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet
 	s.Etcd.StorageConfig.ServerList = []string{"http://localhost:2379"}
 	go func() {
 		if err := app.Run(s); err != nil {