split secure serving options

cmd/kube-apiserver/app/server.go

@@ -202,9 +202,9 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	// Default to the private server key for service account token signing
-	if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.TLSPrivateKeyFile != "" {
-		if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.TLSPrivateKeyFile) {
-			s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.TLSPrivateKeyFile}
+	if len(s.ServiceAccountKeyFiles) == 0 && s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile != "" {
+		if authenticator.IsValidServiceAccountKeyFile(s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile) {
+			s.ServiceAccountKeyFiles = []string{s.GenericServerRunOptions.SecureServingOptions.ServerCert.CertKey.KeyFile}
 		} else {
 			glog.Warning("No TLS key provided, service account token authentication disabled")
 		}
@@ -225,7 +225,7 @@ func Run(s *options.ServerRunOptions) error {
 		Anonymous:                   s.GenericServerRunOptions.AnonymousAuth,
 		AnyToken:                    s.GenericServerRunOptions.EnableAnyToken,
 		BasicAuthFile:               s.GenericServerRunOptions.BasicAuthFile,
-		ClientCAFile:                s.GenericServerRunOptions.ClientCAFile,
+		ClientCAFile:                s.GenericServerRunOptions.SecureServingOptions.ClientCA,
 		TokenAuthFile:               s.GenericServerRunOptions.TokenAuthFile,
 		OIDCIssuerURL:               s.GenericServerRunOptions.OIDCIssuerURL,
 		OIDCClientID:                s.GenericServerRunOptions.OIDCClientID,

examples/apiserver/apiserver.go

@@ -56,7 +56,7 @@ func newStorageFactory() genericapiserver.StorageFactory {
 }
 
 func NewServerRunOptions() *genericoptions.ServerRunOptions {
-	serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions()
+	serverOptions := genericoptions.NewServerRunOptions().WithEtcdOptions().WithSecureServingOptions()
 	serverOptions.InsecurePort = InsecurePort
 	return serverOptions
 }

federation/cmd/federation-apiserver/app/server.go

@@ -121,7 +121,7 @@ func Run(s *options.ServerRunOptions) error {
 		Anonymous:           s.GenericServerRunOptions.AnonymousAuth,
 		AnyToken:            s.GenericServerRunOptions.EnableAnyToken,
 		BasicAuthFile:       s.GenericServerRunOptions.BasicAuthFile,
-		ClientCAFile:        s.GenericServerRunOptions.ClientCAFile,
+		ClientCAFile:        s.GenericServerRunOptions.SecureServingOptions.ClientCA,
 		TokenAuthFile:       s.GenericServerRunOptions.TokenAuthFile,
 		OIDCIssuerURL:       s.GenericServerRunOptions.OIDCIssuerURL,
 		OIDCClientID:        s.GenericServerRunOptions.OIDCClientID,

pkg/genericapiserver/config.go

@@ -226,7 +226,6 @@ func NewConfig() *Config {
 	defaultOptions := options.NewServerRunOptions()
 	// unset fields that can be overridden to avoid setting values so that we won't end up with lingering values.
 	// TODO we probably want to run the defaults the other way.  A default here drives it in the CLI flags
-	defaultOptions.SecurePort = 0
 	defaultOptions.InsecurePort = 0
 	defaultOptions.AuditLogPath = ""
 	return config.ApplyOptions(defaultOptions)
@@ -243,28 +242,28 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
 		}
 	}
 
-	if options.SecurePort > 0 {
+	if options.SecureServingOptions != nil && options.SecureServingOptions.ServingOptions.BindPort > 0 {
 		secureServingInfo := &SecureServingInfo{
 			ServingInfo: ServingInfo{
-				BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)),
+				BindAddress: net.JoinHostPort(options.SecureServingOptions.ServingOptions.BindAddress.String(), strconv.Itoa(options.SecureServingOptions.ServingOptions.BindPort)),
 			},
 			ServerCert: GeneratableKeyCert{
 				CertKey: CertKey{
-					CertFile: options.TLSCertFile,
-					KeyFile:  options.TLSPrivateKeyFile,
+					CertFile: options.SecureServingOptions.ServerCert.CertKey.CertFile,
+					KeyFile:  options.SecureServingOptions.ServerCert.CertKey.KeyFile,
 				},
 			},
 			SNICerts: []NamedCertKey{},
-			ClientCA: options.ClientCAFile,
+			ClientCA: options.SecureServingOptions.ClientCA,
 		}
-		if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
+		if options.SecureServingOptions.ServerCert.CertKey.CertFile == "" && options.SecureServingOptions.ServerCert.CertKey.KeyFile == "" {
 			secureServingInfo.ServerCert.Generate = true
-			secureServingInfo.ServerCert.CertFile = path.Join(options.CertDirectory, "apiserver.crt")
-			secureServingInfo.ServerCert.KeyFile = path.Join(options.CertDirectory, "apiserver.key")
+			secureServingInfo.ServerCert.CertFile = path.Join(options.SecureServingOptions.ServerCert.CertDirectory, options.SecureServingOptions.ServerCert.PairName+".crt")
+			secureServingInfo.ServerCert.KeyFile = path.Join(options.SecureServingOptions.ServerCert.CertDirectory, options.SecureServingOptions.ServerCert.PairName+".key")
 		}
 
 		secureServingInfo.SNICerts = nil
-		for _, nkc := range options.SNICertKeys {
+		for _, nkc := range options.SecureServingOptions.SNICertKeys {
 			secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{
 				CertKey: CertKey{
 					KeyFile:  nkc.KeyFile,
@@ -275,7 +274,7 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
 		}
 
 		c.SecureServingInfo = secureServingInfo
-		c.ReadWritePort = options.SecurePort
+		c.ReadWritePort = options.SecureServingOptions.ServingOptions.BindPort
 	}
 
 	if options.InsecurePort > 0 {
@@ -488,8 +487,8 @@ func DefaultAndValidateRunOptions(options *options.ServerRunOptions) {
 	// If advertise-address is not specified, use bind-address. If bind-address
 	// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
 	// interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster)
-	if options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified() {
-		hostIP, err := utilnet.ChooseBindAddress(options.BindAddress)
+	if options.SecureServingOptions != nil && (options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified()) {
+		hostIP, err := utilnet.ChooseBindAddress(options.SecureServingOptions.ServingOptions.BindAddress)
 		if err != nil {
 			glog.Fatalf("Unable to find suitable network address.error='%v' . "+
 				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)

pkg/genericapiserver/options/server_run_options.go

@@ -54,7 +54,8 @@ var AuthorizationModeChoices = []string{ModeAlwaysAllow, ModeAlwaysDeny, ModeABA
 
 // ServerRunOptions contains the options while running a generic api server.
 type ServerRunOptions struct {
-	Etcd *EtcdOptions
+	Etcd                 *EtcdOptions
+	SecureServingOptions *SecureServingOptions
 
 	AdmissionControl           string
 	AdmissionControlConfigFile string
@@ -70,9 +71,6 @@ type ServerRunOptions struct {
 
 	AnonymousAuth                bool
 	BasicAuthFile                string
-	BindAddress                  net.IP
-	CertDirectory                string
-	ClientCAFile                 string
 	CloudConfigFile              string
 	CloudProvider                string
 	CorsAllowedOriginList        []string
@@ -107,7 +105,6 @@ type ServerRunOptions struct {
 	RequestHeaderClientCAFile    string
 	RequestHeaderAllowedNames    []string
 	RuntimeConfig                config.ConfigurationMap
-	SecurePort                   int
 	ServiceClusterIPRange        net.IPNet // TODO: make this a list
 	ServiceNodePortRange         utilnet.PortRange
 	StorageVersions              string
@@ -117,9 +114,6 @@ type ServerRunOptions struct {
 	DefaultStorageVersions string
 	TargetRAMMB            int
 	TLSCAFile              string
-	TLSCertFile            string
-	TLSPrivateKeyFile      string
-	SNICertKeys            []config.NamedCertKey
 	TokenAuthFile          string
 	EnableAnyToken         bool
 	WatchCacheSizes        []string
@@ -132,8 +126,6 @@ func NewServerRunOptions() *ServerRunOptions {
 		AuthorizationMode:                        "AlwaysAllow",
 		AuthorizationWebhookCacheAuthorizedTTL:   5 * time.Minute,
 		AuthorizationWebhookCacheUnauthorizedTTL: 30 * time.Second,
-		BindAddress:                              net.ParseIP("0.0.0.0"),
-		CertDirectory:                            "/var/run/kubernetes",
 		DefaultStorageMediaType:                  "application/json",
 		DefaultStorageVersions:                   registered.AllPreferredGroupVersions(),
 		DeleteCollectionWorkers:                  1,
@@ -149,7 +141,6 @@ func NewServerRunOptions() *ServerRunOptions {
 		MaxRequestsInFlight:                      400,
 		MinRequestTimeout:                        1800,
 		RuntimeConfig:                            make(config.ConfigurationMap),
-		SecurePort:                               6443,
 		ServiceNodePortRange:                     DefaultServiceNodePortRange,
 		StorageVersions:                          registered.AllPreferredGroupVersions(),
 	}
@@ -159,6 +150,10 @@ func (o *ServerRunOptions) WithEtcdOptions() *ServerRunOptions {
 	o.Etcd = NewDefaultEtcdOptions()
 	return o
 }
+func (o *ServerRunOptions) WithSecureServingOptions() *ServerRunOptions {
+	o.SecureServingOptions = NewDefaultSecureServingOptions()
+	return o
+}
 
 // StorageGroupsToEncodingVersion returns a map from group name to group version,
 // computed from s.StorageVersions flag.
@@ -225,15 +220,16 @@ func (s *ServerRunOptions) NewSelfClientConfig(token string) (*restclient.Config
 		Burst: 100,
 	}
 
-	// Use secure port if the TLSCAFile is specified
-	if s.SecurePort > 0 && len(s.TLSCAFile) > 0 {
-		host := s.BindAddress.String()
+	// Use secure port if the ServerCA is specified
+	if s.SecureServingOptions != nil && s.SecureServingOptions.ServingOptions.BindPort > 0 && len(s.SecureServingOptions.ServerCA) > 0 {
+		host := s.SecureServingOptions.ServingOptions.BindAddress.String()
 		if host == "0.0.0.0" {
 			host = "localhost"
 		}
-		clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.SecurePort))
-		clientConfig.CAFile = s.TLSCAFile
+		clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.SecureServingOptions.ServingOptions.BindPort))
+		clientConfig.CAFile = s.SecureServingOptions.ServerCA
 		clientConfig.BearerToken = token
+
 	} else if s.InsecurePort > 0 {
 		clientConfig.Host = net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
 	} else {
@@ -293,24 +289,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"If set, the file that will be used to admit requests to the secure port of the API server "+
 		"via http basic authentication.")
 
-	fs.IPVar(&s.BindAddress, "public-address-override", s.BindAddress,
-		"DEPRECATED: see --bind-address instead.")
-	fs.MarkDeprecated("public-address-override", "see --bind-address instead.")
-
-	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
-		"The IP address on which to listen for the --secure-port port. The "+
-		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
-		"clients. If blank, all interfaces will be used (0.0.0.0).")
-
-	fs.StringVar(&s.CertDirectory, "cert-dir", s.CertDirectory, ""+
-		"The directory where the TLS certs are located (by default /var/run/kubernetes). "+
-		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
-
-	fs.StringVar(&s.ClientCAFile, "client-ca-file", s.ClientCAFile, ""+
-		"If set, any request presenting a client certificate signed by one of "+
-		"the authorities in the client-ca-file is authenticated with an identity "+
-		"corresponding to the CommonName of the client certificate.")
-
 	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider,
 		"The provider for cloud services. Empty string for no provider.")
 
@@ -448,10 +426,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"apis/<groupVersion>/<resource> can be used to turn on/off specific resources. api/all and "+
 		"api/legacy are special keys to control all and legacy api versions respectively.")
 
-	fs.IntVar(&s.SecurePort, "secure-port", s.SecurePort, ""+
-		"The port on which to serve HTTPS with authentication and authorization. If 0, "+
-		"don't serve HTTPS at all.")
-
 	fs.IPNetVar(&s.ServiceClusterIPRange, "service-cluster-ip-range", s.ServiceClusterIPRange, ""+
 		"A CIDR notation IP range from which to assign service cluster IPs. This must not "+
 		"overlap with any IP ranges assigned to nodes for pods.")
@@ -481,28 +455,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"It defaults to a list of preferred versions of all registered groups, "+
 		"which is derived from the KUBE_API_VERSIONS environment variable.")
 
-	fs.StringVar(&s.TLSCAFile, "tls-ca-file", s.TLSCAFile, "If set, this "+
-		"certificate authority will used for secure access from Admission "+
-		"Controllers. This must be a valid PEM-encoded CA bundle.")
-
-	fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+
-		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
-		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
-		"--tls-private-key-file are not provided, a self-signed certificate and key "+
-		"are generated for the public address and saved to /var/run/kubernetes.")
-
-	fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile,
-		"File containing the default x509 private key matching --tls-cert-file.")
-
-	fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
-		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
-		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
-		"segments. If no domain patterns are provided, the names of the certificate are "+
-		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
-		"trump over extracted names. For multiple key/certificate pairs, use the "+
-		"--tls-sni-cert-key multiple times. "+
-		"Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
-
 	fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, ""+
 		"If set, the file that will be used to secure the secure port of the API server "+
 		"via token authentication.")

pkg/genericapiserver/options/serving_options.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/kubernetes/pkg/util/config"
+)
+
+type ServingOptions struct {
+	BindAddress net.IP
+	BindPort    int
+}
+
+type SecureServingOptions struct {
+	ServingOptions ServingOptions
+
+	// ServerCert is the TLS cert info for serving secure traffic
+	ServerCert GeneratableKeyCert
+	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
+	SNICertKeys []config.NamedCertKey
+	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
+	ClientCA string
+
+	// ServerCA is the certificate bundle for the signer of your serving certificate.  Used for building a loopback
+	// connection to the API server for admission.
+	ServerCA string
+}
+
+type CertKey struct {
+	// CertFile is a file containing a PEM-encoded certificate
+	CertFile string
+	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
+	KeyFile string
+}
+
+type GeneratableKeyCert struct {
+	CertKey CertKey
+
+	// CertDirectory is a directory that will contain the certificates.  If the cert and key aren't specifically set
+	// this will be used to derive a match with the "pair-name"
+	CertDirectory string
+	// PairName is the name which will be used with CertDirectory to make a cert and key names
+	// It becomes CertDirector/PairName.crt and CertDirector/PairName.key
+	PairName string
+}
+
+func NewDefaultSecureServingOptions() *SecureServingOptions {
+	return &SecureServingOptions{
+		ServingOptions: ServingOptions{
+			BindAddress: net.ParseIP("0.0.0.0"),
+			BindPort:    6443,
+		},
+		ServerCert: GeneratableKeyCert{
+			PairName:      "apiserver",
+			CertDirectory: "/var/run/kubernetes",
+		},
+	}
+}
+
+func (s *SecureServingOptions) Validate() []error {
+	errors := []error{}
+	if s == nil {
+		return errors
+	}
+
+	errors = append(errors, s.ServingOptions.Validate("secure-port")...)
+	return errors
+}
+
+func (s ServingOptions) Validate(portArg string) []error {
+	errors := []error{}
+
+	if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("--%v %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", portArg, s.BindPort))
+	}
+
+	return errors
+}
+
+func (s *SecureServingOptions) AddSecureServingFlags(fs *pflag.FlagSet) {
+	fs.IPVar(&s.ServingOptions.BindAddress, "bind-address", s.ServingOptions.BindAddress, ""+
+		"The IP address on which to listen for the --secure-port port. The "+
+		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
+		"clients. If blank, all interfaces will be used (0.0.0.0).")
+
+	fs.IntVar(&s.ServingOptions.BindPort, "secure-port", s.ServingOptions.BindPort, ""+
+		"The port on which to serve HTTPS with authentication and authorization. If 0, "+
+		"don't serve HTTPS at all.")
+
+	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
+		"The directory where the TLS certs are located (by default /var/run/kubernetes). "+
+		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
+
+	fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
+		"--tls-private-key-file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to /var/run/kubernetes.")
+
+	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
+		"File containing the default x509 private key matching --tls-cert-file.")
+
+	fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
+		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
+		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
+		"segments. If no domain patterns are provided, the names of the certificate are "+
+		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
+		"trump over extracted names. For multiple key/certificate pairs, use the "+
+		"--tls-sni-cert-key multiple times. "+
+		"Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
+
+	fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
+		"If set, any request presenting a client certificate signed by one of "+
+		"the authorities in the client-ca-file is authenticated with an identity "+
+		"corresponding to the CommonName of the client certificate.")
+
+	fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+
+		"certificate authority will used for secure access from Admission "+
+		"Controllers. This must be a valid PEM-encoded CA bundle.")
+
+}
+
+func (s *SecureServingOptions) AddDeprecatedSecureServingFlags(fs *pflag.FlagSet) {
+	fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress,
+		"DEPRECATED: see --bind-address instead.")
+	fs.MarkDeprecated("public-address-override", "see --bind-address instead.")
+
+}

pkg/genericapiserver/validation/universal_validation.go

@@ -51,19 +51,17 @@ func verifyServiceNodePort(options *options.ServerRunOptions) []error {
 
 func verifySecureAndInsecurePort(options *options.ServerRunOptions) []error {
 	errors := []error{}
-	if options.SecurePort < 0 || options.SecurePort > 65535 {
-		errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", options.SecurePort))
-	}
+	errors = append(errors, options.SecureServingOptions.Validate()...)
 
 	if options.InsecurePort < 0 || options.InsecurePort > 65535 {
 		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 65535, inclusive. 0 for turning off insecure port.", options.InsecurePort))
 	}
 
-	if options.SecurePort == 0 && options.InsecurePort == 0 {
+	if (options.SecureServingOptions == nil || options.SecureServingOptions.ServingOptions.BindPort == 0) && options.InsecurePort == 0 {
 		glog.Fatalf("--secure-port and --insecure-port cannot be turned off at the same time.")
 	}
 
-	if options.SecurePort == options.InsecurePort {
+	if options.SecureServingOptions != nil && options.SecureServingOptions.ServingOptions.BindPort == options.InsecurePort {
 		errors = append(errors, fmt.Errorf("--secure-port and --insecure-port cannot use the same port."))
 	}
 	return errors

test/integration/examples/apiserver_test.go

@@ -64,7 +64,7 @@ func TestRunSecureServer(t *testing.T) {
 	go func() {
 		options := apiserver.NewServerRunOptions()
 		options.InsecurePort = 0
-		options.SecurePort = apiserver.SecurePort
+		options.SecureServingOptions.ServingOptions.BindPort = apiserver.SecurePort
 		if err := apiserver.Run(options, stopCh); err != nil {
 			t.Fatalf("Error in bringing up the server: %v", err)
 		}