Stratify certificate loading and self-sign cert generation

cmd/kube-apiserver/app/server.go

@@ -21,6 +21,7 @@ package app
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/http"
 	"net/url"
@@ -49,7 +50,6 @@ import (
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
-	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/master"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	"k8s.io/kubernetes/pkg/runtime/schema"
@@ -85,20 +85,25 @@ func Run(s *options.ServerRunOptions) error {
 		return err
 	}
 
-	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
-	genericConfig := genericapiserver.NewConfig(). // create the new config
-							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
-							ApplySecureServingOptions(s.SecureServing).
-							ApplyInsecureServingOptions(s.InsecureServing).
-							ApplyAuthenticationOptions(s.Authentication).
-							ApplyRBACSuperUser(s.Authorization.RBACSuperUser)
-
 	serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.GenericServerRunOptions.ServiceClusterIPRange)
 	if err != nil {
-		glog.Fatalf("Error determining service IP ranges: %v", err)
+		return fmt.Errorf("error determining service IP ranges: %v", err)
+	}
+
+	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil {
+		return fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
-	if err := genericConfig.MaybeGenerateServingCerts(apiServerServiceIP); err != nil {
-		glog.Fatalf("Failed to generate service certificate: %v", err)
+
+	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
+
+	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+								ApplyInsecureServingOptions(s.InsecureServing).
+								ApplyAuthenticationOptions(s.Authentication).
+								ApplyRBACSuperUser(s.Authorization.RBACSuperUser).
+								ApplySecureServingOptions(s.SecureServing)
+	if err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
 	}
 
 	capabilities.Initialize(capabilities.Capabilities{
@@ -232,7 +237,7 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	privilegedLoopbackToken := uuid.NewRandom().String()
-	selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken)
+	selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
 	if err != nil {
 		glog.Fatalf("Failed to create clientset: %v", err)
 	}

examples/apiserver/apiserver.go

@@ -36,6 +36,8 @@ import (
 
 	// Install the testgroup API
 	_ "k8s.io/kubernetes/cmd/libs/go2idl/client-gen/test_apis/testgroup/install"
+
+	"github.com/golang/glog"
 )
 
 const (
@@ -93,20 +95,21 @@ func (serverOptions *ServerRunOptions) Run(stopCh <-chan struct{}) error {
 	if errs := serverOptions.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
 		return utilerrors.NewAggregate(errs)
 	}
+	if err := serverOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(serverOptions.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
+		glog.Fatalf("Error creating self-signed certificates: %v", err)
+	}
 
-	config := genericapiserver.NewConfig().
+	config, err := genericapiserver.NewConfig().
 		ApplyOptions(serverOptions.GenericServerRunOptions).
-		ApplySecureServingOptions(serverOptions.SecureServing).
 		ApplyInsecureServingOptions(serverOptions.InsecureServing).
 		ApplyAuthenticationOptions(serverOptions.Authentication).
-		Complete()
-	if err := config.MaybeGenerateServingCerts(); err != nil {
-		// this wasn't treated as fatal for this process before
-		fmt.Printf("Error creating cert: %v", err)
+		ApplySecureServingOptions(serverOptions.SecureServing)
+	if err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
 	}
 
 	config.Authorizer = authorizer.NewAlwaysAllowAuthorizer()
-	s, err := config.New()
+	s, err := config.Complete().New()
 	if err != nil {
 		return fmt.Errorf("Error in bringing up the server: %v", err)
 	}

federation/cmd/federation-apiserver/app/server.go

@@ -20,6 +20,7 @@ limitations under the License.
 package app
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
@@ -37,7 +38,6 @@ import (
 	"k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/genericapiserver"
 	"k8s.io/kubernetes/pkg/genericapiserver/authorizer"
-	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	"k8s.io/kubernetes/pkg/registry/generic"
 	genericregistry "k8s.io/kubernetes/pkg/registry/generic/registry"
@@ -73,16 +73,19 @@ func Run(s *options.ServerRunOptions) error {
 		return err
 	}
 
-	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
-	genericConfig := genericapiserver.NewConfig(). // create the new config
-							ApplyOptions(s.GenericServerRunOptions). // apply the options selected
-							ApplySecureServingOptions(s.SecureServing).
-							ApplyInsecureServingOptions(s.InsecureServing).
-							ApplyAuthenticationOptions(s.Authentication).
-							ApplyRBACSuperUser(s.Authorization.RBACSuperUser)
+	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String()); err != nil {
+		return fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
 
-	if err := genericConfig.MaybeGenerateServingCerts(); err != nil {
-		glog.Fatalf("Failed to generate service certificate: %v", err)
+	genericapiserver.DefaultAndValidateRunOptions(s.GenericServerRunOptions)
+	genericConfig, err := genericapiserver.NewConfig(). // create the new config
+								ApplyOptions(s.GenericServerRunOptions). // apply the options selected
+								ApplyInsecureServingOptions(s.InsecureServing).
+								ApplyAuthenticationOptions(s.Authentication).
+								ApplyRBACSuperUser(s.Authorization.RBACSuperUser).
+								ApplySecureServingOptions(s.SecureServing)
+	if err != nil {
+		return fmt.Errorf("failed to configure https: %s", err)
 	}
 
 	// TODO: register cluster federation resources here.
@@ -130,7 +133,7 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	privilegedLoopbackToken := uuid.NewRandom().String()
-	selfClientConfig, err := genericoptions.NewSelfClientConfig(s.SecureServing, s.InsecureServing, privilegedLoopbackToken)
+	selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
 	if err != nil {
 		glog.Fatalf("Failed to create clientset: %v", err)
 	}

pkg/genericapiserver/config.go

@@ -17,12 +17,14 @@ limitations under the License.
 package genericapiserver
 
 import (
+	"crypto/tls"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"path"
 	"regexp"
 	goruntime "runtime"
 	"sort"
@@ -58,7 +60,6 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver/routes"
 	genericvalidation "k8s.io/kubernetes/pkg/genericapiserver/validation"
 	"k8s.io/kubernetes/pkg/runtime"
-	certutil "k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/pkg/util/sets"
 	"k8s.io/kubernetes/pkg/version"
 	authenticatorunion "k8s.io/kubernetes/plugin/pkg/auth/authenticator/request/union"
@@ -169,33 +170,19 @@ type ServingInfo struct {
 type SecureServingInfo struct {
 	ServingInfo
 
-	// ServerCert is the TLS cert info for serving secure traffic
-	ServerCert GeneratableKeyCert
-	// SNICerts are named CertKeys for serving secure traffic with SNI support.
-	SNICerts []NamedCertKey
-	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
-	ClientCA string
-}
-
-type CertKey struct {
-	// CertFile is a file containing a PEM-encoded certificate
-	CertFile string
-	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
-	KeyFile string
-}
+	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
+	// allowed to be in SNICerts.
+	Cert *tls.Certificate
 
-type NamedCertKey struct {
-	CertKey
+	// CACert is an optional certificate authority used for the loopback connection of the Admission controllers.
+	// If this is nil, the certificate authority is extracted from Cert or a matching SNI certificate.
+	CACert *tls.Certificate
 
-	// Names is a list of domain patterns: fully qualified domain names, possibly prefixed with
-	// wildcard segments.
-	Names []string
-}
+	// SNICerts are the TLS certificates by name used for SNI.
+	SNICerts map[string]*tls.Certificate
 
-type GeneratableKeyCert struct {
-	CertKey
-	// Generate indicates that the cert/key pair should be generated if its not present.
-	Generate bool
+	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
+	ClientCA string
 }
 
 // NewConfig returns a Config struct with the default values
@@ -238,45 +225,69 @@ func NewConfig() *Config {
 	return config.ApplyOptions(defaultOptions)
 }
 
-func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) *Config {
+func (c *Config) ApplySecureServingOptions(secureServing *options.SecureServingOptions) (*Config, error) {
 	if secureServing == nil || secureServing.ServingOptions.BindPort <= 0 {
-		return c
+		return c, nil
 	}
 
 	secureServingInfo := &SecureServingInfo{
 		ServingInfo: ServingInfo{
 			BindAddress: net.JoinHostPort(secureServing.ServingOptions.BindAddress.String(), strconv.Itoa(secureServing.ServingOptions.BindPort)),
 		},
-		ServerCert: GeneratableKeyCert{
-			CertKey: CertKey{
-				CertFile: secureServing.ServerCert.CertKey.CertFile,
-				KeyFile:  secureServing.ServerCert.CertKey.KeyFile,
-			},
-		},
-		SNICerts: []NamedCertKey{},
 		ClientCA: secureServing.ClientCA,
 	}
-	if secureServing.ServerCert.CertKey.CertFile == "" && secureServing.ServerCert.CertKey.KeyFile == "" {
-		secureServingInfo.ServerCert.Generate = true
-		secureServingInfo.ServerCert.CertFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".crt")
-		secureServingInfo.ServerCert.KeyFile = path.Join(secureServing.ServerCert.CertDirectory, secureServing.ServerCert.PairName+".key")
+
+	serverCertFile, serverKeyFile := secureServing.ServerCert.CertKey.CertFile, secureServing.ServerCert.CertKey.KeyFile
+
+	// load main cert
+	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
+		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("unable to load server certificate: %v", err)
+		}
+		secureServingInfo.Cert = &tlsCert
 	}
 
-	secureServingInfo.SNICerts = nil
-	for _, nkc := range secureServing.SNICertKeys {
-		secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{
-			CertKey: CertKey{
-				KeyFile:  nkc.KeyFile,
-				CertFile: nkc.CertFile,
-			},
-			Names: nkc.Names,
+	// optionally load CA cert
+	if len(secureServing.ServerCert.CACertFile) != 0 {
+		pemData, err := ioutil.ReadFile(secureServing.ServerCert.CACertFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read certificate authority from %q: %v", secureServing.ServerCert.CACertFile, err)
+		}
+		block, pemData := pem.Decode(pemData)
+		if block == nil {
+			return nil, fmt.Errorf("no certificate found in certificate authority file %q", secureServing.ServerCert.CACertFile)
+		}
+		if block.Type != "CERTIFICATE" {
+			return nil, fmt.Errorf("expected CERTIFICATE block in certiticate authority file %q, found: %s", secureServing.ServerCert.CACertFile, block.Type)
+		}
+		secureServingInfo.CACert = &tls.Certificate{
+			Certificate: [][]byte{block.Bytes},
+		}
+	}
+
+	// load SNI certs
+	namedTlsCerts := make([]namedTlsCert, 0, len(secureServing.SNICertKeys))
+	for _, nck := range secureServing.SNICertKeys {
+		tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile)
+		namedTlsCerts = append(namedTlsCerts, namedTlsCert{
+			tlsCert: tlsCert,
+			names:   nck.Names,
 		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to load SNI cert and key: %v", err)
+		}
+	}
+	var err error
+	secureServingInfo.SNICerts, err = getNamedCertificateMap(namedTlsCerts)
+	if err != nil {
+		return nil, err
 	}
 
 	c.SecureServingInfo = secureServingInfo
 	c.ReadWritePort = secureServing.ServingOptions.BindPort
 
-	return c
+	return c, nil
 }
 
 func (c *Config) ApplyInsecureServingOptions(insecureServing *options.ServingOptions) *Config {
@@ -456,38 +467,6 @@ func (c completedConfig) New() (*GenericAPIServer, error) {
 	return s, nil
 }
 
-// MaybeGenerateServingCerts generates serving certificates if requested and needed.
-func (c *Config) MaybeGenerateServingCerts(alternateIPs ...net.IP) error {
-	// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
-	// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
-	if c.SecureServingInfo != nil && c.SecureServingInfo.ServerCert.Generate {
-		canReadCertAndKey, err := certutil.CanReadCertAndKey(c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
-		if err != nil {
-			return err
-		}
-		if canReadCertAndKey {
-			return nil
-		}
-		// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
-		alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
-
-		if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil {
-			return fmt.Errorf("unable to generate self signed cert: %v", err)
-		} else {
-			if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil {
-				return err
-			}
-
-			if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil {
-				return err
-			}
-			glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
-		}
-	}
-
-	return nil
-}
-
 func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) (secure, insecure http.Handler) {
 	generic := func(handler http.Handler) http.Handler {
 		handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")

pkg/genericapiserver/config_selfclient.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package genericapiserver
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"net"
+
+	"k8s.io/kubernetes/pkg/client/restclient"
+)
+
+// NewSelfClientConfig returns a clientconfig which can be used to talk to this apiserver.
+func NewSelfClientConfig(secureServingInfo *SecureServingInfo, insecureServingInfo *ServingInfo, token string) (*restclient.Config, error) {
+	if cfg, err := secureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil {
+		return cfg, err
+	}
+	if cfg, err := insecureServingInfo.NewSelfClientConfig(token); err != nil || cfg != nil {
+		return cfg, err
+	}
+
+	return nil, errors.New("Unable to set url for apiserver local client")
+}
+
+func (s *SecureServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) {
+	if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) {
+		return nil, nil
+	}
+
+	host, port, err := net.SplitHostPort(s.ServingInfo.BindAddress)
+	if err != nil {
+		// should never happen
+		return nil, fmt.Errorf("invalid secure bind address: %q", s.ServingInfo.BindAddress)
+	}
+	if host == "0.0.0.0" {
+		// compare MaybeDefaultWithSelfSignedCerts which adds "localhost" to the cert as alternateDNS
+		host = "localhost"
+	}
+
+	clientConfig := &restclient.Config{
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:         50,
+		Burst:       100,
+		Host:        "https://" + net.JoinHostPort(host, port),
+		BearerToken: token,
+	}
+
+	// find certificate for host: either explicitly given, from the server cert bundle or one of the SNI certs
+	var derCA []byte
+	if s.CACert != nil {
+		derCA = s.CACert.Certificate[0]
+	}
+	if derCA == nil && s.Cert != nil {
+		x509Cert, err := x509.ParseCertificate(s.Cert.Certificate[0])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse server certificate: %v", err)
+		}
+
+		if (net.ParseIP(host) != nil && certMatchesIP(x509Cert, host)) || certMatchesName(x509Cert, host) {
+			derCA = s.Cert.Certificate[0]
+		}
+	}
+	if derCA == nil && net.ParseIP(host) == nil {
+		if cert, found := s.SNICerts[host]; found {
+			derCA = cert.Certificate[0]
+		}
+	}
+	if derCA == nil {
+		return nil, fmt.Errorf("failed to find certificate which matches %q", host)
+	}
+	pemCA := bytes.Buffer{}
+	if err := pem.Encode(&pemCA, &pem.Block{Type: "CERTIFICATE", Bytes: derCA}); err != nil {
+		return nil, err
+	}
+	clientConfig.CAData = pemCA.Bytes()
+
+	return clientConfig, nil
+}
+
+func (s *ServingInfo) NewSelfClientConfig(token string) (*restclient.Config, error) {
+	if s == nil {
+		return nil, nil
+	}
+	return &restclient.Config{
+		Host: s.BindAddress,
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:   50,
+		Burst: 100,
+	}, nil
+}
+
+func certMatchesName(cert *x509.Certificate, name string) bool {
+	for _, certName := range cert.DNSNames {
+		if certName == name {
+			return true
+		}
+	}
+
+	return false
+}
+
+func certMatchesIP(cert *x509.Certificate, ip string) bool {
+	for _, certIP := range cert.IPAddresses {
+		if certIP.String() == ip {
+			return true
+		}
+	}
+
+	return false
+}

pkg/genericapiserver/options/server_run_options.go

@@ -75,7 +75,6 @@ type ServerRunOptions struct {
 	// for testing). This is not actually exposed as a flag.
 	DefaultStorageVersions string
 	TargetRAMMB            int
-	TLSCAFile              string
 	WatchCacheSizes        []string
 }
 

pkg/genericapiserver/options/serving.go

@@ -17,14 +17,14 @@ limitations under the License.
 package options
 
 import (
-	"errors"
 	"fmt"
 	"net"
-	"strconv"
+	"path"
 
+	"github.com/golang/glog"
 	"github.com/spf13/pflag"
 
-	"k8s.io/kubernetes/pkg/client/restclient"
+	certutil "k8s.io/kubernetes/pkg/util/cert"
 	"k8s.io/kubernetes/pkg/util/config"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
 )
@@ -43,14 +43,10 @@ type SecureServingOptions struct {
 	SNICertKeys []config.NamedCertKey
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
-
-	// ServerCA is the certificate bundle for the signer of your serving certificate.  Used for building a loopback
-	// connection to the API server for admission.
-	ServerCA string
 }
 
 type CertKey struct {
-	// CertFile is a file containing a PEM-encoded certificate
+	// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain
 	CertFile string
 	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
 	KeyFile string
@@ -59,6 +55,8 @@ type CertKey struct {
 type GeneratableKeyCert struct {
 	CertKey CertKey
 
+	// CACertFile is an optional file containing the certificate chain for CertKey.CertFile
+	CACertFile string
 	// CertDirectory is a directory that will contain the certificates.  If the cert and key aren't specifically set
 	// this will be used to derive a match with the "pair-name"
 	CertDirectory string
@@ -80,31 +78,6 @@ func NewSecureServingOptions() *SecureServingOptions {
 	}
 }
 
-func (s *SecureServingOptions) NewSelfClientConfig(token string) *restclient.Config {
-	if s == nil || s.ServingOptions.BindPort <= 0 || len(s.ServerCA) == 0 {
-		return nil
-	}
-
-	clientConfig := &restclient.Config{
-		// Increase QPS limits. The client is currently passed to all admission plugins,
-		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
-		// for more details. Once #22422 is fixed, we may want to remove it.
-		QPS:   50,
-		Burst: 100,
-	}
-
-	// Use secure port if the ServerCA is specified
-	host := s.ServingOptions.BindAddress.String()
-	if host == "0.0.0.0" {
-		host = "localhost"
-	}
-	clientConfig.Host = "https://" + net.JoinHostPort(host, strconv.Itoa(s.ServingOptions.BindPort))
-	clientConfig.CAFile = s.ServerCA
-	clientConfig.BearerToken = token
-
-	return clientConfig
-}
-
 func (s *SecureServingOptions) Validate() []error {
 	errors := []error{}
 	if s == nil {
@@ -138,6 +111,11 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
 		"File containing the default x509 private key matching --tls-cert-file.")
 
+	fs.StringVar(&s.ServerCert.CACertFile, "tls-ca-file", s.ServerCert.CACertFile, "If set, this "+
+		"certificate authority will used for secure access from Admission "+
+		"Controllers. This must be a valid PEM-encoded CA bundle. Altneratively, the certificate authority "+
+		"can be appended to the certificate provided by --tls-cert-file.")
+
 	fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
 		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
 		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
@@ -151,18 +129,12 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"If set, any request presenting a client certificate signed by one of "+
 		"the authorities in the client-ca-file is authenticated with an identity "+
 		"corresponding to the CommonName of the client certificate.")
-
-	fs.StringVar(&s.ServerCA, "tls-ca-file", s.ServerCA, "If set, this "+
-		"certificate authority will used for secure access from Admission "+
-		"Controllers. This must be a valid PEM-encoded CA bundle.")
-
 }
 
 func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.IPVar(&s.ServingOptions.BindAddress, "public-address-override", s.ServingOptions.BindAddress,
 		"DEPRECATED: see --bind-address instead.")
 	fs.MarkDeprecated("public-address-override", "see --bind-address instead.")
-
 }
 
 func NewInsecureServingOptions() *ServingOptions {
@@ -182,23 +154,6 @@ func (s ServingOptions) Validate(portArg string) []error {
 	return errors
 }
 
-func (s *ServingOptions) NewSelfClientConfig(token string) *restclient.Config {
-	if s == nil || s.BindPort <= 0 {
-		return nil
-	}
-	clientConfig := &restclient.Config{
-		// Increase QPS limits. The client is currently passed to all admission plugins,
-		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
-		// for more details. Once #22422 is fixed, we may want to remove it.
-		QPS:   50,
-		Burst: 100,
-	}
-
-	clientConfig.Host = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
-
-	return clientConfig
-}
-
 func (s *ServingOptions) DefaultExternalAddress() (net.IP, error) {
 	return utilnet.ChooseBindAddress(s.BindAddress)
 }
@@ -224,14 +179,47 @@ func (s *ServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
 	fs.MarkDeprecated("port", "see --insecure-port instead.")
 }
 
-// Returns a clientconfig which can be used to talk to this apiserver.
-func NewSelfClientConfig(secureServingOptions *SecureServingOptions, insecureServingOptions *ServingOptions, token string) (*restclient.Config, error) {
-	if cfg := secureServingOptions.NewSelfClientConfig(token); cfg != nil {
-		return cfg, nil
+func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateIPs ...net.IP) error {
+	keyCert := &s.ServerCert.CertKey
+
+	if s == nil || len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
+		return nil
+	}
+
+	keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt")
+	keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key")
+
+	canReadCertAndKey, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile)
+	if err != nil {
+		return err
 	}
-	if cfg := insecureServingOptions.NewSelfClientConfig(token); cfg != nil {
-		return cfg, nil
+	if !canReadCertAndKey {
+		// TODO: It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
+		// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
+		// TODO (cjcullen): Is ClusterIP the right address to sign a cert with?
+		alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
+
+		// add either the bind address or localhost to the valid alternates
+		bindIP := s.ServingOptions.BindAddress.String()
+		if bindIP == "0.0.0.0" {
+			alternateDNS = append(alternateDNS, "localhost")
+		} else {
+			alternateIPs = append(alternateIPs, s.ServingOptions.BindAddress)
+		}
+
+		if cert, key, err := certutil.GenerateSelfSignedCertKey(publicAddress, alternateIPs, alternateDNS); err != nil {
+			return fmt.Errorf("unable to generate self signed cert: %v", err)
+		} else {
+			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
+				return err
+			}
+
+			if err := certutil.WriteKey(keyCert.KeyFile, key); err != nil {
+				return err
+			}
+			glog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
+		}
 	}
 
-	return nil, errors.New("Unable to set url for apiserver local client")
+	return nil
 }

pkg/genericapiserver/serve.go

@@ -42,17 +42,12 @@ const (
 // be loaded or the initial listen call fails. The actual server loop (stoppable by closing
 // stopCh) runs in a go routine, i.e. serveSecurely does not block.
 func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
-	namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts)
-	if err != nil {
-		return fmt.Errorf("unable to load SNI certificates: %v", err)
-	}
-
 	secureServer := &http.Server{
 		Addr:           s.SecureServingInfo.BindAddress,
 		Handler:        s.Handler,
 		MaxHeaderBytes: 1 << 20,
 		TLSConfig: &tls.Config{
-			NameToCertificate: namedCerts,
+			NameToCertificate: s.SecureServingInfo.SNICerts,
 			// Can't use SSLv3 because of POODLE and BEAST
 			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
 			// Can't use TLSv1.1 because of RC4 cipher usage
@@ -62,19 +57,15 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 		},
 	}
 
-	if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 {
-		secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
-		secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
-		if err != nil {
-			return fmt.Errorf("unable to load server certificate: %v", err)
-		}
+	if s.SecureServingInfo.Cert != nil {
+		secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert}
 	}
 
 	// append all named certs. Otherwise, the go tls stack will think no SNI processing
 	// is necessary because there is only one cert anyway.
 	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
 	// cert will become the default cert. That's what we expect anyway.
-	for _, c := range namedCerts {
+	for _, c := range s.SecureServingInfo.SNICerts {
 		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
 	}
 
@@ -91,6 +82,7 @@ func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
 	}
 
 	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
+	var err error
 	s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh)
 	return err
 }
@@ -186,48 +178,40 @@ func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int
 	return tcpAddr.Port, nil
 }
 
-// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in
-// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded.
-// Returns nil if len(namedCertKeys) == 0
-func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) {
-	if len(namedCertKeys) == 0 {
-		return nil, nil
-	}
+type namedTlsCert struct {
+	tlsCert tls.Certificate
 
-	// load keys
-	tlsCerts := make([]tls.Certificate, len(namedCertKeys))
-	for i := range namedCertKeys {
-		var err error
-		nkc := &namedCertKeys[i]
-		tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile)
-		if err != nil {
-			return nil, err
-		}
-	}
+	// names is a list of domain patterns: fully qualified domain names, possibly prefixed with
+	// wildcard segments.
+	names []string
+}
 
+// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// cannot be loaded. Returns nil if len(certs) == 0
+func getNamedCertificateMap(certs []namedTlsCert) (map[string]*tls.Certificate, error) {
 	// register certs with implicit names first, reverse order such that earlier trump over the later
-	tlsCertsByName := map[string]*tls.Certificate{}
-	for i := len(namedCertKeys) - 1; i >= 0; i-- {
-		nkc := &namedCertKeys[i]
-		if len(nkc.Names) > 0 {
+	byName := map[string]*tls.Certificate{}
+	for i := len(certs) - 1; i >= 0; i-- {
+		if len(certs[i].names) > 0 {
 			continue
 		}
-		cert := &tlsCerts[i]
+		cert := &certs[i].tlsCert
 
 		// read names from certificate common names and DNS names
 		if len(cert.Certificate) == 0 {
-			return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile)
+			return nil, fmt.Errorf("empty SNI certificate, skipping")
 		}
 		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
 		if err != nil {
-			return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err)
+			return nil, fmt.Errorf("parse error for SNI certificate: %v", err)
 		}
 		cn := x509Cert.Subject.CommonName
 		if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
-			tlsCertsByName[cn] = cert
+			byName[cn] = cert
 		}
 		for _, san := range x509Cert.DNSNames {
-			tlsCertsByName[san] = cert
+			byName[san] = cert
 		}
 		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
 		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
@@ -235,17 +219,14 @@ func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certi
 
 	// register certs with explicit names last, overwriting every of the implicit ones,
 	// again in reverse order.
-	for i := len(namedCertKeys) - 1; i >= 0; i-- {
-		nkc := &namedCertKeys[i]
-		if len(nkc.Names) == 0 {
-			continue
-		}
-		for _, name := range nkc.Names {
-			tlsCertsByName[name] = &tlsCerts[i]
+	for i := len(certs) - 1; i >= 0; i-- {
+		namedCert := &certs[i]
+		for _, name := range namedCert.names {
+			byName[name] = &certs[i].tlsCert
 		}
 	}
 
-	return tlsCertsByName, nil
+	return byName, nil
 }
 
 // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted

pkg/genericapiserver/serve_test.go

@@ -20,15 +20,18 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"os"
 	"testing"
 
-	utilcert "k8s.io/kubernetes/pkg/util/cert"
-
 	"github.com/stretchr/testify/assert"
+
+	"k8s.io/kubernetes/pkg/genericapiserver/options"
+	utilcert "k8s.io/kubernetes/pkg/util/cert"
+	"k8s.io/kubernetes/pkg/util/config"
 )
 
 type TestCertSpec struct {
@@ -41,47 +44,6 @@ type NamedTestCertSpec struct {
 	explicitNames []string // as --tls-sni-cert-key explicit names
 }
 
-func createTestCerts(spec TestCertSpec) (certFilePath, keyFilePath string, err error) {
-	var ips []net.IP
-	for _, ip := range spec.ips {
-		ips = append(ips, net.ParseIP(ip))
-	}
-
-	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, ips, spec.names)
-	if err != nil {
-		return "", "", err
-	}
-
-	certFile, err := ioutil.TempFile(os.TempDir(), "cert")
-	if err != nil {
-		return "", "", err
-	}
-
-	keyFile, err := ioutil.TempFile(os.TempDir(), "key")
-	if err != nil {
-		os.Remove(certFile.Name())
-		return "", "", err
-	}
-
-	_, err = certFile.Write(certPem)
-	if err != nil {
-		os.Remove(certFile.Name())
-		os.Remove(keyFile.Name())
-		return "", "", err
-	}
-	certFile.Close()
-
-	_, err = keyFile.Write(keyPem)
-	if err != nil {
-		os.Remove(certFile.Name())
-		os.Remove(keyFile.Name())
-		return "", "", err
-	}
-	keyFile.Close()
-
-	return certFile.Name(), keyFile.Name(), nil
-}
-
 func TestGetNamedCertificateMap(t *testing.T) {
 	tests := []struct {
 		certs         []NamedTestCertSpec
@@ -225,26 +187,21 @@ func TestGetNamedCertificateMap(t *testing.T) {
 
 NextTest:
 	for i, test := range tests {
-		var namedCertKeys []NamedCertKey
+		var namedTLSCerts []namedTlsCert
 		bySignature := map[string]int{} // index in test.certs by cert signature
 		for j, c := range test.certs {
-			certFile, keyFile, err := createTestCerts(c.TestCertSpec)
+			cert, err := createTestTLSCerts(c.TestCertSpec)
 			if err != nil {
 				t.Errorf("%d - failed to create cert %d: %v", i, j, err)
 				continue NextTest
 			}
-			defer os.Remove(certFile)
-			defer os.Remove(keyFile)
 
-			namedCertKeys = append(namedCertKeys, NamedCertKey{
-				CertKey: CertKey{
-					KeyFile:  keyFile,
-					CertFile: certFile,
-				},
-				Names: c.explicitNames,
+			namedTLSCerts = append(namedTLSCerts, namedTlsCert{
+				tlsCert: cert,
+				names:   c.explicitNames,
 			})
 
-			sig, err := certFileSignature(certFile, keyFile)
+			sig, err := certSignature(cert)
 			if err != nil {
 				t.Errorf("%d - failed to get signature for %d: %v", i, j, err)
 				continue NextTest
@@ -252,7 +209,7 @@ NextTest:
 			bySignature[sig] = j
 		}
 
-		certMap, err := getNamedCertificateMap(namedCertKeys)
+		certMap, err := getNamedCertificateMap(namedTLSCerts)
 		if err == nil && len(test.errorString) != 0 {
 			t.Errorf("%d - expected no error, got: %v", i, err)
 		} else if err != nil && err.Error() != test.errorString {
@@ -363,19 +320,30 @@ func TestServerRunWithSNI(t *testing.T) {
 		},
 	}
 
+	tempDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tempDir)
+
 NextTest:
 	for i, test := range tests {
 		// create server cert
-		serverCertFile, serverKeyFile, err := createTestCerts(test.Cert)
+		serverCertBundleFile, serverKeyFile, err := createTestCertFiles(tempDir, test.Cert)
 		if err != nil {
 			t.Errorf("%d - failed to create server cert: %v", i, err)
+			continue NextTest
 		}
-		defer os.Remove(serverCertFile)
-		defer os.Remove(serverKeyFile)
+		ca, err := caCertFromBundle(serverCertBundleFile)
+		if err != nil {
+			t.Errorf("%d - failed to extract ca cert from server cert bundle: %v", i, err)
+			continue NextTest
+		}
+		caCerts := []*x509.Certificate{ca}
 
 		// create SNI certs
-		var namedCertKeys []NamedCertKey
-		serverSig, err := certFileSignature(serverCertFile, serverKeyFile)
+		var namedCertKeys []config.NamedCertKey
+		serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile)
 		if err != nil {
 			t.Errorf("%d - failed to get server cert signature: %v", i, err)
 			continue NextTest
@@ -384,24 +352,27 @@ NextTest:
 			serverSig: -1,
 		}
 		for j, c := range test.SNICerts {
-			certFile, keyFile, err := createTestCerts(c.TestCertSpec)
+			certBundleFile, keyFile, err := createTestCertFiles(tempDir, c.TestCertSpec)
 			if err != nil {
 				t.Errorf("%d - failed to create SNI cert %d: %v", i, j, err)
 				continue NextTest
 			}
-			defer os.Remove(certFile)
-			defer os.Remove(keyFile)
 
-			namedCertKeys = append(namedCertKeys, NamedCertKey{
-				CertKey: CertKey{
-					KeyFile:  keyFile,
-					CertFile: certFile,
-				},
-				Names: c.explicitNames,
+			namedCertKeys = append(namedCertKeys, config.NamedCertKey{
+				KeyFile:  keyFile,
+				CertFile: certBundleFile,
+				Names:    c.explicitNames,
 			})
 
+			ca, err := caCertFromBundle(certBundleFile)
+			if err != nil {
+				t.Errorf("%d - failed to extract ca cert from SNI cert %d: %v", i, j, err)
+				continue NextTest
+			}
+			caCerts = append(caCerts, ca)
+
 			// store index in namedCertKeys with the signature as the key
-			sig, err := certFileSignature(certFile, keyFile)
+			sig, err := certFileSignature(certBundleFile, keyFile)
 			if err != nil {
 				t.Errorf("%d - failed get SNI cert %d signature: %v", i, j, err)
 				continue NextTest
@@ -416,17 +387,22 @@ NextTest:
 		defer etcdserver.Terminate(t)
 
 		config.EnableIndex = true
-		config.SecureServingInfo = &SecureServingInfo{
-			ServingInfo: ServingInfo{
-				BindAddress: "localhost:0",
+		_, err = config.ApplySecureServingOptions(&options.SecureServingOptions{
+			ServingOptions: options.ServingOptions{
+				BindAddress: net.ParseIP("127.0.0.1"),
+				BindPort:    6443,
 			},
-			ServerCert: GeneratableKeyCert{
-				CertKey: CertKey{
-					CertFile: serverCertFile,
+			ServerCert: options.GeneratableKeyCert{
+				CertKey: options.CertKey{
+					CertFile: serverCertBundleFile,
 					KeyFile:  serverKeyFile,
 				},
 			},
-			SNICerts: namedCertKeys,
+			SNICertKeys: namedCertKeys,
+		})
+		if err != nil {
+			t.Errorf("%d - failed applying the SecureServingOptions: %v", i, err)
+			continue NextTest
 		}
 		config.InsecureServingInfo = nil
 
@@ -436,27 +412,18 @@ NextTest:
 			continue NextTest
 		}
 
+		// patch in a 0-port to enable auto port allocation
+		s.SecureServingInfo.BindAddress = "127.0.0.1:0"
+
 		if err := s.serveSecurely(stopCh); err != nil {
 			t.Errorf("%d - failed running the server: %v", i, err)
 			continue NextTest
 		}
 
-		// load certificates into a pool
+		// load ca certificates into a pool
 		roots := x509.NewCertPool()
-		certFiles := []string{serverCertFile}
-		for _, c := range namedCertKeys {
-			certFiles = append(certFiles, c.CertFile)
-		}
-		for _, certFile := range certFiles {
-			bs, err := ioutil.ReadFile(certFile)
-			if err != nil {
-				t.Errorf("%d - error reading %q: %v", i, certFile, err)
-				continue NextTest
-			}
-			if ok := roots.AppendCertsFromPEM(bs); !ok {
-				t.Errorf("%d - error adding cert %q to the pool", i, certFile)
-				continue NextTest
-			}
+		for _, caCert := range caCerts {
+			roots.AddCert(caCert)
 		}
 
 		// try to dial
@@ -485,6 +452,77 @@ NextTest:
 	}
 }
 
+func parseIPList(ips []string) []net.IP {
+	var netIPs []net.IP
+	for _, ip := range ips {
+		netIPs = append(netIPs, net.ParseIP(ip))
+	}
+	return netIPs
+}
+
+func createTestTLSCerts(spec TestCertSpec) (tlsCert tls.Certificate, err error) {
+	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names)
+	if err != nil {
+		return tlsCert, err
+	}
+
+	tlsCert, err = tls.X509KeyPair(certPem, keyPem)
+	return tlsCert, err
+}
+
+func createTestCertFiles(dir string, spec TestCertSpec) (certFilePath, keyFilePath string, err error) {
+	certPem, keyPem, err := utilcert.GenerateSelfSignedCertKey(spec.host, parseIPList(spec.ips), spec.names)
+	if err != nil {
+		return "", "", err
+	}
+
+	certFile, err := ioutil.TempFile(dir, "cert")
+	if err != nil {
+		return "", "", err
+	}
+
+	keyFile, err := ioutil.TempFile(dir, "key")
+	if err != nil {
+		return "", "", err
+	}
+
+	_, err = certFile.Write(certPem)
+	if err != nil {
+		return "", "", err
+	}
+	certFile.Close()
+
+	_, err = keyFile.Write(keyPem)
+	if err != nil {
+		return "", "", err
+	}
+	keyFile.Close()
+
+	return certFile.Name(), keyFile.Name(), nil
+}
+
+func caCertFromBundle(bundlePath string) (*x509.Certificate, error) {
+	pemData, err := ioutil.ReadFile(bundlePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// fetch last block
+	var block *pem.Block
+	for {
+		var nextBlock *pem.Block
+		nextBlock, pemData = pem.Decode(pemData)
+		if nextBlock == nil {
+			if block == nil {
+				return nil, fmt.Errorf("no certificate found in %q", bundlePath)
+
+			}
+			return x509.ParseCertificate(block.Bytes)
+		}
+		block = nextBlock
+	}
+}
+
 func x509CertSignature(cert *x509.Certificate) string {
 	return base64.StdEncoding.EncodeToString(cert.Signature)
 }
@@ -494,13 +532,13 @@ func certFileSignature(certFile, keyFile string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	return certSignature(cert)
+}
 
+func certSignature(cert tls.Certificate) (string, error) {
 	x509Certs, err := x509.ParseCertificates(cert.Certificate[0])
 	if err != nil {
 		return "", err
 	}
-	if len(x509Certs) == 0 {
-		return "", fmt.Errorf("expected at least one cert after reparsing cert %q", certFile)
-	}
 	return x509CertSignature(x509Certs[0]), nil
 }