Merge pull request #35109 from sttts/sttts-sni

cmd/kube-apiserver/app/server.go

@@ -335,6 +335,6 @@ func Run(s *options.ServerRunOptions) error {
 	}
 
 	sharedInformers.Start(wait.NeverStop)
-	m.GenericAPIServer.PrepareRun().Run()
+	m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
 	return nil
 }

cmd/kubelet/app/server.go

@@ -502,9 +502,19 @@ func InitializeTLS(kc *componentconfig.KubeletConfiguration) (*server.TLSOptions
 		kc.TLSCertFile = path.Join(kc.CertDirectory, "kubelet.crt")
 		kc.TLSPrivateKeyFile = path.Join(kc.CertDirectory, "kubelet.key")
 		if !certutil.CanReadCertOrKey(kc.TLSCertFile, kc.TLSPrivateKeyFile) {
-			if err := certutil.GenerateSelfSignedCert(nodeutil.GetHostname(kc.HostnameOverride), kc.TLSCertFile, kc.TLSPrivateKeyFile, nil, nil); err != nil {
+			cert, key, err := certutil.GenerateSelfSignedCertKey(nodeutil.GetHostname(kc.HostnameOverride), nil, nil)
+			if err != nil {
 				return nil, fmt.Errorf("unable to generate self signed cert: %v", err)
 			}
+
+			if err := certutil.WriteCert(kc.TLSCertFile, cert); err != nil {
+				return nil, err
+			}
+
+			if err := certutil.WriteKey(kc.TLSPrivateKeyFile, key); err != nil {
+				return nil, err
+			}
+
 			glog.V(4).Infof("Using self-signed cert (%s, %s)", kc.TLSCertFile, kc.TLSPrivateKeyFile)
 		}
 	}

examples/apiserver/apiserver.go

@@ -60,7 +60,7 @@ func NewServerRunOptions() *genericoptions.ServerRunOptions {
 	return serverOptions
 }
 
-func Run(serverOptions *genericoptions.ServerRunOptions) error {
+func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error {
 	// Set ServiceClusterIPRange
 	_, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24")
 	serverOptions.ServiceClusterIPRange = *serviceClusterIPRange
@@ -105,6 +105,6 @@ func Run(serverOptions *genericoptions.ServerRunOptions) error {
 	if err := s.InstallAPIGroup(&apiGroupInfo); err != nil {
 		return fmt.Errorf("Error in installing API: %v", err)
 	}
-	s.PrepareRun().Run()
+	s.PrepareRun().Run(stopCh)
 	return nil
 }

examples/apiserver/server/main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"k8s.io/kubernetes/examples/apiserver"
 	"k8s.io/kubernetes/pkg/util/flag"
+	"k8s.io/kubernetes/pkg/util/wait"
 
 	"github.com/golang/glog"
 	"github.com/spf13/pflag"
@@ -32,7 +33,7 @@ func main() {
 	serverRunOptions.AddEtcdStorageFlags(pflag.CommandLine)
 	flag.InitFlags()
 
-	if err := apiserver.Run(serverRunOptions); err != nil {
+	if err := apiserver.Run(serverRunOptions, wait.NeverStop); err != nil {
 		glog.Fatalf("Error in bringing up the server: %v", err)
 	}
 }

federation/cmd/federation-apiserver/app/server.go

@@ -225,7 +225,7 @@ func Run(s *options.ServerRunOptions) error {
 	installExtensionsAPIs(m, restOptionsFactory)
 
 	sharedInformers.Start(wait.NeverStop)
-	m.PrepareRun().Run()
+	m.PrepareRun().Run(wait.NeverStop)
 	return nil
 }
 

hack/verify-flags/known-flags.txt

@@ -555,6 +555,7 @@ tls-ca-file
 tls-cert-file
 tls-private-key-file
 to-version
+tls-sni-cert-key
 token-auth-file
 ttl-keys-prefix
 ttl-secs

pkg/genericapiserver/BUILD

@@ -21,6 +21,7 @@ go_library(
         "resource_config.go",
         "resource_encoding_config.go",
         "reststorage_interfaces.go",
+        "serve.go",
         "storage_factory.go",
         "tunneler.go",
     ],
@@ -66,6 +67,7 @@ go_library(
         "//vendor:github.com/emicklei/go-restful",
         "//vendor:github.com/go-openapi/spec",
         "//vendor:github.com/golang/glog",
+        "//vendor:github.com/pkg/errors",
         "//vendor:github.com/prometheus/client_golang/prometheus",
         "//vendor:gopkg.in/natefinch/lumberjack.v2",
     ],
@@ -77,6 +79,7 @@ go_test(
         "default_storage_factory_builder_test.go",
         "genericapiserver_test.go",
         "resource_config_test.go",
+        "serve_test.go",
         "server_run_options_test.go",
         "storage_factory_test.go",
         "tunneler_test.go",
@@ -100,6 +103,7 @@ go_test(
         "//pkg/registry/core/service/ipallocator:go_default_library",
         "//pkg/storage/etcd/testing:go_default_library",
         "//pkg/storage/storagebackend:go_default_library",
+        "//pkg/util/cert:go_default_library",
         "//pkg/util/clock:go_default_library",
         "//pkg/util/sets:go_default_library",
         "//pkg/version:go_default_library",

pkg/genericapiserver/config.go

@@ -110,7 +110,7 @@ type Config struct {
 	// same value for this field. (Numbers > 1 currently untested.)
 	MasterCount int
 
-	SecureServingInfo   *ServingInfo
+	SecureServingInfo   *SecureServingInfo
 	InsecureServingInfo *ServingInfo
 
 	// The port on PublicAddress where a read-write server will be installed.
@@ -177,17 +177,36 @@ type Config struct {
 type ServingInfo struct {
 	// BindAddress is the ip:port to serve on
 	BindAddress string
+}
+
+type SecureServingInfo struct {
+	ServingInfo
+
 	// ServerCert is the TLS cert info for serving secure traffic
-	ServerCert CertInfo
+	ServerCert GeneratableKeyCert
+	// SNICerts are named CertKeys for serving secure traffic with SNI support.
+	SNICerts []NamedCertKey
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
 }
 
-type CertInfo struct {
+type CertKey struct {
 	// CertFile is a file containing a PEM-encoded certificate
 	CertFile string
 	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
 	KeyFile string
+}
+
+type NamedCertKey struct {
+	CertKey
+
+	// Names is a list of domain patterns: fully qualified domain names, possibly prefixed with
+	// wildcard segments.
+	Names []string
+}
+
+type GeneratableKeyCert struct {
+	CertKey
 	// Generate indicates that the cert/key pair should be generated if its not present.
 	Generate bool
 }
@@ -248,12 +267,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
 	}
 
 	if options.SecurePort > 0 {
-		secureServingInfo := &ServingInfo{
-			BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)),
-			ServerCert: CertInfo{
-				CertFile: options.TLSCertFile,
-				KeyFile:  options.TLSPrivateKeyFile,
+		secureServingInfo := &SecureServingInfo{
+			ServingInfo: ServingInfo{
+				BindAddress: net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)),
 			},
+			ServerCert: GeneratableKeyCert{
+				CertKey: CertKey{
+					CertFile: options.TLSCertFile,
+					KeyFile:  options.TLSPrivateKeyFile,
+				},
+			},
+			SNICerts: []NamedCertKey{},
 			ClientCA: options.ClientCAFile,
 		}
 		if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" {
@@ -262,6 +286,17 @@ func (c *Config) ApplyOptions(options *options.ServerRunOptions) *Config {
 			secureServingInfo.ServerCert.KeyFile = path.Join(options.CertDirectory, "apiserver.key")
 		}
 
+		secureServingInfo.SNICerts = nil
+		for _, nkc := range options.SNICertKeys {
+			secureServingInfo.SNICerts = append(secureServingInfo.SNICerts, NamedCertKey{
+				CertKey: CertKey{
+					KeyFile:  nkc.KeyFile,
+					CertFile: nkc.CertFile,
+				},
+				Names: nkc.Names,
+			})
+		}
+
 		c.SecureServingInfo = secureServingInfo
 		c.ReadWritePort = options.SecurePort
 	}
@@ -434,9 +469,16 @@ func (c completedConfig) MaybeGenerateServingCerts() error {
 		alternateIPs := []net.IP{c.ServiceReadWriteIP}
 		alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}
 
-		if err := certutil.GenerateSelfSignedCert(c.PublicAddress.String(), c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile, alternateIPs, alternateDNS); err != nil {
-			return fmt.Errorf("Unable to generate self signed cert: %v", err)
+		if cert, key, err := certutil.GenerateSelfSignedCertKey(c.PublicAddress.String(), alternateIPs, alternateDNS); err != nil {
+			return fmt.Errorf("unable to generate self signed cert: %v", err)
 		} else {
+			if err := certutil.WriteCert(c.SecureServingInfo.ServerCert.CertFile, cert); err != nil {
+				return err
+			}
+
+			if err := certutil.WriteKey(c.SecureServingInfo.ServerCert.KeyFile, key); err != nil {
+				return err
+			}
 			glog.Infof("Generated self-signed cert (%s, %s)", c.SecureServingInfo.ServerCert.CertFile, c.SecureServingInfo.ServerCert.KeyFile)
 		}
 	}

pkg/genericapiserver/genericapiserver.go

@@ -17,7 +17,6 @@ limitations under the License.
 package genericapiserver
 
 import (
-	"crypto/tls"
 	"fmt"
 	"mime"
 	"net"
@@ -44,9 +43,7 @@ import (
 	"k8s.io/kubernetes/pkg/genericapiserver/openapi/common"
 	"k8s.io/kubernetes/pkg/genericapiserver/routes"
 	"k8s.io/kubernetes/pkg/runtime"
-	certutil "k8s.io/kubernetes/pkg/util/cert"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
-	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/util/sets"
 )
 
@@ -112,9 +109,12 @@ type GenericAPIServer struct {
 	// The registered APIs
 	HandlerContainer *genericmux.APIContainer
 
-	SecureServingInfo   *ServingInfo
+	SecureServingInfo   *SecureServingInfo
 	InsecureServingInfo *ServingInfo
 
+	// numerical ports, set after listening
+	effectiveSecurePort, effectiveInsecurePort int
+
 	// ExternalAddress is the address (hostname or IP and port) that should be used in
 	// external (public internet) URLs for this GenericAPIServer.
 	ExternalAddress string
@@ -180,7 +180,6 @@ type preparedGenericAPIServer struct {
 
 // PrepareRun does post API installation setup steps.
 func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
-	// install APIs which depend on other APIs to be installed
 	if s.enableSwaggerSupport {
 		routes.Swagger{ExternalAddress: s.ExternalAddress}.Install(s.HandlerContainer)
 	}
@@ -192,76 +191,18 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 	return preparedGenericAPIServer{s}
 }
 
-func (s preparedGenericAPIServer) Run() {
+// Run spawns the http servers (secure and insecure). It only returns if stopCh is closed
+// or one of the ports cannot be listened on initially.
+func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) {
 	if s.SecureServingInfo != nil && s.Handler != nil {
-		secureServer := &http.Server{
-			Addr:           s.SecureServingInfo.BindAddress,
-			Handler:        s.Handler,
-			MaxHeaderBytes: 1 << 20,
-			TLSConfig: &tls.Config{
-				// Can't use SSLv3 because of POODLE and BEAST
-				// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
-				// Can't use TLSv1.1 because of RC4 cipher usage
-				MinVersion: tls.VersionTLS12,
-			},
+		if err := s.serveSecurely(stopCh); err != nil {
+			glog.Fatal(err)
 		}
-
-		if len(s.SecureServingInfo.ClientCA) > 0 {
-			clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
-			if err != nil {
-				glog.Fatalf("Unable to load client CA file: %v", err)
-			}
-			// Populate PeerCertificates in requests, but don't reject connections without certificates
-			// This allows certificates to be validated by authenticators, while still allowing other auth types
-			secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
-			// Specify allowed CAs for client certificates
-			secureServer.TLSConfig.ClientCAs = clientCAs
-			// "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server
-			secureServer.TLSConfig.NextProtos = []string{"h2"}
-
-		}
-
-		glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
-		go func() {
-			defer utilruntime.HandleCrash()
-
-			for {
-				if err := secureServer.ListenAndServeTLS(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile); err != nil {
-					glog.Errorf("Unable to listen for secure (%v); will try again.", err)
-				}
-				time.Sleep(15 * time.Second)
-			}
-		}()
 	}
 
 	if s.InsecureServingInfo != nil && s.InsecureHandler != nil {
-		insecureServer := &http.Server{
-			Addr:           s.InsecureServingInfo.BindAddress,
-			Handler:        s.InsecureHandler,
-			MaxHeaderBytes: 1 << 20,
-		}
-		glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
-		go func() {
-			defer utilruntime.HandleCrash()
-
-			for {
-				if err := insecureServer.ListenAndServe(); err != nil {
-					glog.Errorf("Unable to listen for insecure (%v); will try again.", err)
-				}
-				time.Sleep(15 * time.Second)
-			}
-		}()
-	}
-
-	// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) per port
-	if s.SecureServingInfo != nil {
-		if err := waitForSuccessfulDial(true, "tcp", s.SecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
-			glog.Fatalf("Secure server never started: %v", err)
-		}
-	}
-	if s.InsecureServingInfo != nil {
-		if err := waitForSuccessfulDial(false, "tcp", s.InsecureServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100); err != nil {
-			glog.Fatalf("Insecure server never started: %v", err)
+		if err := s.serveInsecurely(stopCh); err != nil {
+			glog.Fatal(err)
 		}
 	}
 
@@ -272,7 +213,7 @@ func (s preparedGenericAPIServer) Run() {
 		glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
 	}
 
-	select {}
+	<-stopCh
 }
 
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
@@ -471,27 +412,3 @@ func NewDefaultAPIGroupInfo(group string) APIGroupInfo {
 		NegotiatedSerializer:         api.Codecs,
 	}
 }
-
-// waitForSuccessfulDial attempts to connect to the given address, closing and returning nil on the first successful connection.
-func waitForSuccessfulDial(https bool, network, address string, timeout, interval time.Duration, retries int) error {
-	var (
-		conn net.Conn
-		err  error
-	)
-	for i := 0; i <= retries; i++ {
-		dialer := net.Dialer{Timeout: timeout}
-		if https {
-			conn, err = tls.DialWithDialer(&dialer, network, address, &tls.Config{InsecureSkipVerify: true})
-		} else {
-			conn, err = dialer.Dial(network, address)
-		}
-		if err != nil {
-			glog.V(5).Infof("Got error %#v, trying again: %#v\n", err, address)
-			time.Sleep(interval)
-			continue
-		}
-		conn.Close()
-		return nil
-	}
-	return err
-}

pkg/genericapiserver/options/server_run_options.go

@@ -118,6 +118,7 @@ type ServerRunOptions struct {
 	TLSCAFile              string
 	TLSCertFile            string
 	TLSPrivateKeyFile      string
+	SNICertKeys            []config.NamedCertKey
 	TokenAuthFile          string
 	EnableAnyToken         bool
 	WatchCacheSizes        []string
@@ -488,13 +489,22 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"Controllers. This must be a valid PEM-encoded CA bundle.")
 
 	fs.StringVar(&s.TLSCertFile, "tls-cert-file", s.TLSCertFile, ""+
-		"File containing x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
 		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
 		"--tls-private-key-file are not provided, a self-signed certificate and key "+
 		"are generated for the public address and saved to /var/run/kubernetes.")
 
 	fs.StringVar(&s.TLSPrivateKeyFile, "tls-private-key-file", s.TLSPrivateKeyFile,
-		"File containing x509 private key matching --tls-cert-file.")
+		"File containing the default x509 private key matching --tls-cert-file.")
+
+	fs.Var(config.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
+		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
+		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
+		"segments. If no domain patterns are provided, the names of the certificate are "+
+		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
+		"trump over extracted names. For multiple key/certificate pairs, use the "+
+		"--tls-sni-cert-key multiple times. "+
+		"Examples: \"example.key,example.crt\" or \"*.foo.com,foo.com:foo.key,foo.crt\".")
 
 	fs.StringVar(&s.TokenAuthFile, "token-auth-file", s.TokenAuthFile, ""+
 		"If set, the file that will be used to secure the secure port of the API server "+

pkg/genericapiserver/serve.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package genericapiserver
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	certutil "k8s.io/kubernetes/pkg/util/cert"
+	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+)
+
+const (
+	defaultKeepAlivePeriod = 3 * time.Minute
+)
+
+// serveSecurely runs the secure http server. It fails only if certificates cannot
+// be loaded or the initial listen call fails. The actual server loop (stoppable by closing
+// stopCh) runs in a go routine, i.e. serveSecurely does not block.
+func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
+	namedCerts, err := getNamedCertificateMap(s.SecureServingInfo.SNICerts)
+	if err != nil {
+		return fmt.Errorf("unable to load SNI certificates: %v", err)
+	}
+
+	secureServer := &http.Server{
+		Addr:           s.SecureServingInfo.BindAddress,
+		Handler:        s.Handler,
+		MaxHeaderBytes: 1 << 20,
+		TLSConfig: &tls.Config{
+			NameToCertificate: namedCerts,
+			// Can't use SSLv3 because of POODLE and BEAST
+			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+			// Can't use TLSv1.1 because of RC4 cipher usage
+			MinVersion: tls.VersionTLS12,
+			// enable HTTP2 for go's 1.7 HTTP Server
+			NextProtos: []string{"h2", "http/1.1"},
+		},
+	}
+
+	if len(s.SecureServingInfo.ServerCert.CertFile) != 0 || len(s.SecureServingInfo.ServerCert.KeyFile) != 0 {
+		secureServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
+		secureServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(s.SecureServingInfo.ServerCert.CertFile, s.SecureServingInfo.ServerCert.KeyFile)
+		if err != nil {
+			return fmt.Errorf("unable to load server certificate: %v", err)
+		}
+	}
+
+	// append all named certs. Otherwise, the go tls stack will think no SNI processing
+	// is necessary because there is only one cert anyway.
+	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
+	// cert will become the default cert. That's what we expect anyway.
+	for _, c := range namedCerts {
+		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
+	}
+
+	if len(s.SecureServingInfo.ClientCA) > 0 {
+		clientCAs, err := certutil.NewPool(s.SecureServingInfo.ClientCA)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file: %v", err)
+		}
+		// Populate PeerCertificates in requests, but don't reject connections without certificates
+		// This allows certificates to be validated by authenticators, while still allowing other auth types
+		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
+		// Specify allowed CAs for client certificates
+		secureServer.TLSConfig.ClientCAs = clientCAs
+	}
+
+	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
+	s.effectiveSecurePort, err = runServer(secureServer, stopCh)
+	return err
+}
+
+// serveInsecurely run the insecure http server. It fails only if the initial listen
+// call fails. The actual server loop (stoppable by closing stopCh) runs in a go
+// routine, i.e. serveInsecurely does not block.
+func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error {
+	insecureServer := &http.Server{
+		Addr:           s.InsecureServingInfo.BindAddress,
+		Handler:        s.InsecureHandler,
+		MaxHeaderBytes: 1 << 20,
+	}
+	glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
+	var err error
+	s.effectiveInsecurePort, err = runServer(insecureServer, stopCh)
+	return err
+}
+
+// runServer listens on the given port, then spawns a go-routine continuously serving
+// until the stopCh is closed. The port is returned. This function does not block.
+func runServer(server *http.Server, stopCh <-chan struct{}) (int, error) {
+	if len(server.Addr) == 0 {
+		return 0, errors.New("address cannot be empty")
+	}
+
+	// first listen is synchronous (fail early!)
+	ln, err := net.Listen("tcp", server.Addr)
+	if err != nil {
+		return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	lock := sync.Mutex{} // to avoid we close an old listener during a listen retry
+	go func() {
+		<-stopCh
+		lock.Lock()
+		defer lock.Unlock()
+		ln.Close()
+	}()
+
+	go func() {
+		defer utilruntime.HandleCrash()
+
+		for {
+			var listener net.Listener
+			listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
+			if server.TLSConfig != nil {
+				listener = tls.NewListener(listener, server.TLSConfig)
+			}
+
+			err := server.Serve(listener)
+			glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err)
+
+			// listen again
+			func() {
+				lock.Lock()
+				defer lock.Unlock()
+				for {
+					time.Sleep(15 * time.Second)
+
+					ln, err = net.Listen("tcp", server.Addr)
+					if err == nil {
+						return
+					}
+					select {
+					case <-stopCh:
+						return
+					default:
+					}
+					glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err)
+				}
+			}()
+
+			select {
+			case <-stopCh:
+				return
+			default:
+			}
+		}
+	}()
+
+	return tcpAddr.Port, nil
+}
+
+// getNamedCertificateMap returns a map of strings to *tls.Certificate, suitable for use in
+// tls.Config#NamedCertificates. Returns an error if any of the certs cannot be loaded.
+// Returns nil if len(namedCertKeys) == 0
+func getNamedCertificateMap(namedCertKeys []NamedCertKey) (map[string]*tls.Certificate, error) {
+	if len(namedCertKeys) == 0 {
+		return nil, nil
+	}
+
+	// load keys
+	tlsCerts := make([]tls.Certificate, len(namedCertKeys))
+	for i := range namedCertKeys {
+		var err error
+		nkc := &namedCertKeys[i]
+		tlsCerts[i], err = tls.LoadX509KeyPair(nkc.CertFile, nkc.KeyFile)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// register certs with implicit names first, reverse order such that earlier trump over the later
+	tlsCertsByName := map[string]*tls.Certificate{}
+	for i := len(namedCertKeys) - 1; i >= 0; i-- {
+		nkc := &namedCertKeys[i]
+		if len(nkc.Names) > 0 {
+			continue
+		}
+		cert := &tlsCerts[i]
+
+		// read names from certificate common names and DNS names
+		if len(cert.Certificate) == 0 {
+			return nil, fmt.Errorf("no certificate found in %q", nkc.CertFile)
+		}
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return nil, fmt.Errorf("parse error for certificate in %q: %v", nkc.CertFile, err)
+		}
+		if len(x509Cert.Subject.CommonName) > 0 {
+			tlsCertsByName[x509Cert.Subject.CommonName] = cert
+		}
+		for _, san := range x509Cert.DNSNames {
+			tlsCertsByName[san] = cert
+		}
+		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
+		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
+	}
+
+	// register certs with explicit names last, overwriting every of the implicit ones,
+	// again in reverse order.
+	for i := len(namedCertKeys) - 1; i >= 0; i-- {
+		nkc := &namedCertKeys[i]
+		if len(nkc.Names) == 0 {
+			continue
+		}
+		for _, name := range nkc.Names {
+			tlsCertsByName[name] = &tlsCerts[i]
+		}
+	}
+
+	return tlsCertsByName, nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+//
+// Copied from Go 1.7.2 net/http/server.go
+type tcpKeepAliveListener struct {
+	*net.TCPListener
+}
+
+func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
+	tc, err := ln.AcceptTCP()
+	if err != nil {
+		return nil, err
+	}
+	tc.SetKeepAlive(true)
+	tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
+	return tc, nil
+}

pkg/util/cert/cert.go

@@ -126,22 +126,19 @@ func MakeEllipticPrivateKeyPEM() ([]byte, error) {
 	return pem.EncodeToMemory(privateKeyPemBlock), nil
 }
 
-// GenerateSelfSignedCert creates a self-signed certificate and key for the given host.
+// GenerateSelfSignedCertKey creates a self-signed certificate and key for the given host.
 // Host may be an IP or a DNS name
 // You may also specify additional subject alt names (either ip or dns names) for the certificate
-// The certificate will be created with file mode 0644. The key will be created with file mode 0600.
-// If the certificate or key files already exist, they will be overwritten.
-// Any parent directories of the certPath or keyPath will be created as needed with file mode 0755.
-func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.IP, alternateDNS []string) error {
+func GenerateSelfSignedCertKey(host string, alternateIPs []net.IP, alternateDNS []string) ([]byte, []byte, error) {
 	priv, err := rsa.GenerateKey(cryptorand.Reader, 2048)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	template := x509.Certificate{
 		SerialNumber: big.NewInt(1),
 		Subject: pkix.Name{
-			CommonName: fmt.Sprintf("%s@%d", host, time.Now().Unix()),
+			CommonName: host,
 		},
 		NotBefore: time.Now(),
 		NotAfter:  time.Now().Add(time.Hour * 24 * 365),
@@ -163,30 +160,22 @@ func GenerateSelfSignedCert(host, certPath, keyPath string, alternateIPs []net.I
 
 	derBytes, err := x509.CreateCertificate(cryptorand.Reader, &template, &template, &priv.PublicKey, priv)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	// Generate cert
 	certBuffer := bytes.Buffer{}
 	if err := pem.Encode(&certBuffer, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	// Generate key
 	keyBuffer := bytes.Buffer{}
 	if err := pem.Encode(&keyBuffer, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
-		return err
+		return nil, nil, err
 	}
 
-	if err := WriteCert(certPath, certBuffer.Bytes()); err != nil {
-		return err
-	}
-
-	if err := WriteKey(keyPath, keyBuffer.Bytes()); err != nil {
-		return err
-	}
-
-	return nil
+	return certBuffer.Bytes(), keyBuffer.Bytes(), nil
 }
 
 // FormatBytesCert receives byte array certificate and formats in human-readable format

pkg/util/config/BUILD

@@ -17,6 +17,7 @@ go_library(
         "configuration_map.go",
         "doc.go",
         "feature_gate.go",
+        "namedcertkey_flag.go",
     ],
     tags = ["automanaged"],
     deps = [
@@ -31,6 +32,7 @@ go_test(
     srcs = [
         "config_test.go",
         "feature_gate_test.go",
+        "namedcertkey_flag_test.go",
     ],
     library = "go_default_library",
     tags = ["automanaged"],

pkg/util/config/namedcertkey_flag.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"errors"
+	"flag"
+	"strings"
+)
+
+// NamedCertKey is a flag value parsing "certfile,keyfile" and "certfile,keyfile:name,name,name".
+type NamedCertKey struct {
+	Names             []string
+	CertFile, KeyFile string
+}
+
+var _ flag.Value = &NamedCertKey{}
+
+func (nkc *NamedCertKey) String() string {
+	s := nkc.CertFile + "," + nkc.KeyFile
+	if len(nkc.Names) > 0 {
+		s = s + ":" + strings.Join(nkc.Names, ",")
+	}
+	return s
+}
+
+func (nkc *NamedCertKey) Set(value string) error {
+	cs := strings.SplitN(value, ":", 2)
+	var keycert string
+	if len(cs) == 2 {
+		var names string
+		keycert, names = strings.TrimSpace(cs[0]), strings.TrimSpace(cs[1])
+		if names == "" {
+			return errors.New("empty names list is not allowed")
+		}
+		nkc.Names = nil
+		for _, name := range strings.Split(names, ",") {
+			nkc.Names = append(nkc.Names, strings.TrimSpace(name))
+		}
+	} else {
+		nkc.Names = nil
+		keycert = strings.TrimSpace(cs[0])
+	}
+	cs = strings.Split(keycert, ",")
+	if len(cs) != 2 {
+		return errors.New("expected comma separated certificate and key file paths")
+	}
+	nkc.CertFile = strings.TrimSpace(cs[0])
+	nkc.KeyFile = strings.TrimSpace(cs[1])
+	return nil
+}
+
+func (*NamedCertKey) Type() string {
+	return "namedCertKey"
+}
+
+// NamedCertKeyArray is a flag value parsing NamedCertKeys, each passed with its own
+// flag instance (in contrast to comma separated slices).
+type NamedCertKeyArray struct {
+	value   *[]NamedCertKey
+	changed bool
+}
+
+var _ flag.Value = &NamedCertKey{}
+
+// NewNamedKeyCertArray creates a new NamedCertKeyArray with the internal value
+// pointing to p.
+func NewNamedCertKeyArray(p *[]NamedCertKey) *NamedCertKeyArray {
+	return &NamedCertKeyArray{
+		value: p,
+	}
+}
+
+func (a *NamedCertKeyArray) Set(val string) error {
+	nkc := NamedCertKey{}
+	err := nkc.Set(val)
+	if err != nil {
+		return err
+	}
+	if !a.changed {
+		*a.value = []NamedCertKey{nkc}
+		a.changed = true
+	} else {
+		*a.value = append(*a.value, nkc)
+	}
+	return nil
+}
+
+func (a *NamedCertKeyArray) Type() string {
+	return "namedCertKey"
+}
+
+func (a *NamedCertKeyArray) String() string {
+	nkcs := make([]string, 0, len(*a.value))
+	for i := range *a.value {
+		nkcs = append(nkcs, (*a.value)[i].String())
+	}
+	return "[" + strings.Join(nkcs, ";") + "]"
+}

pkg/util/config/namedcertkey_flag_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"github.com/spf13/pflag"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestNamedCertKeyArrayFlag(t *testing.T) {
+	tests := []struct {
+		args       []string
+		def        []NamedCertKey
+		expected   []NamedCertKey
+		parseError string
+	}{
+		{
+			args:     []string{},
+			expected: nil,
+		},
+		{
+			args: []string{"foo.crt,foo.key"},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+			}},
+		},
+		{
+			args: []string{"  foo.crt , foo.key    "},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+			}},
+		},
+		{
+			args: []string{"foo.crt,foo.key:abc"},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+				Names:    []string{"abc"},
+			}},
+		},
+		{
+			args: []string{"foo.crt,foo.key: abc  "},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+				Names:    []string{"abc"},
+			}},
+		},
+		{
+			args:       []string{"foo.crt,foo.key:"},
+			parseError: "empty names list is not allowed",
+		},
+		{
+			args:       []string{""},
+			parseError: "expected comma separated certificate and key file paths",
+		},
+		{
+			args:       []string{"   "},
+			parseError: "expected comma separated certificate and key file paths",
+		},
+		{
+			args:       []string{"a,b,c"},
+			parseError: "expected comma separated certificate and key file paths",
+		},
+		{
+			args: []string{"foo.crt,foo.key:abc,def,ghi"},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+				Names:    []string{"abc", "def", "ghi"},
+			}},
+		},
+		{
+			args: []string{"foo.crt,foo.key:*.*.*"},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+				Names:    []string{"*.*.*"},
+			}},
+		},
+		{
+			args: []string{"foo.crt,foo.key", "bar.crt,bar.key"},
+			expected: []NamedCertKey{{
+				KeyFile:  "foo.key",
+				CertFile: "foo.crt",
+			}, {
+				KeyFile:  "bar.key",
+				CertFile: "bar.crt",
+			}},
+		},
+	}
+	for i, test := range tests {
+		fs := pflag.NewFlagSet("testNamedCertKeyArray", pflag.ContinueOnError)
+		var nkcs []NamedCertKey
+		for _, d := range test.def {
+			nkcs = append(nkcs, d)
+		}
+		fs.Var(NewNamedCertKeyArray(&nkcs), "tls-sni-cert-key", "usage")
+
+		args := []string{}
+		for _, a := range test.args {
+			args = append(args, fmt.Sprintf("--tls-sni-cert-key=%s", a))
+		}
+
+		err := fs.Parse(args)
+		if test.parseError != "" {
+			if err == nil {
+				t.Errorf("%d: expected error %q, got nil", i, test.parseError)
+			} else if !strings.Contains(err.Error(), test.parseError) {
+				t.Errorf("%d: expected error %q, got %q", i, test.parseError, err)
+			}
+		} else if err != nil {
+			t.Errorf("%d: expected nil error, got %v", i, err)
+		}
+		if !reflect.DeepEqual(nkcs, test.expected) {
+			t.Errorf("%d: expected %+v, got %+v", i, test.expected, nkcs)
+		}
+	}
+}

test/integration/discoverysummarizer/discoverysummarizer_test.go

@@ -65,12 +65,13 @@ func runDiscoverySummarizer(t *testing.T) string {
 	return serverURL
 }
 
-func runAPIServer(t *testing.T) string {
+func runAPIServer(t *testing.T, stopCh <-chan struct{}) string {
 	serverRunOptions := apiserver.NewServerRunOptions()
-	// Change the port, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel.
-	serverRunOptions.InsecurePort = 8083
+	// Change the ports, because otherwise it will fail if examples/apiserver/apiserver_test and this are run in parallel.
+	serverRunOptions.SecurePort = 6443 + 3
+	serverRunOptions.InsecurePort = 8080 + 3
 	go func() {
-		if err := apiserver.Run(serverRunOptions); err != nil {
+		if err := apiserver.Run(serverRunOptions, stopCh); err != nil {
 			t.Fatalf("Error in bringing up the example apiserver: %v", err)
 		}
 	}()
@@ -98,7 +99,9 @@ func TestRunDiscoverySummarizer(t *testing.T) {
 	testResponse(t, discoveryURL, "/randomPath", http.StatusNotFound)
 
 	// Run the APIServer now to test the good case.
-	runAPIServer(t)
+	stopCh := make(chan struct{})
+	runAPIServer(t, stopCh)
+	defer close(stopCh)
 
 	// Test /api path.
 	// There is no server running at that URL, so we will get a 500.

test/integration/examples/apiserver_test.go

@@ -42,11 +42,13 @@ var groupVersionForDiscovery = unversioned.GroupVersionForDiscovery{
 
 func TestRunServer(t *testing.T) {
 	serverIP := fmt.Sprintf("http://localhost:%d", apiserver.InsecurePort)
+	stopCh := make(chan struct{})
 	go func() {
-		if err := apiserver.Run(apiserver.NewServerRunOptions()); err != nil {
+		if err := apiserver.Run(apiserver.NewServerRunOptions(), stopCh); err != nil {
 			t.Fatalf("Error in bringing up the server: %v", err)
 		}
 	}()
+	defer close(stopCh)
 	if err := waitForApiserverUp(serverIP); err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -58,14 +60,16 @@ func TestRunServer(t *testing.T) {
 
 func TestRunSecureServer(t *testing.T) {
 	serverIP := fmt.Sprintf("https://localhost:%d", apiserver.SecurePort)
+	stopCh := make(chan struct{})
 	go func() {
 		options := apiserver.NewServerRunOptions()
 		options.InsecurePort = 0
 		options.SecurePort = apiserver.SecurePort
-		if err := apiserver.Run(options); err != nil {
+		if err := apiserver.Run(options, stopCh); err != nil {
 			t.Fatalf("Error in bringing up the server: %v", err)
 		}
 	}()
+	defer close(stopCh)
 	if err := waitForApiserverUp(serverIP); err != nil {
 		t.Fatalf("%v", err)
 	}

test/integration/federation/server_test.go

@@ -77,7 +77,8 @@ func TestLongRunningRequestRegexp(t *testing.T) {
 	}
 }
 
-var insecurePort = 8082
+var securePort = 6443 + 2
+var insecurePort = 8080 + 2
 var serverIP = fmt.Sprintf("http://localhost:%v", insecurePort)
 var groupVersions = []unversioned.GroupVersion{
 	fed_v1b1.SchemeGroupVersion,
@@ -86,6 +87,7 @@ var groupVersions = []unversioned.GroupVersion{
 
 func TestRun(t *testing.T) {
 	s := options.NewServerRunOptions()
+	s.GenericServerRunOptions.SecurePort = securePort
 	s.GenericServerRunOptions.InsecurePort = insecurePort
 	_, ipNet, _ := net.ParseCIDR("10.10.10.0/24")
 	s.GenericServerRunOptions.ServiceClusterIPRange = *ipNet