Allow kube-scheduler to tolerate cluster auth config lookup failure

cmd/kube-scheduler/app/options/options.go

@@ -104,6 +104,7 @@ func NewOptions() (*Options, error) {
 		},
 	}
 
+	o.Authentication.TolerateInClusterLookupFailure = true
 	o.Authentication.RemoteKubeConfigFileOptional = true
 	o.Authorization.RemoteKubeConfigFileOptional = true
 	o.Authorization.AlwaysAllowPaths = []string{"/healthz"}

staging/src/k8s.io/apiserver/pkg/server/options/BUILD

@@ -123,6 +123,7 @@ go_test(
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
     ],
 )
 

staging/src/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -117,7 +117,12 @@ type DelegatingAuthenticationOptions struct {
 	ClientCert    ClientCertAuthenticationOptions
 	RequestHeader RequestHeaderAuthenticationOptions
 
+	// SkipInClusterLookup indicates missing authentication configuration should not be retrieved from the cluster configmap
 	SkipInClusterLookup bool
+
+	// TolerateInClusterLookupFailure indicates failures to look up authentication configuration from the cluster configmap should not be fatal.
+	// Setting this can result in an authenticator that will reject all requests.
+	TolerateInClusterLookupFailure bool
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -160,6 +165,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.SkipInClusterLookup, "authentication-skip-lookup", s.SkipInClusterLookup, ""+
 		"If false, the authentication-kubeconfig will be used to lookup missing authentication "+
 		"configuration from the cluster.")
+	fs.BoolVar(&s.TolerateInClusterLookupFailure, "authentication-tolerate-lookup-failure", s.TolerateInClusterLookupFailure, ""+
+		"If true, failures to look up missing authentication configuration from the cluster are not considered fatal. "+
+		"Note that this can result in authentication that treats all requests as anonymous.")
 }
 
 func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
@@ -187,7 +195,13 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 	if !s.SkipInClusterLookup {
 		err := s.lookupMissingConfigInCluster(client)
 		if err != nil {
-			return err
+			if s.TolerateInClusterLookupFailure {
+				klog.Warningf("Error looking up in-cluster authentication configuration: %v", err)
+				klog.Warningf("Continuing without authentication configuration. This may treat all requests as anonymous.")
+				klog.Warningf("To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false")
+			} else {
+				return err
+			}
 		}
 	}
 

staging/src/k8s.io/apiserver/pkg/server/options/authentication_test.go

@@ -17,10 +17,15 @@ limitations under the License.
 package options
 
 import (
+	"io/ioutil"
+	"net/http"
+	"os"
 	"reflect"
 	"testing"
 
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/server"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
 
 func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
@@ -66,3 +71,131 @@ func TestToAuthenticationRequestHeaderConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestApplyToFallback(t *testing.T) {
+
+	f, err := ioutil.TempFile("", "authkubeconfig")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+
+	if err := ioutil.WriteFile(f.Name(), []byte(`
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: http://localhost:56789
+  name: cluster
+contexts:
+- context:
+    cluster: cluster
+  name: cluster
+current-context: cluster
+`), os.FileMode(0755)); err != nil {
+		t.Fatal(err)
+	}
+	remoteKubeconfig := f.Name()
+
+	testcases := []struct {
+		name                 string
+		options              *DelegatingAuthenticationOptions
+		expectError          bool
+		expectAuthenticator  bool
+		expectTokenAnonymous bool
+		expectTokenErrors    bool
+	}{
+		{
+			name:                "empty",
+			options:             nil,
+			expectError:         false,
+			expectAuthenticator: false,
+		},
+		{
+			name:                "default",
+			options:             NewDelegatingAuthenticationOptions(),
+			expectError:         true, // in-cluster client building fails, no kubeconfig provided
+			expectAuthenticator: false,
+		},
+		{
+			name: "optional kubeconfig",
+			options: func() *DelegatingAuthenticationOptions {
+				opts := NewDelegatingAuthenticationOptions()
+				opts.RemoteKubeConfigFileOptional = true
+				return opts
+			}(),
+			expectError:          false, // in-cluster client building fails, no kubeconfig required
+			expectAuthenticator:  true,
+			expectTokenAnonymous: true, // no token validator available
+		},
+		{
+			name: "valid client, failed cluster info lookup",
+			options: func() *DelegatingAuthenticationOptions {
+				opts := NewDelegatingAuthenticationOptions()
+				opts.RemoteKubeConfigFile = remoteKubeconfig
+				return opts
+			}(),
+			expectError:         true, // client building is valid, remote config lookup fails
+			expectAuthenticator: false,
+		},
+		{
+			name: "valid client, skip cluster info lookup",
+			options: func() *DelegatingAuthenticationOptions {
+				opts := NewDelegatingAuthenticationOptions()
+				opts.RemoteKubeConfigFile = remoteKubeconfig
+				opts.SkipInClusterLookup = true
+				return opts
+			}(),
+			expectError:         false, // client building is valid, skipped cluster lookup
+			expectAuthenticator: true,
+			expectTokenErrors:   true, // client fails making tokenreview calls
+		},
+		{
+			name: "valid client, tolerate failed cluster info lookup",
+			options: func() *DelegatingAuthenticationOptions {
+				opts := NewDelegatingAuthenticationOptions()
+				opts.RemoteKubeConfigFile = remoteKubeconfig
+				opts.TolerateInClusterLookupFailure = true
+				return opts
+			}(),
+			expectError:         false, // client is valid, skipped cluster lookup
+			expectAuthenticator: true,  // anonymous auth
+			expectTokenErrors:   true,  // client fails making tokenreview calls
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			c := &server.AuthenticationInfo{}
+			servingInfo := &server.SecureServingInfo{}
+			openAPIConfig := &openapicommon.Config{}
+
+			err := tc.options.ApplyTo(c, servingInfo, openAPIConfig)
+			if (err != nil) != tc.expectError {
+				t.Errorf("expected error=%v, got %v", tc.expectError, err)
+			}
+			if (c.Authenticator != nil) != tc.expectAuthenticator {
+				t.Errorf("expected authenticator=%v, got %#v", tc.expectError, c.Authenticator)
+			}
+			if c.Authenticator != nil {
+				{
+					result, ok, err := c.Authenticator.AuthenticateRequest(&http.Request{})
+					if err != nil || !ok || result == nil || result.User.GetName() != "system:anonymous" {
+						t.Errorf("expected anonymous, got %#v, %#v, %#v", result, ok, err)
+					}
+				}
+				{
+					result, ok, err := c.Authenticator.AuthenticateRequest(&http.Request{Header: http.Header{"Authorization": []string{"Bearer foo"}}})
+					if tc.expectTokenAnonymous {
+						if err != nil || !ok || result == nil || result.User.GetName() != "system:anonymous" {
+							t.Errorf("expected anonymous, got %#v, %#v, %#v", result, ok, err)
+						}
+					}
+					if tc.expectTokenErrors != (err != nil) {
+						t.Errorf("expected error=%v, got %#v, %#v, %#v", tc.expectTokenErrors, result, ok, err)
+					}
+				}
+			}
+		})
+	}
+}