Merge pull request #53947 from crassirostris/retry-webhook-net-errors

Automatic merge from submit-queue (batch tested with PRs 53958, 53947). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Always retry connection reset error in webhook Fixes https://github.com/kubernetes/kubernetes/issues/52909 Audit logging uses webhook to send events to the backend and currently even a little blip in networking can cause several hundreds of events to be lost. This PR adds an additional check, that is similar to [the one in the rest package](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/client-go/rest/request.go#L657), but ignores the fact that the request is not GET and always retries "Connection reset by peers" error. ```release-note Webhook always retries connection reset error. ```

Merge pull request #53947 from crassirostris/retry-webhook-net-errors
14a1a154 · Kubernetes Submit Queue · GitHub · 1e356871 · 59bacba0 · 14a1a154
Commit 14a1a154 authored Oct 18, 2017 by Kubernetes Submit Queue Committed by GitHub Oct 18, 2017
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 5 deletions

BUILD staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD +1 -0

webhook.go staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go +3 -5

No files found.
--- a/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD
+++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/BUILD
@@ -16,6 +16,7 @@ go_library(
        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
        "//vendor/k8s.io/client-go/rest:go_default_library",
        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",

--- a/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go
+++ b/staging/src/k8s.io/apiserver/pkg/util/webhook/webhook.go
@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -104,17 +105,14 @@ func WithExponentialBackoff(initialBackoff time.Duration, webhookFn func() error
 	var err error
 	wait.ExponentialBackoff(backoff, func() (bool, error) {
 		err = webhookFn()
-		// these errors indicate a need to retry an authentication check
+		// these errors indicate a transient error that should be retried.
-		if apierrors.IsServerTimeout(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) {
+		if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) {
 			return false, nil
 		}
 		// if the error sends the Retry-After header, we respect it as an explicit confirmation we should retry.
 		if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry {
 			return false, nil
 		}
-		if apierrors.IsInternalError(err) {
-			return false, nil
-		}
 		if err != nil {
 			return false, err
 		}