Commit b721a3e0 authored by Brad Davidson's avatar Brad Davidson Committed by Brad Davidson

Fix 10 second etcd-snapshot request timeout

The default clientaccess request timeout is too short. Wait longer by default, and add the s3 timeout if s3 is enabled. Signed-off-by: 's avatarBrad Davidson <brad.davidson@rancher.com> (cherry picked from commit d3b60543)
parent 08a1ee51
...@@ -2,6 +2,7 @@ package etcdsnapshot ...@@ -2,6 +2,7 @@ package etcdsnapshot
import ( import (
"bytes" "bytes"
"context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"os" "os"
...@@ -26,6 +27,8 @@ import ( ...@@ -26,6 +27,8 @@ import (
"k8s.io/cli-runtime/pkg/printers" "k8s.io/cli-runtime/pkg/printers"
) )
var timeout = 2 * time.Minute
// commandSetup setups up common things needed // commandSetup setups up common things needed
// for each etcd command. // for each etcd command.
func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) { func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *clientaccess.Info, error) {
...@@ -58,6 +61,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c ...@@ -58,6 +61,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
sr.S3.SecretKey = cfg.EtcdS3SecretKey sr.S3.SecretKey = cfg.EtcdS3SecretKey
sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify sr.S3.SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
sr.S3.Timeout = metav1.Duration{Duration: cfg.EtcdS3Timeout} sr.S3.Timeout = metav1.Duration{Duration: cfg.EtcdS3Timeout}
// extend request timeout to allow the S3 operation to complete
timeout += cfg.EtcdS3Timeout
} }
dataDir, err := server.ResolveDataDir(cfg.DataDir) dataDir, err := server.ResolveDataDir(cfg.DataDir)
...@@ -78,6 +83,11 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c ...@@ -78,6 +83,11 @@ func commandSetup(app *cli.Context, cfg *cmds.Server) (*etcd.SnapshotRequest, *c
} }
func wrapServerError(err error) error { func wrapServerError(err error) error {
if errors.Is(err, context.DeadlineExceeded) {
// if the request timed out the server log likely won't contain anything useful,
// since the operation may have actualy succeeded despite the client timing out the request.
return err
}
return errors.Wrap(err, "see server log for details") return errors.Wrap(err, "see server log for details")
} }
...@@ -110,7 +120,7 @@ func save(app *cli.Context, cfg *cmds.Server) error { ...@@ -110,7 +120,7 @@ func save(app *cli.Context, cfg *cmds.Server) error {
if err != nil { if err != nil {
return err return err
} }
r, err := info.Post("/db/snapshot", b) r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
if err != nil { if err != nil {
return wrapServerError(err) return wrapServerError(err)
} }
...@@ -151,7 +161,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error { ...@@ -151,7 +161,7 @@ func delete(app *cli.Context, cfg *cmds.Server) error {
if err != nil { if err != nil {
return err return err
} }
r, err := info.Post("/db/snapshot", b) r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
if err != nil { if err != nil {
return wrapServerError(err) return wrapServerError(err)
} }
...@@ -206,7 +216,7 @@ func list(app *cli.Context, cfg *cmds.Server) error { ...@@ -206,7 +216,7 @@ func list(app *cli.Context, cfg *cmds.Server) error {
if err != nil { if err != nil {
return err return err
} }
r, err := info.Post("/db/snapshot", b) r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
if err != nil { if err != nil {
return wrapServerError(err) return wrapServerError(err)
} }
...@@ -269,7 +279,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error { ...@@ -269,7 +279,7 @@ func prune(app *cli.Context, cfg *cmds.Server) error {
if err != nil { if err != nil {
return err return err
} }
r, err := info.Post("/db/snapshot", b) r, err := info.Post("/db/snapshot", b, clientaccess.WithTimeout(timeout))
if err != nil { if err != nil {
return wrapServerError(err) return wrapServerError(err)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment