Move etcd snapshot management CLI to request/response

go.mod

@@ -151,6 +151,7 @@ require (
 	k8s.io/api v0.29.3
 	k8s.io/apimachinery v0.29.3
 	k8s.io/apiserver v0.29.3
+	k8s.io/cli-runtime v0.22.2
 	k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
 	k8s.io/cloud-provider v0.29.3
 	k8s.io/cluster-bootstrap v0.0.0
@@ -485,7 +486,6 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.3 // indirect
-	k8s.io/cli-runtime v0.22.2 // indirect
 	k8s.io/code-generator v0.29.3 // indirect
 	k8s.io/controller-manager v0.25.4 // indirect
 	k8s.io/csi-translation-lib v0.0.0 // indirect

pkg/cli/cmds/etcd_snapshot.go

@@ -21,6 +21,14 @@ var EtcdSnapshotFlags = []cli.Flag{
 		Destination: &AgentConfig.NodeName,
 	},
 	DataDirFlag,
+	ServerToken,
+	&cli.StringFlag{
+		Name:        "server, s",
+		Usage:       "(cluster) Server to connect to",
+		EnvVar:      version.ProgramUpper + "_URL",
+		Value:       "https://127.0.0.1:6443",
+		Destination: &ServerConfig.ServerURL,
+	},
 	&cli.StringFlag{
 		Name:        "dir,etcd-snapshot-dir",
 		Usage:       "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -66,7 +66,6 @@ func Enable(app *cli.Context) error {
 }
 
 func Disable(app *cli.Context) error {
-
 	if err := cmds.InitLogging(); err != nil {
 		return err
 	}

pkg/clientaccess/token.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -18,6 +19,9 @@ import (
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/sirupsen/logrus"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/net"
 )
 
 const (
@@ -291,7 +295,6 @@ func (i *Info) Get(path string, option ...ClientOption) ([]byte, error) {
 
 // Put makes a request to a subpath of info's BaseURL
 func (i *Info) Put(path string, body []byte, option ...ClientOption) error {
-
 	u, err := url.Parse(i.BaseURL)
 	if err != nil {
 		return err
@@ -305,6 +308,21 @@ func (i *Info) Put(path string, body []byte, option ...ClientOption) error {
 	return put(p.String(), body, GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile, option...), i.Username, i.Password, i.Token())
 }
 
+// Post makes a request to a subpath of info's BaseURL
+func (i *Info) Post(path string, body []byte, option ...ClientOption) ([]byte, error) {
+	u, err := url.Parse(i.BaseURL)
+	if err != nil {
+		return nil, err
+	}
+	p, err := url.Parse(path)
+	if err != nil {
+		return nil, err
+	}
+	p.Scheme = u.Scheme
+	p.Host = u.Host
+	return post(p.String(), body, GetHTTPClient(i.CACerts, i.CertFile, i.KeyFile, option...), i.Username, i.Password, i.Token())
+}
+
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
 // and storing the CA bundle.
 func (i *Info) setServer(server string) error {
@@ -400,13 +418,8 @@ func get(u string, client *http.Client, username, password, token string) ([]byt
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return nil, fmt.Errorf("%s: %s", u, resp.Status)
-	}
 
-	return io.ReadAll(resp.Body)
+	return readBody(resp)
 }
 
 // put makes a request to a url using a provided client and credentials,
@@ -427,14 +440,59 @@ func put(u string, body []byte, client *http.Client, username, password, token s
 	if err != nil {
 		return err
 	}
+
+	_, err = readBody(resp)
+	return err
+}
+
+// post makes a request to a url using a provided client and credentials,
+// returning the response body and error.
+func post(u string, body []byte, client *http.Client, username, password, token string) ([]byte, error) {
+	req, err := http.NewRequest(http.MethodPost, u, bytes.NewBuffer(body))
+	if err != nil {
+		return nil, err
+	}
+
+	if token != "" {
+		req.Header.Add("Authorization", "Bearer "+token)
+	} else if username != "" {
+		req.SetBasicAuth(username, password)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	return readBody(resp)
+}
+
+// readBody attempts to get the body from the response. If the response status
+// code is not in the 2XX range, an error is returned. An attempt is made to
+// decode the error body as a metav1.Status and return a StatusError, if
+// possible.
+func readBody(resp *http.Response) ([]byte, error) {
 	defer resp.Body.Close()
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
 
-	respBody, _ := io.ReadAll(resp.Body)
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return fmt.Errorf("%s: %s %s", u, resp.Status, string(respBody))
+	warnings, _ := net.ParseWarningHeaders(resp.Header["Warning"])
+	for _, warning := range warnings {
+		if warning.Code == 299 && len(warning.Text) != 0 {
+			logrus.Warnf(warning.Text)
+		}
 	}
 
-	return nil
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		status := metav1.Status{}
+		if err := json.Unmarshal(b, &status); err == nil && status.Kind == "Status" {
+			return nil, &apierrors.StatusError{ErrStatus: status}
+		}
+		return nil, fmt.Errorf("%s: %s", resp.Request.URL, resp.Status)
+	}
+	return b, nil
 }
 
 // FormatToken takes a username:password string or join token, and a path to a certificate bundle, and

pkg/cluster/managed/drivers.go

@@ -23,7 +23,7 @@ type Driver interface {
 	Test(ctx context.Context) error
 	Restore(ctx context.Context) error
 	EndpointName() string
-	Snapshot(ctx context.Context) error
+	Snapshot(ctx context.Context) (*SnapshotResult, error)
 	ReconcileSnapshotData(ctx context.Context) error
 	GetMembersClientURLs(ctx context.Context) ([]string, error)
 	RemoveSelf(ctx context.Context) error
@@ -40,3 +40,10 @@ func Registered() []Driver {
 func Default() Driver {
 	return drivers[0]
 }
+
+// SnapshotResult is returned by the Snapshot function,
+// and lists the names of created and deleted snapshots.
+type SnapshotResult struct {
+	Created []string `json:"created,omitempty"`
+	Deleted []string `json:"deleted,omitempty"`
+}

pkg/configfilearg/defaultparser_test.go

@@ -48,7 +48,7 @@ func Test_UnitMustParse(t *testing.T) {
 			name:   "Etcd-snapshot with config with known and unknown flags",
 			args:   []string{"k3s", "etcd-snapshot", "save"},
 			config: "./testdata/defaultdata.yaml",
-			want:   []string{"k3s", "etcd-snapshot", "save", "--etcd-s3=true", "--etcd-s3-bucket=my-backup"},
+			want:   []string{"k3s", "etcd-snapshot", "save", "--token=12345", "--etcd-s3=true", "--etcd-s3-bucket=my-backup"},
 		},
 		{
 			name: "Agent with known flags",

pkg/etcd/etcd.go

@@ -25,6 +25,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	"github.com/k3s-io/k3s/pkg/server/auth"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/kine/pkg/client"
@@ -664,7 +665,9 @@ func (e *ETCD) setName(force bool) error {
 // handler wraps the handler with routes for database info
 func (e *ETCD) handler(next http.Handler) http.Handler {
 	mux := mux.NewRouter().SkipClean(true)
+	mux.Use(auth.Middleware(e.config, version.Program+":server"))
 	mux.Handle("/db/info", e.infoHandler())
+	mux.Handle("/db/snapshot", e.snapshotHandler())
 	mux.NotFoundHandler = next
 	return mux
 }
@@ -673,6 +676,11 @@ func (e *ETCD) handler(next http.Handler) http.Handler {
 // If we can't retrieve an actual MemberList from etcd, we return a canned response with only the local node listed.
 func (e *ETCD) infoHandler() http.Handler {
 	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if req.Method != http.MethodGet {
+			util.SendError(fmt.Errorf("method not allowed"), rw, req, http.StatusMethodNotAllowed)
+			return
+		}
+
 		ctx, cancel := context.WithTimeout(req.Context(), 2*time.Second)
 		defer cancel()
 

pkg/etcd/s3.go

@@ -93,7 +93,7 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) {
 
 	exists, err := c.BucketExists(ctx, config.EtcdS3BucketName)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "failed to test for existence of bucket %s", config.EtcdS3BucketName)
 	}
 	if !exists {
 		return nil, fmt.Errorf("bucket %s does not exist", config.EtcdS3BucketName)
@@ -280,9 +280,10 @@ func (s *S3) snapshotPrefix() string {
 }
 
 // snapshotRetention prunes snapshots in the configured S3 compatible backend for this specific node.
-func (s *S3) snapshotRetention(ctx context.Context) error {
+// Returns a list of pruned snapshot names.
+func (s *S3) snapshotRetention(ctx context.Context) ([]string, error) {
 	if s.config.EtcdSnapshotRetention < 1 {
-		return nil
+		return nil, nil
 	}
 	logrus.Infof("Applying snapshot retention=%d to snapshots stored in s3://%s/%s", s.config.EtcdSnapshotRetention, s.config.EtcdS3BucketName, s.snapshotPrefix())
 
@@ -297,7 +298,7 @@ func (s *S3) snapshotRetention(ctx context.Context) error {
 	}
 	for info := range s.client.ListObjects(toCtx, s.config.EtcdS3BucketName, opts) {
 		if info.Err != nil {
-			return info.Err
+			return nil, info.Err
 		}
 
 		// skip metadata
@@ -309,7 +310,7 @@ func (s *S3) snapshotRetention(ctx context.Context) error {
 	}
 
 	if len(snapshotFiles) <= s.config.EtcdSnapshotRetention {
-		return nil
+		return nil, nil
 	}
 
 	// sort newest-first so we can prune entries past the retention count
@@ -317,21 +318,16 @@ func (s *S3) snapshotRetention(ctx context.Context) error {
 		return snapshotFiles[j].LastModified.Before(snapshotFiles[i].LastModified)
 	})
 
+	deleted := []string{}
 	for _, df := range snapshotFiles[s.config.EtcdSnapshotRetention:] {
 		logrus.Infof("Removing S3 snapshot: s3://%s/%s", s.config.EtcdS3BucketName, df.Key)
-		if err := s.client.RemoveObject(toCtx, s.config.EtcdS3BucketName, df.Key, minio.RemoveObjectOptions{}); err != nil {
-			return err
-		}
-		metadataKey := path.Join(path.Dir(df.Key), metadataDir, path.Base(df.Key))
-		if err := s.client.RemoveObject(toCtx, s.config.EtcdS3BucketName, metadataKey, minio.RemoveObjectOptions{}); err != nil {
-			if isNotExist(err) {
-				return nil
-			}
-			return err
+		if err := s.deleteSnapshot(ctx, df.Key); err != nil {
+			return deleted, err
 		}
+		deleted = append(deleted, df.Key)
 	}
 
-	return nil
+	return deleted, nil
 }
 
 func (s *S3) deleteSnapshot(ctx context.Context, key string) error {

pkg/etcd/snapshot_handler.go

+package etcd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	"github.com/k3s-io/k3s/pkg/cluster/managed"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type SnapshotOperation string
+
+const (
+	SnapshotOperationSave   SnapshotOperation = "save"
+	SnapshotOperationList   SnapshotOperation = "list"
+	SnapshotOperationPrune  SnapshotOperation = "prune"
+	SnapshotOperationDelete SnapshotOperation = "delete"
+)
+
+type SnapshotRequestS3 struct {
+	s3Config
+	Timeout   metav1.Duration `json:"timeout"`
+	AccessKey string          `json:"accessKey"`
+	SecretKey string          `json:"secretKey"`
+}
+
+type SnapshotRequest struct {
+	Operation SnapshotOperation `json:"operation"`
+	Name      []string          `json:"name,omitempty"`
+	Dir       *string           `json:"dir,omitempty"`
+	Compress  *bool             `json:"compress,omitempty"`
+	Retention *int              `json:"retention,omitempty"`
+
+	S3 *SnapshotRequestS3 `json:"s3,omitempty"`
+
+	ctx context.Context
+}
+
+func (sr *SnapshotRequest) context() context.Context {
+	if sr.ctx != nil {
+		return sr.ctx
+	}
+	return context.Background()
+}
+
+// snapshotHandler handles snapshot save/list/prune requests from the CLI.
+func (e *ETCD) snapshotHandler() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		sr, err := getSnapshotRequest(req)
+		if err != nil {
+			util.SendErrorWithID(err, "etcd-snapshot", rw, req, http.StatusInternalServerError)
+		}
+		switch sr.Operation {
+		case SnapshotOperationList:
+			err = e.withRequest(sr).handleList(rw, req)
+		case SnapshotOperationSave:
+			err = e.withRequest(sr).handleSave(rw, req)
+		case SnapshotOperationPrune:
+			err = e.withRequest(sr).handlePrune(rw, req)
+		case SnapshotOperationDelete:
+			err = e.withRequest(sr).handleDelete(rw, req, sr.Name)
+		default:
+			err = e.handleInvalid(rw, req)
+		}
+		if err != nil {
+			logrus.Warnf("Error in etcd-snapshot handler: %v", err)
+		}
+	})
+}
+
+func (e *ETCD) handleList(rw http.ResponseWriter, req *http.Request) error {
+	if err := e.initS3IfNil(req.Context()); err != nil {
+		util.SendError(err, rw, req, http.StatusBadRequest)
+		return nil
+	}
+	sf, err := e.ListSnapshots(req.Context())
+	if sf == nil {
+		util.SendErrorWithID(err, "etcd-snapshot", rw, req, http.StatusInternalServerError)
+		return nil
+	}
+	sendSnapshotList(rw, req, sf)
+	return err
+}
+
+func (e *ETCD) handleSave(rw http.ResponseWriter, req *http.Request) error {
+	if err := e.initS3IfNil(req.Context()); err != nil {
+		util.SendError(err, rw, req, http.StatusBadRequest)
+		return nil
+	}
+	sr, err := e.Snapshot(req.Context())
+	if sr == nil {
+		util.SendErrorWithID(err, "etcd-snapshot", rw, req, http.StatusInternalServerError)
+		return nil
+	}
+	sendSnapshotResponse(rw, req, sr)
+	return err
+}
+
+func (e *ETCD) handlePrune(rw http.ResponseWriter, req *http.Request) error {
+	if err := e.initS3IfNil(req.Context()); err != nil {
+		util.SendError(err, rw, req, http.StatusBadRequest)
+		return nil
+	}
+	sr, err := e.PruneSnapshots(req.Context())
+	if sr == nil {
+		util.SendError(err, rw, req, http.StatusInternalServerError)
+		return nil
+	}
+	sendSnapshotResponse(rw, req, sr)
+	return err
+}
+
+func (e *ETCD) handleDelete(rw http.ResponseWriter, req *http.Request, snapshots []string) error {
+	if err := e.initS3IfNil(req.Context()); err != nil {
+		util.SendError(err, rw, req, http.StatusBadRequest)
+		return nil
+	}
+	sr, err := e.DeleteSnapshots(req.Context(), snapshots)
+	if sr == nil {
+		util.SendError(err, rw, req, http.StatusInternalServerError)
+		return nil
+	}
+	sendSnapshotResponse(rw, req, sr)
+	return err
+}
+
+func (e *ETCD) handleInvalid(rw http.ResponseWriter, req *http.Request) error {
+	util.SendErrorWithID(fmt.Errorf("invalid snapshot operation"), "etcd-snapshot", rw, req, http.StatusBadRequest)
+	return nil
+}
+
+// withRequest returns a modified ETCD struct that is overriden
+// with etcd snapshot config from the snapshot request.
+func (e *ETCD) withRequest(sr *SnapshotRequest) *ETCD {
+	re := &ETCD{
+		client: e.client,
+		config: &config.Control{
+			CriticalControlArgs:   e.config.CriticalControlArgs,
+			Runtime:               e.config.Runtime,
+			DataDir:               e.config.DataDir,
+			Datastore:             e.config.Datastore,
+			EtcdSnapshotCompress:  e.config.EtcdSnapshotCompress,
+			EtcdSnapshotName:      e.config.EtcdSnapshotName,
+			EtcdSnapshotRetention: e.config.EtcdSnapshotRetention,
+		},
+		name:        e.name,
+		address:     e.address,
+		cron:        e.cron,
+		cancel:      e.cancel,
+		snapshotSem: e.snapshotSem,
+	}
+	if len(sr.Name) > 0 {
+		re.config.EtcdSnapshotName = sr.Name[0]
+	}
+	if sr.Compress != nil {
+		re.config.EtcdSnapshotCompress = *sr.Compress
+	}
+	if sr.Dir != nil {
+		re.config.EtcdSnapshotDir = *sr.Dir
+	}
+	if sr.Retention != nil {
+		re.config.EtcdSnapshotRetention = *sr.Retention
+	}
+	if sr.S3 != nil {
+		re.config.EtcdS3 = true
+		re.config.EtcdS3BucketName = sr.S3.Bucket
+		re.config.EtcdS3AccessKey = sr.S3.AccessKey
+		re.config.EtcdS3SecretKey = sr.S3.SecretKey
+		re.config.EtcdS3Endpoint = sr.S3.Endpoint
+		re.config.EtcdS3EndpointCA = sr.S3.EndpointCA
+		re.config.EtcdS3SkipSSLVerify = sr.S3.SkipSSLVerify
+		re.config.EtcdS3Insecure = sr.S3.Insecure
+		re.config.EtcdS3Region = sr.S3.Region
+		re.config.EtcdS3Timeout = sr.S3.Timeout.Duration
+	}
+	return re
+}
+
+// getSnapshotRequest unmarshalls the snapshot operation request from a client.
+func getSnapshotRequest(req *http.Request) (*SnapshotRequest, error) {
+	if req.Method != http.MethodPost {
+		return nil, http.ErrNotSupported
+	}
+	sr := &SnapshotRequest{}
+	b, err := io.ReadAll(req.Body)
+	if err != nil {
+		return nil, err
+	}
+	if err := json.Unmarshal(b, &sr); err != nil {
+		return nil, err
+	}
+	sr.ctx = req.Context()
+	return sr, nil
+}
+
+func sendSnapshotResponse(rw http.ResponseWriter, req *http.Request, sr *managed.SnapshotResult) {
+	b, err := json.Marshal(sr)
+	if err != nil {
+		util.SendErrorWithID(err, "etcd-snapshot", rw, req, http.StatusInternalServerError)
+		return
+	}
+	rw.Header().Set("Content-Type", "application/json")
+	rw.Write(b)
+}
+
+func sendSnapshotList(rw http.ResponseWriter, req *http.Request, sf *k3s.ETCDSnapshotFileList) {
+	b, err := json.Marshal(sf)
+	if err != nil {
+		util.SendErrorWithID(err, "etcd-snapshot", rw, req, http.StatusInternalServerError)
+		return
+	}
+	rw.Header().Set("Content-Type", "application/json")
+	rw.Write(b)
+}

pkg/server/auth.go → pkg/server/auth/auth.go

-package server
+package auth
 
 import (
 	"net/http"
@@ -51,7 +51,7 @@ func doAuth(roles []string, serverConfig *config.Control, next http.Handler, rw
 	next.ServeHTTP(rw, req)
 }
 
-func authMiddleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
+func Middleware(serverConfig *config.Control, roles ...string) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 			doAuth(roles, serverConfig, next, rw, req)

pkg/server/cert.go

@@ -19,6 +19,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/cluster"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
+	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
@@ -35,7 +36,7 @@ func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
 		}
 		force, _ := strconv.ParseBool(req.FormValue("force"))
 		if err := caCertReplace(server, req.Body, force); err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err, "certificate")
+			util.SendErrorWithID(err, "certificate", resp, req, http.StatusInternalServerError)
 			return
 		}
 		logrus.Infof("certificate: Cluster Certificate Authority data has been updated, %s must be restarted.", version.Program)

pkg/server/router.go

@@ -20,6 +20,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/nodepassword"
+	"github.com/k3s-io/k3s/pkg/server/auth"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -51,7 +52,7 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	prefix := "/v1-" + version.Program
 	authed := mux.NewRouter().SkipClean(true)
-	authed.Use(authMiddleware(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
+	authed.Use(auth.Middleware(serverConfig, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
 	authed.Path(prefix + "/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig, serverConfig.Runtime.ServingKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kubelet.crt").Handler(clientKubeletCert(serverConfig, serverConfig.Runtime.ClientKubeletKey, nodeAuth))
 	authed.Path(prefix + "/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert, serverConfig.Runtime.ClientKubeProxyKey))
@@ -70,23 +71,22 @@ func router(ctx context.Context, config *Config, cfg *cmds.Server) http.Handler
 
 	nodeAuthed := mux.NewRouter().SkipClean(true)
 	nodeAuthed.NotFoundHandler = authed
-	nodeAuthed.Use(authMiddleware(serverConfig, user.NodesGroup))
+	nodeAuthed.Use(auth.Middleware(serverConfig, user.NodesGroup))
 	nodeAuthed.Path(prefix + "/connect").Handler(serverConfig.Runtime.Tunnel)
 
 	serverAuthed := mux.NewRouter().SkipClean(true)
 	serverAuthed.NotFoundHandler = nodeAuthed
-	serverAuthed.Use(authMiddleware(serverConfig, version.Program+":server"))
+	serverAuthed.Use(auth.Middleware(serverConfig, version.Program+":server"))
 	serverAuthed.Path(prefix + "/encrypt/status").Handler(encryptionStatusHandler(serverConfig))
 	serverAuthed.Path(prefix + "/encrypt/config").Handler(encryptionConfigHandler(ctx, serverConfig))
 	serverAuthed.Path(prefix + "/cert/cacerts").Handler(caCertReplaceHandler(serverConfig))
-	serverAuthed.Path("/db/info").Handler(nodeAuthed)
 	serverAuthed.Path(prefix + "/server-bootstrap").Handler(bootstrapHandler(serverConfig.Runtime))
 	serverAuthed.Path(prefix + "/token").Handler(tokenRequestHandler(ctx, serverConfig))
 
 	systemAuthed := mux.NewRouter().SkipClean(true)
 	systemAuthed.NotFoundHandler = serverAuthed
 	systemAuthed.MethodNotAllowedHandler = serverAuthed
-	systemAuthed.Use(authMiddleware(serverConfig, user.SystemPrivilegedGroup))
+	systemAuthed.Use(auth.Middleware(serverConfig, user.SystemPrivilegedGroup))
 	systemAuthed.Methods(http.MethodConnect).Handler(serverConfig.Runtime.Tunnel)
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")

pkg/server/secrets-encrypt.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"math/big"
 	"net/http"
 	"os"
 	"strings"
@@ -45,12 +44,12 @@ type EncryptionRequest struct {
 	Skip   bool    `json:"skip"`
 }
 
-func getEncryptionRequest(req *http.Request) (EncryptionRequest, error) {
+func getEncryptionRequest(req *http.Request) (*EncryptionRequest, error) {
 	b, err := io.ReadAll(req.Body)
 	if err != nil {
-		return EncryptionRequest{}, err
+		return nil, err
 	}
-	result := EncryptionRequest{}
+	result := &EncryptionRequest{}
 	err = json.Unmarshal(b, &result)
 	return result, err
 }
@@ -63,14 +62,15 @@ func encryptionStatusHandler(server *config.Control) http.Handler {
 		}
 		status, err := encryptionStatus(server)
 		if err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
+			util.SendErrorWithID(err, "secret-encrypt", resp, req, http.StatusInternalServerError)
 			return
 		}
 		b, err := json.Marshal(status)
 		if err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err, "secrets-encrypt")
+			util.SendErrorWithID(err, "secret-encrypt", resp, req, http.StatusInternalServerError)
 			return
 		}
+		resp.Header().Set("Content-Type", "application/json")
 		resp.Write(b)
 	})
 }
@@ -192,7 +192,7 @@ func encryptionConfigHandler(ctx context.Context, server *config.Control) http.H
 		}
 
 		if err != nil {
-			genErrorMessage(resp, http.StatusBadRequest, err, "secrets-encrypt")
+			util.SendErrorWithID(err, "secret-encrypt", resp, req, http.StatusBadRequest)
 			return
 		}
 		// If a user kills the k3s server immediately after this call, we run into issues where the files
@@ -464,18 +464,3 @@ func verifyEncryptionHashAnnotation(runtime *config.ControlRuntime, core core.In
 
 	return nil
 }
-
-// genErrorMessage sends and logs a random error ID so that logs can be correlated
-// between the REST API (which does not provide any detailed error output, to avoid
-// information disclosure) and the server logs.
-func genErrorMessage(resp http.ResponseWriter, statusCode int, passedErr error, component string) {
-	errID, err := rand.Int(rand.Reader, big.NewInt(99999))
-	if err != nil {
-		resp.WriteHeader(http.StatusInternalServerError)
-		resp.Write([]byte(err.Error()))
-		return
-	}
-	logrus.Warnf("%s error ID %05d: %s", component, errID, passedErr.Error())
-	resp.WriteHeader(statusCode)
-	resp.Write([]byte(fmt.Sprintf("%s error ID %05d", component, errID)))
-}

pkg/server/token.go

@@ -45,7 +45,7 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl
 			return
 		}
 		if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil {
-			genErrorMessage(resp, http.StatusInternalServerError, err, "token")
+			util.SendErrorWithID(err, "token", resp, req, http.StatusInternalServerError)
 			return
 		}
 		resp.WriteHeader(http.StatusOK)

pkg/util/apierrors.go

 package util
 
 import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
 	"net/http"
 
 	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
@@ -14,6 +17,15 @@ import (
 
 var ErrNotReady = errors.New("apiserver not ready")
 
+// SendErrorWithID sends and logs a random error ID so that logs can be correlated
+// between the REST API (which does not provide any detailed error output, to avoid
+// information disclosure) and the server logs.
+func SendErrorWithID(err error, component string, resp http.ResponseWriter, req *http.Request, status ...int) {
+	errID, _ := rand.Int(rand.Reader, big.NewInt(99999))
+	logrus.Errorf("%s error ID %05d: %v", component, errID, err)
+	SendError(fmt.Errorf("%s error ID %05d", component, errID), resp, req, status...)
+}
+
 // SendError sends a properly formatted error response
 func SendError(err error, resp http.ResponseWriter, req *http.Request, status ...int) {
 	var code int

tests/e2e/s3/s3_test.go

@@ -89,21 +89,18 @@ var _ = Describe("Verify Create", Ordered, func() {
 		It("save s3 snapshot", func() {
 			res, err := e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(ContainSubstring("S3 bucket test exists"))
-			Expect(res).To(ContainSubstring("Uploading snapshot"))
-			Expect(res).To(ContainSubstring("S3 upload complete for"))
+			Expect(res).To(ContainSubstring("Snapshot on-demand-server-0"))
 		})
 		It("lists saved s3 snapshot", func() {
 			res, err := e2e.RunCmdOnNode("k3s etcd-snapshot list", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(ContainSubstring("on-demand-server-0"))
+			Expect(res).To(ContainSubstring("file:///var/lib/rancher/k3s/server/db/snapshots/on-demand-server-0"))
 		})
 		It("save 3 more s3 snapshots", func() {
 			for _, i := range []string{"1", "2", "3"} {
 				res, err := e2e.RunCmdOnNode("k3s etcd-snapshot save --name special-"+i, serverNodeNames[0])
 				Expect(err).NotTo(HaveOccurred())
-				Expect(res).To(ContainSubstring("Uploading snapshot"))
-				Expect(res).To(ContainSubstring("S3 upload complete for special-" + i))
+				Expect(res).To(ContainSubstring("Snapshot special-" + i + "-server-0"))
 			}
 		})
 		It("lists saved s3 snapshot", func() {
@@ -117,18 +114,12 @@ var _ = Describe("Verify Create", Ordered, func() {
 		It("delete first on-demand s3 snapshot", func() {
 			_, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot ls >> ./snapshotname.txt", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			snapshotName, err := e2e.RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt |head -1", serverNodeNames[0])
+			snapshotName, err := e2e.RunCmdOnNode("grep -Eo 'on-demand-server-0-([0-9]+)' ./snapshotname.txt | head -1", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 			res, err := e2e.RunCmdOnNode("sudo k3s etcd-snapshot delete "+snapshotName, serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(ContainSubstring("Reconciling ETCDSnapshotFile resources"))
-			Expect(res).To(ContainSubstring("Snapshot " + strings.TrimSpace(snapshotName) + " deleted from S3"))
-			Expect(res).To(ContainSubstring("Reconciliation of ETCDSnapshotFile resources complete"))
+			Expect(res).To(ContainSubstring("Snapshot " + strings.TrimSpace(snapshotName) + " deleted"))
 		})
-
-		// TODO, there is currently a bug that prevents pruning on s3 snapshots that are not prefixed with "on-demand"
-		// https://github.com/rancher/rke2/issues/3714
-		// Once fixed, ensure that the snapshots list are actually reduced to 2
 		It("prunes s3 snapshots", func() {
 			_, err := e2e.RunCmdOnNode("k3s etcd-snapshot save", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
@@ -136,11 +127,8 @@ var _ = Describe("Verify Create", Ordered, func() {
 			Expect(err).NotTo(HaveOccurred())
 			res, err := e2e.RunCmdOnNode("k3s etcd-snapshot prune --snapshot-retention 2", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
-			Expect(res).To(ContainSubstring("Reconciliation of ETCDSnapshotFile resources complete"))
-
-			_, err = e2e.RunCmdOnNode("k3s etcd-snapshot ls|grep 'on-demand'|wc -l>count", serverNodeNames[0])
-			Expect(err).NotTo(HaveOccurred())
-			res, err = e2e.RunCmdOnNode("grep '^[4]$' ./count", serverNodeNames[0])
+			// There should now be 4 on-demand snapshots - 2 local, and 2 on s3
+			res, err = e2e.RunCmdOnNode("k3s etcd-snapshot ls 2>/dev/null | grep on-demand | wc -l", serverNodeNames[0])
 			Expect(err).NotTo(HaveOccurred())
 			Expect(strings.TrimSpace(res)).To(Equal("4"))
 		})

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -58,13 +58,13 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 			snapshotName := reg.FindString(lsResult)
 			Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).
-				To(ContainSubstring("Snapshot " + snapshotName + " deleted locally"))
+				To(ContainSubstring("Snapshot " + snapshotName + " deleted"))
 		})
 	})
 	When("saving a custom name", func() {
 		It("saves an etcd snapshot with a custom name", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save --name ALIVEBEEF")).
-				To(ContainSubstring("Saving etcd snapshot to /var/lib/rancher/k3s/server/db/snapshots/ALIVEBEEF"))
+				To(ContainSubstring("Snapshot ALIVEBEEF-"))
 		})
 		It("deletes that snapshot", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
@@ -73,7 +73,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 			snapshotName := reg.FindString(lsResult)
 			Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).
-				To(ContainSubstring("Snapshot " + snapshotName + " deleted locally"))
+				To(ContainSubstring("Snapshot " + snapshotName + " deleted"))
 		})
 	})
 	When("using etcd snapshot prune", func() {
@@ -98,7 +98,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 		})
 		It("prunes snapshots down to 2", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "prune --snapshot-retention 2 --name PRUNE_TEST")).
-				To(ContainSubstring("Removing local snapshot"))
+				To(ContainSubstring(" deleted."))
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
 			reg, err := regexp.Compile(`(?m):///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
@@ -113,7 +113,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 			Expect(err).ToNot(HaveOccurred())
 			for _, snapshotName := range reg.FindAllString(lsResult, -1) {
 				Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).
-					To(ContainSubstring("Snapshot " + snapshotName + " deleted locally"))
+					To(ContainSubstring("Snapshot " + snapshotName + " deleted"))
 			}
 		})
 	})