Merge pull request #11931 from brandond/2025-03-backports_release-1.29

cmd/k3s/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"io"
 	"io/fs"
 	"os"
@@ -20,7 +21,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/flock"
 	"github.com/k3s-io/k3s/pkg/untar"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -207,7 +208,7 @@ func stageAndRunCLI(cli *cli.Context, cmd string, dataDir string, args []string)
 func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) error {
 	dir, err := extract(dataDir)
 	if err != nil {
-		return errors.Wrap(err, "extracting data")
+		return pkgerrors.WithMessage(err, "extracting data")
 	}
 	logrus.Debugf("Asset dir %s", dir)
 

cmd/k3s/main_linux.go

@@ -7,14 +7,14 @@ import (
 	"os"
 	"syscall"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const programPostfix = ""
 
 func runExec(cmd string, args []string, calledAsInternal bool) (err error) {
 	if err := syscall.Exec(cmd, args, os.Environ()); err != nil {
-		return errors.Wrapf(err, "exec %s failed", cmd)
+		return pkgerrors.WithMessagef(err, "exec %s failed", cmd)
 	}
 	return nil
 }

contrib/util/check-config.sh

@@ -386,10 +386,10 @@ fi
 flags="
   NAMESPACES NET_NS PID_NS IPC_NS UTS_NS
   CGROUPS CGROUP_PIDS CGROUP_CPUACCT CGROUP_DEVICE CGROUP_FREEZER CGROUP_SCHED CPUSETS MEMCG
-  KEYS
+  SECCOMP KEYS
   VETH BRIDGE BRIDGE_NETFILTER
   IP_NF_FILTER IP_NF_TARGET_MASQUERADE IP_NF_TARGET_REJECT
-  NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS NETFILTER_XT_MATCH_COMMENT NETFILTER_XT_MATCH_MULTIPORT
+  NETFILTER_XT_MATCH_ADDRTYPE NETFILTER_XT_MATCH_CONNTRACK NETFILTER_XT_MATCH_IPVS NETFILTER_XT_MATCH_COMMENT NETFILTER_XT_MATCH_MULTIPORT NETFILTER_XT_MATCH_STATISTIC
   IP_NF_NAT NF_NAT
   POSIX_MQUEUE
 "
@@ -406,9 +406,6 @@ echo 'Optional Features:'
   check_flags USER_NS
   check_distro_userns
 }
-{
-  check_flags SECCOMP
-}
 # {
 #   check_flags MEMCG_SWAP MEMCG_SWAP_ENABLED
 #   if [ -e /sys/fs/cgroup/memory/memory.memsw.limit_in_bytes ]; then

go.mod

@@ -7,7 +7,8 @@ replace (
 	github.com/Mirantis/cri-dockerd => github.com/k3s-io/cri-dockerd v0.3.12-k3s1 // k3s/release-1.28
 	github.com/cilium/ebpf => github.com/cilium/ebpf v0.12.3
 	github.com/cloudnativelabs/kube-router/v2 => github.com/k3s-io/kube-router/v2 v2.2.1
-	github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.23-k3s2
+	github.com/containerd/containerd => github.com/k3s-io/containerd v1.7.26-k3s1
+	github.com/containerd/containerd/api => github.com/containerd/containerd/api v1.8.0
 	github.com/containerd/imgcrypt => github.com/containerd/imgcrypt v1.1.11
 	github.com/distribution/reference => github.com/distribution/reference v0.5.0
 	github.com/docker/distribution => github.com/docker/distribution v2.8.3+incompatible
@@ -17,21 +18,20 @@ replace (
 	github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.14.1
 	github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.29.0-k3s1
 	github.com/open-policy-agent/opa => github.com/open-policy-agent/opa v0.59.0 // github.com/Microsoft/hcsshim using bad version v0.42.2
-	github.com/opencontainers/runc => github.com/k3s-io/runc v1.2.4-k3s1
 	github.com/opencontainers/selinux => github.com/opencontainers/selinux v1.11.0
 	github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.18.0
 	github.com/prometheus/common => github.com/prometheus/common v0.45.0
 	github.com/rancher/wrangler => github.com/rancher/wrangler v1.1.1-0.20230818201331-3604a6be798d
 	github.com/spegel-org/spegel => github.com/k3s-io/spegel v0.0.30-k3s1.31
 	github.com/ugorji/go => github.com/ugorji/go v1.2.11
-	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.18-k3s1
-	go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.18-k3s1
-	go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.18-k3s1
+	go.etcd.io/etcd/api/v3 => github.com/k3s-io/etcd/api/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/client/pkg/v3 => github.com/k3s-io/etcd/client/pkg/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/client/v2 => github.com/k3s-io/etcd/client/v2 v2.305.19-k3s1.30
+	go.etcd.io/etcd/client/v3 => github.com/k3s-io/etcd/client/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/etcdutl/v3 => github.com/k3s-io/etcd/etcdutl/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/pkg/v3 => github.com/k3s-io/etcd/pkg/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/raft/v3 => github.com/k3s-io/etcd/raft/v3 v3.5.19-k3s1.30
+	go.etcd.io/etcd/server/v3 => github.com/k3s-io/etcd/server/v3 v3.5.19-k3s1.30
 	go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful => go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.44.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.45.0
 	golang.org/x/crypto => golang.org/x/crypto v0.17.0
@@ -74,6 +74,8 @@ replace (
 	k8s.io/sample-apiserver => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-apiserver v1.29.14-k3s1
 	k8s.io/sample-cli-plugin => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.29.14-k3s1
 	k8s.io/sample-controller => github.com/k3s-io/kubernetes/staging/src/k8s.io/sample-controller v1.29.14-k3s1
+	sigs.k8s.io/json => sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd
+	sigs.k8s.io/structured-merge-diff/v4 => sigs.k8s.io/structured-merge-diff/v4 v4.4.1
 	sourcegraph.com/sourcegraph/go-diff => github.com/sourcegraph/go-diff v0.6.0
 )
 
@@ -109,6 +111,7 @@ require (
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/joho/godotenv v1.5.1
 	github.com/json-iterator/go v1.1.12
+	github.com/k3s-io/api v0.1.0-k3s1.29
 	github.com/k3s-io/helm-controller v0.15.16
 	github.com/k3s-io/kine v0.13.9
 	github.com/klauspost/compress v1.17.11
@@ -119,8 +122,8 @@ require (
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/onsi/ginkgo/v2 v2.22.0
 	github.com/onsi/gomega v1.34.2
-	github.com/opencontainers/runc v1.1.12
-	github.com/opencontainers/selinux v1.11.0
+	github.com/opencontainers/runc v1.2.5
+	github.com/opencontainers/selinux v1.11.1
 	github.com/otiai10/copy v1.7.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.20.5
@@ -140,17 +143,17 @@ require (
 	github.com/urfave/cli v1.22.15
 	github.com/vishvananda/netlink v1.3.0
 	github.com/yl2chen/cidranger v1.0.2
-	go.etcd.io/etcd/api/v3 v3.5.18
-	go.etcd.io/etcd/client/pkg/v3 v3.5.18
-	go.etcd.io/etcd/client/v3 v3.5.18
+	go.etcd.io/etcd/api/v3 v3.5.19
+	go.etcd.io/etcd/client/pkg/v3 v3.5.19
+	go.etcd.io/etcd/client/v3 v3.5.19
 	go.etcd.io/etcd/etcdutl/v3 v3.5.18
-	go.etcd.io/etcd/server/v3 v3.5.18
+	go.etcd.io/etcd/server/v3 v3.5.19
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.32.0
+	golang.org/x/crypto v0.35.0
 	golang.org/x/mod v0.22.0
-	golang.org/x/net v0.34.0
-	golang.org/x/sync v0.10.0
-	golang.org/x/sys v0.29.0
+	golang.org/x/net v0.36.0
+	golang.org/x/sync v0.11.0
+	golang.org/x/sys v0.30.0
 	google.golang.org/grpc v1.70.0
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.31.5
@@ -222,19 +225,19 @@ require (
 	github.com/containerd/go-runc v1.1.0 // indirect
 	github.com/containerd/imgcrypt v1.2.0-rc1 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/nri v0.6.1 // indirect
+	github.com/containerd/nri v0.8.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
-	github.com/containerd/ttrpc v1.2.5 // indirect
+	github.com/containerd/ttrpc v1.2.7 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/containernetworking/cni v1.2.3 // indirect
 	github.com/containernetworking/plugins v1.5.1 // indirect
-	github.com/containers/ocicrypt v1.2.0 // indirect
+	github.com/containers/ocicrypt v1.2.1 // indirect
 	github.com/coreos/go-oidc v2.2.1+incompatible // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/cyphar/filepath-securejoin v0.3.5 // indirect
+	github.com/cyphar/filepath-securejoin v0.4.1 // indirect
 	github.com/danwinship/knftables v0.0.13 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/daviddengcn/go-colortext v1.0.0 // indirect
@@ -263,7 +266,7 @@ require (
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.4 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
@@ -436,6 +439,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/seccomp/libseccomp-golang v0.10.0 // indirect
 	github.com/shengdoushi/base58 v1.0.0 // indirect
+	github.com/smallstep/pkcs7 v0.1.1 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.12.0 // indirect
@@ -457,10 +461,9 @@ require (
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	go.etcd.io/bbolt v1.3.11 // indirect
-	go.etcd.io/etcd/client/v2 v2.305.18 // indirect
-	go.etcd.io/etcd/pkg/v3 v3.5.18 // indirect
-	go.etcd.io/etcd/raft/v3 v3.5.18 // indirect
-	go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.19 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.19 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.19 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.53.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect
@@ -480,7 +483,7 @@ require (
 	golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect
 	golang.org/x/oauth2 v0.25.0 // indirect
 	golang.org/x/term v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
 	golang.org/x/time v0.8.0 // indirect
 	golang.org/x/tools v0.28.0 // indirect
 	golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect
@@ -498,7 +501,6 @@ require (
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.30.0 // indirect
-	k8s.io/code-generator v0.30.0 // indirect
 	k8s.io/controller-manager v0.25.4 // indirect
 	k8s.io/csi-translation-lib v0.0.0 // indirect
 	k8s.io/dynamic-resource-allocation v0.0.0 // indirect
@@ -507,7 +509,7 @@ require (
 	k8s.io/kms v0.0.0 // indirect
 	k8s.io/kube-aggregator v0.30.0 // indirect
 	k8s.io/kube-controller-manager v0.0.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
+	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
 	k8s.io/kube-scheduler v0.0.0 // indirect
 	k8s.io/kubelet v0.28.6 // indirect
 	k8s.io/legacy-cloud-providers v0.0.0 // indirect
@@ -520,7 +522,7 @@ require (
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	tags.cncf.io/container-device-interface v0.8.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
+	tags.cncf.io/container-device-interface v0.8.1 // indirect
 	tags.cncf.io/container-device-interface/specs-go v0.8.0 // indirect
 )

go.sum

@@ -387,11 +387,18 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
 github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA=
 github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI=
+github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
+github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
+github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=
 github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM=
 github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
@@ -422,7 +429,6 @@ github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4q
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro=
 github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
-github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig=
 github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0=
 github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
@@ -448,8 +454,8 @@ github.com/containerd/imgcrypt v1.1.11 h1:3RULIeLouE7B5l9NzMq0HdPWG0DP5deEVxB5UK
 github.com/containerd/imgcrypt v1.1.11/go.mod h1:nXL4jp1GrtO758b16DVsxaHHzu9PravAsKARYmyHR58=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
-github.com/containerd/nri v0.6.1 h1:xSQ6elnQ4Ynidm9u49ARK9wRKHs80HCUI+bkXOxV4mA=
-github.com/containerd/nri v0.6.1/go.mod h1:7+sX3wNx+LR7RzhjnJiUkFDhn18P5Bg/0VnJ/uXpRJM=
+github.com/containerd/nri v0.8.0 h1:n1S753B9lX8RFrHYeSgwVvS1yaUcHjxbB+f+xzEncRI=
+github.com/containerd/nri v0.8.0/go.mod h1:uSkgBrCdEtAiEz4vnrq8gmAC4EnVAM5Klt0OuK5rZYQ=
 github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
 github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
 github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA=
@@ -460,9 +466,10 @@ github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9N
 github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ=
 github.com/containerd/ttrpc v1.2.2/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf1G5tYZak=
-github.com/containerd/ttrpc v1.2.3/go.mod h1:ieWsXucbb8Mj9PH0rXCw1i8IunRbbAiDkpXkbfflWBM=
-github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU=
 github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
+github.com/containerd/ttrpc v1.2.6-0.20240827082320-b5cd6e4b3287/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
+github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ=
+github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
 github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
 github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
@@ -477,8 +484,8 @@ github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+
 github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+E5J/EcKOE4gQ=
 github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM=
 github.com/containers/ocicrypt v1.1.10/go.mod h1:YfzSSr06PTHQwSTUKqDSjish9BeW1E4HUmreluQcMd8=
-github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sirVuPM=
-github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U=
+github.com/containers/ocicrypt v1.2.1 h1:0qIOTT9DoYwcKmxSt8QJt+VzMY18onl9jUXsxpVhSmM=
+github.com/containers/ocicrypt v1.2.1/go.mod h1:aD0AAqfMp0MtwqWgHM1bUwe1anx0VazI108CRrSKINQ=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -512,8 +519,9 @@ github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
-github.com/cyphar/filepath-securejoin v0.3.5 h1:L81NHjquoQmcPgXcttUS9qTSR/+bXry6pbSINQGpjj4=
-github.com/cyphar/filepath-securejoin v0.3.5/go.mod h1:edhVd3c6OXKjUmSrVa/tGJRS9joFTxlslFCAyaxigkE=
+github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s=
+github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
@@ -648,8 +656,8 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
-github.com/go-jose/go-jose/v4 v4.0.2 h1:R3l3kkBds16bO7ZFAEEcofK0MkrAJt3jlJznWZG0nvk=
-github.com/go-jose/go-jose/v4 v4.0.2/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
+github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E=
+github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
 github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U=
@@ -694,6 +702,9 @@ github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
 github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
@@ -815,6 +826,7 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
 github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
 github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -925,6 +937,7 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
 github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
@@ -1003,28 +1016,30 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
-github.com/k3s-io/containerd v1.7.23-k3s2 h1:QPS588RkyWooj/M+C64EAd1wDeqLmW/RB0DNW08x7ls=
-github.com/k3s-io/containerd v1.7.23-k3s2/go.mod h1:+vNritUoqQTlFedsBErSkyu1yqf3eYikE3oGbEICi+g=
+github.com/k3s-io/api v0.1.0-k3s1.29 h1:+u/L+fNEEKUkNEon0v6qx5kUzPLL9nllTPXMeRPEQ+8=
+github.com/k3s-io/api v0.1.0-k3s1.29/go.mod h1:LMyYnnmCH9wYAD7Bu8fmHUZLqEIvrJ6Hr4Xu8aXOLhs=
+github.com/k3s-io/containerd v1.7.26-k3s1 h1:qn7P7ZlOUBHZOMR1+EKUMo8/75yDFq9E7yt1+Uo7/oA=
+github.com/k3s-io/containerd v1.7.26-k3s1/go.mod h1:fYwK9zVBds6h6/Ie4EoePjhSbPx7gltISrLWZK8d7uk=
 github.com/k3s-io/cri-dockerd v0.3.12-k3s1 h1:jGTy2U1Nn8d9o23NwLV1NAigTBvePTA7XaZMifZ01Q4=
 github.com/k3s-io/cri-dockerd v0.3.12-k3s1/go.mod h1:S98trivsinxuNGQANgrZ9ComFqQkVv7vUvsXSNBRCFs=
 github.com/k3s-io/cri-tools v1.29.0-k3s1 h1:16IXZ5lbPCmZM8FkgSMAPkhI4O2wVGExe3qEZbisFT0=
 github.com/k3s-io/cri-tools v1.29.0-k3s1/go.mod h1:fZeWlv+qq4gZ005I13j4JcvgFb6ZobVTtON3PqM5JVc=
-github.com/k3s-io/etcd/api/v3 v3.5.18-k3s1 h1:VCVN5BltorEP72gKnf+346X+3SoLRhc1vLmB1XViAbI=
-github.com/k3s-io/etcd/api/v3 v3.5.18-k3s1/go.mod h1:uY03Ob2H50077J7Qq0DeehjM/A9S8PhVfbQ1mSaMopU=
-github.com/k3s-io/etcd/client/pkg/v3 v3.5.18-k3s1 h1:sz+MGVwhBWFCgVtIRDsWfDKYi28V8zZL/yiwUQqghuY=
-github.com/k3s-io/etcd/client/pkg/v3 v3.5.18-k3s1/go.mod h1:BxVf2o5wXG9ZJV+/Cu7QNUiJYk4A29sAhoI5tIRsCu4=
-github.com/k3s-io/etcd/client/v2 v2.305.18-k3s1 h1:5MqoMBVHcYtci7AnAWhAGlpv0tpdGjmUzTKz4AQdP1A=
-github.com/k3s-io/etcd/client/v2 v2.305.18-k3s1/go.mod h1:JikXfwJymsNv633PzkAb5xnVZmROgNWr4E68YCEz4jo=
-github.com/k3s-io/etcd/client/v3 v3.5.18-k3s1 h1:4V0hwSU3k6yfmaLPTCUopHz4pHJQWYFEjDbWveYEO4w=
-github.com/k3s-io/etcd/client/v3 v3.5.18-k3s1/go.mod h1:kmemwOsPU9broExyhYsBxX4spCTDX3yLgPMWtpBXG6E=
-github.com/k3s-io/etcd/etcdutl/v3 v3.5.18-k3s1 h1:jSoW/HJaipmWMLv7ViEFdIF2KNsDrg1t1khVxrUOv1k=
-github.com/k3s-io/etcd/etcdutl/v3 v3.5.18-k3s1/go.mod h1:2R27zhWnQMPl7MaoyGxHgG+6DOyXgEUxMrpcgwWFEiQ=
-github.com/k3s-io/etcd/pkg/v3 v3.5.18-k3s1 h1:dtIzhSUbviliTc7sriA9nhRDtr0Gt2pUf+6508g53Ac=
-github.com/k3s-io/etcd/pkg/v3 v3.5.18-k3s1/go.mod h1:gb4CDXuN/OgzUgj+VmUFumLYQ2FUMDC6r/plLIjHPI8=
-github.com/k3s-io/etcd/raft/v3 v3.5.18-k3s1 h1:CnlB21GpOGN/dINcjLizWk5qkI56h/33bpSv5/xEGCQ=
-github.com/k3s-io/etcd/raft/v3 v3.5.18-k3s1/go.mod h1:XBaZHTJt3nLnpS8hMDR55Sxrq76cEC4xWYMBYSY3jcs=
-github.com/k3s-io/etcd/server/v3 v3.5.18-k3s1 h1:YHhvwef4h6F53c3BeVSROhXrka/CdFSeInY+s7m67yw=
-github.com/k3s-io/etcd/server/v3 v3.5.18-k3s1/go.mod h1:waeL2uw6TdXniXaus105tiK1aSbblIBi21uk8y7D6Ng=
+github.com/k3s-io/etcd/api/v3 v3.5.19-k3s1.30 h1:8JbTrtBa2007x6nVW8rrNqeYX0KatelNm0zM1DsAA1c=
+github.com/k3s-io/etcd/api/v3 v3.5.19-k3s1.30/go.mod h1:mheu6z9SF3NzA8zO8EbViuQmtLqt1SuPOYDq9Tr1cXc=
+github.com/k3s-io/etcd/client/pkg/v3 v3.5.19-k3s1.30 h1:xl9yxkXHsT+A6iVY+I0jY4k4sGw+dibDTrUITgJt16E=
+github.com/k3s-io/etcd/client/pkg/v3 v3.5.19-k3s1.30/go.mod h1:iBkkJ99KTDat+sQpOiCLk0n5yWKxDojOBbo8SiDG1to=
+github.com/k3s-io/etcd/client/v2 v2.305.19-k3s1.30 h1:pqzYo1Dw5yWB+rhjmFJtjMLsWR6fgKiszW7btbGKVAg=
+github.com/k3s-io/etcd/client/v2 v2.305.19-k3s1.30/go.mod h1:yREuCyYOgrTek+5BhbLaBofbPdLgg8P7ezN1ou9R6lA=
+github.com/k3s-io/etcd/client/v3 v3.5.19-k3s1.30 h1:v45I43GHOBzzj/IB7InK4HvMAmLyq+SpKwlnSVYzxX8=
+github.com/k3s-io/etcd/client/v3 v3.5.19-k3s1.30/go.mod h1:/ynIMsvn5bwZKuP7rVAZNTJJqXZzvVwYv9OwoEaDYqU=
+github.com/k3s-io/etcd/etcdutl/v3 v3.5.19-k3s1.30 h1:UtzUyoZlHScTVbHU56lcW0IatxMYAjqj3FgsmGNIaNk=
+github.com/k3s-io/etcd/etcdutl/v3 v3.5.19-k3s1.30/go.mod h1:c9YyblMTQ/HTnJd4cmYFbr13ybxmsatq8QKuuehuCjs=
+github.com/k3s-io/etcd/pkg/v3 v3.5.19-k3s1.30 h1:VPQ6WTWZiUAijGl3YqhYq6fNSm0kl3md4g/ks2WCEy4=
+github.com/k3s-io/etcd/pkg/v3 v3.5.19-k3s1.30/go.mod h1:8yhd7lYPZ1R2TQL3XEiYvr8KKDztbSR98AJy6gWMzyo=
+github.com/k3s-io/etcd/raft/v3 v3.5.19-k3s1.30 h1:0ad4novLl2W08IAAw9f9depOoAMu+spad2fG7PWf0Rk=
+github.com/k3s-io/etcd/raft/v3 v3.5.19-k3s1.30/go.mod h1:34HNpx9+iBu2/nFEwEubME/ksLfqmBw7jFwXJTiaziQ=
+github.com/k3s-io/etcd/server/v3 v3.5.19-k3s1.30 h1:/ZBGPwjJpBWPlNQL3Ly/fOaE+su4QhEIUJvweO7JH3I=
+github.com/k3s-io/etcd/server/v3 v3.5.19-k3s1.30/go.mod h1:CBNk5QBtfYrGyKDzdrkADyFD3bmPaLhPHv6rdqiCNsk=
 github.com/k3s-io/helm-controller v0.15.16 h1:JGW6BUEf/33ZCR7Fe47NTw/CWn9uCrkzNDlQYgentHw=
 github.com/k3s-io/helm-controller v0.15.16/go.mod h1:xGN8KNHGkmlaCwyjNx281cR8IEqC9MJgyMaS3wn1k+o=
 github.com/k3s-io/kine v0.13.9 h1:Dcobn5rXfl0tGCTPJzLRsowxAnK/4hhLzRGuPXhRJVQ=
@@ -1051,8 +1066,6 @@ github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.29.14-k3s1 h1:
 github.com/k3s-io/kubernetes/staging/src/k8s.io/cloud-provider v1.29.14-k3s1/go.mod h1:N4Rw9sJ86OQXDc5dUFx2MnIOSttMkGPtLZyEoCzry8Q=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.29.14-k3s1 h1:zcIfToCltvrJtsdSYT7al+t5KAP1WeRL7xvo9ZI9t+w=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.29.14-k3s1/go.mod h1:7nqgik19CFiTPjB0wYiE96lCsAPpytEAPT8Eqlhdrd8=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.29.14-k3s1 h1:CecO6NMk11BkNGsaA9HYOOGv+zcccO529VQeftgk5bU=
-github.com/k3s-io/kubernetes/staging/src/k8s.io/code-generator v1.29.14-k3s1/go.mod h1:aOKsghBDeKfSUqZ5vaOK6151HkaMkDsFBMkVR25teFQ=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.29.14-k3s1 h1:dECg4o8EF2FijI4D5OYf69nVqAHkTnetwHGPpl+DlGM=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/component-base v1.29.14-k3s1/go.mod h1:6oRMFYEkMBlkkBVznsi7HH/kV5F7nhyd6cqXCRCdzdo=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/component-helpers v1.29.14-k3s1 h1:WxpeRHdN5V9gTRQ7SUV8tG8favDQy0C7SxB7GhvhRaE=
@@ -1089,8 +1102,6 @@ github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.14-k3s1 h1:2sL
 github.com/k3s-io/kubernetes/staging/src/k8s.io/mount-utils v1.29.14-k3s1/go.mod h1:4KmkE88Y4LDYrotr6iqMrolXDcWWY7UqmroXTO/sxFw=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.14-k3s1 h1:nhjhZg/rSaTM3emgPLBLVnOUHzl339ABFRa6OzzaIeU=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/pod-security-admission v1.29.14-k3s1/go.mod h1:3nvUgy9DAoVbLCBJcIBDCldv+vAc7hcHl6xJFRSpvb4=
-github.com/k3s-io/runc v1.2.4-k3s1 h1:wCqlmxJiVUsZvdNsSUUm7Ng0kCK2wAgyBt4lcp2lLP8=
-github.com/k3s-io/runc v1.2.4-k3s1/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=
 github.com/k3s-io/spegel v0.0.30-k3s1.31 h1:ZFDMwJ/WJux1+eL8VFmK/zsrTdtw4ApUQtRbtVvFWB8=
 github.com/k3s-io/spegel v0.0.30-k3s1.31/go.mod h1:ZmsNeffliGQQpXzVrnMYz2rMZ4UwBJVWvAbvq0qJ3aE=
 github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI=
@@ -1132,6 +1143,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
 github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
 github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
 github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
@@ -1264,6 +1276,7 @@ github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
+github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
 github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
 github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
@@ -1380,6 +1393,10 @@ github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3Ro
 github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0=
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
+github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs=
+github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc=
+github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
+github.com/onsi/ginkgo/v2 v2.19.1/go.mod h1:O3DtEWQkPa/F7fBMgmZQKKsluAy8pd3rEQdrjkPb9zA=
 github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
 github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -1402,6 +1419,10 @@ github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRah
 github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
 github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
 github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
+github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY=
+github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0=
+github.com/onsi/gomega v1.34.0/go.mod h1:MIKI8c+f+QLWk+hxbePD4i0LMJSExPaZOVfkoex4cAo=
 github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
 github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
 github.com/open-policy-agent/opa v0.59.0/go.mod h1:rdJSkEc4oQ+0074/3Fsgno5bkPsYxTjU5aLNmMujIvI=
@@ -1414,6 +1435,10 @@ github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVn
 github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
 github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
+github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8=
+github.com/opencontainers/runc v1.2.5 h1:8KAkq3Wrem8bApgOHyhRI/8IeLXIfmZ6Qaw6DNSLnA4=
+github.com/opencontainers/runc v1.2.5/go.mod h1:dOQeFo29xZKBNeRBI0B19mJtfHv68YgCTh1X+YphA+4=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
@@ -1428,6 +1453,7 @@ github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaL
 github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
 github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
@@ -1578,6 +1604,7 @@ github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZ
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY=
 github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
@@ -1614,6 +1641,8 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/smallstep/pkcs7 v0.1.1 h1:x+rPdt2W088V9Vkjho4KtoggyktZJlMduZAtRHm68LU=
+github.com/smallstep/pkcs7 v0.1.1/go.mod h1:dL6j5AIz9GHjVEBTXtW+QliALcgM19RtXaTeyxI+AfA=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
 github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
@@ -1697,6 +1726,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7
 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
@@ -1769,7 +1799,6 @@ go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.etcd.io/gofail v0.1.0/go.mod h1:VZBCXYGZhHAinaBiiqYvuDynvahNsAyLFwB3kEHKz1M=
-go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M=
 go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
@@ -2028,8 +2057,9 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
 golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
@@ -2038,6 +2068,7 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
 golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2061,11 +2092,13 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2161,6 +2194,9 @@ golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8
 golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
 golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
+golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
+golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8=
 golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw=
@@ -2328,6 +2364,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
@@ -2354,6 +2391,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
@@ -2415,8 +2453,8 @@ k8s.io/gengo v0.0.0-20240228010128-51d4e06bde70 h1:D9H6wq7PAmub2g4XUrekNWMFVI0JI
 k8s.io/gengo v0.0.0-20240228010128-51d4e06bde70/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
-k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo=
-k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
+k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y=
+k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
@@ -2474,7 +2512,6 @@ sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3 h1:vq2Tt
 sigs.k8s.io/kustomize/kustomize/v5 v5.0.4-0.20230601165947-6ce0bf390ce3/go.mod h1:/d88dHCvoy7d0AKFT0yytezSGZKjsZBVs9YTkBHSGFk=
 sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 h1:W6cLQc5pnqM7vh3b7HvGNfXrJ/xL6BDMS0v1V/HHg5U=
 sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3/go.mod h1:JWP1Fj0VWGHyw3YUPjXSQnRnrwezrZSrApfX5S0nIag=
-sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
@@ -2482,8 +2519,8 @@ sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
 tags.cncf.io/container-device-interface v0.7.2/go.mod h1:Xb1PvXv2BhfNb3tla4r9JL129ck1Lxv9KuU6eVOfKto=
-tags.cncf.io/container-device-interface v0.8.0 h1:8bCFo/g9WODjWx3m6EYl3GfUG31eKJbaggyBDxEldRc=
-tags.cncf.io/container-device-interface v0.8.0/go.mod h1:Apb7N4VdILW0EVdEMRYXIDVRZfNJZ+kmEUss2kRRQ6Y=
+tags.cncf.io/container-device-interface v0.8.1 h1:c0jN4Mt6781jD67NdPajmZlD1qrqQyov/Xfoab37lj0=
+tags.cncf.io/container-device-interface v0.8.1/go.mod h1:Apb7N4VdILW0EVdEMRYXIDVRZfNJZ+kmEUss2kRRQ6Y=
 tags.cncf.io/container-device-interface/specs-go v0.7.0/go.mod h1:hMAwAbMZyBLdmYqWgYcKH0F/yctNpV3P35f+/088A80=
 tags.cncf.io/container-device-interface/specs-go v0.8.0 h1:QYGFzGxvYK/ZLMrjhvY0RjpUavIn4KcmRmVP/JjdBTA=
 tags.cncf.io/container-device-interface/specs-go v0.8.0/go.mod h1:BhJIkjjPh4qpys+qm4DAYtUyryaTDg9zris+AczXyws=

hack/crdgen.go

@@ -3,8 +3,8 @@ package main
 import (
 	"os"
 
+	_ "github.com/k3s-io/api/pkg/generated/controllers/k3s.cattle.io/v1"
 	k3scrd "github.com/k3s-io/k3s/pkg/crd"
-	_ "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
 	"github.com/rancher/wrangler/pkg/crd"
 )
 

main.go

-//go:generate go run pkg/codegen/cleanup/main.go
-//go:generate rm -rf pkg/generated
 //go:generate go run pkg/codegen/main.go
 //go:generate go fmt pkg/deploy/zz_generated_bindata.go
 //go:generate go fmt pkg/static/zz_generated_bindata.go

package/Dockerfile

 FROM alpine:3.20 as base
 RUN apk add -U ca-certificates zstd tzdata
 COPY build/out/data-linux.tar.zst /
-RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \
+RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware /image/var/lib/rancher/k3s/data/cni && \
     zstdcat -d /data-linux.tar.zst | tar -xa -C /image && \
+    for FILE in cni $(/image/bin/find /image/bin -lname cni -printf "%f\n"); do ln -s /bin/cni /image/var/lib/rancher/k3s/data/cni/$FILE; done && \
     echo "root:x:0:0:root:/:/bin/sh" > /image/etc/passwd && \
     echo "root:x:0:" > /image/etc/group && \
     cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
@@ -17,12 +18,12 @@ RUN mkdir -p /etc && \
     chmod 1777 /tmp
 
 FROM scratch
-COPY --from=collect / /
 VOLUME /var/lib/kubelet
 VOLUME /var/lib/rancher/k3s
 VOLUME /var/lib/cni
 VOLUME /var/log
-ENV PATH="$PATH:/bin/aux"
+COPY --from=collect / /
+ENV PATH="/var/lib/rancher/k3s/data/cni:$PATH:/bin/aux"
 ENV CRI_CONFIG_FILE="/var/lib/rancher/k3s/agent/etc/crictl.yaml"
 ENTRYPOINT ["/bin/k3s"]
 CMD ["agent"]

pkg/agent/config/config.go

@@ -9,6 +9,7 @@ import (
 	"crypto/x509"
 	"encoding/hex"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -32,7 +33,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/pkg/slice"
@@ -247,7 +248,7 @@ func upgradeOldNodePasswordPath(oldNodePasswordFile, newNodePasswordFile string)
 func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile, nodePasswordFile string, info *clientaccess.Info) error {
 	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to create certificate request %s", certFile)
 	}
 
 	basename := filepath.Base(certFile)
@@ -261,11 +262,11 @@ func getKubeletServingCert(nodeName string, nodeIPs []net.IP, certFile, keyFile,
 	// must be used instead of the one we signed the CSR with.
 	certBytes, keyBytes := splitCertKeyPEM(body)
 	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to write cert %s", certFile)
 	}
 	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", keyFile)
+			return pkgerrors.WithMessagef(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
@@ -279,7 +280,7 @@ func getHostFile(filename string, info *clientaccess.Info) error {
 		return err
 	}
 	if err := os.WriteFile(filename, fileBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", filename)
+		return pkgerrors.WithMessagef(err, "failed to write cert %s", filename)
 	}
 	return nil
 }
@@ -291,7 +292,7 @@ func getHostFile(filename string, info *clientaccess.Info) error {
 func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
 	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to create certificate request %s", certFile)
 	}
 
 	basename := filepath.Base(certFile)
@@ -305,11 +306,11 @@ func getClientCert(certFile, keyFile string, info *clientaccess.Info) error {
 	// must be used instead of the one we signed the CSR with.
 	certBytes, keyBytes := splitCertKeyPEM(fileBytes)
 	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to write cert %s", certFile)
 	}
 	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", keyFile)
+			return pkgerrors.WithMessagef(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
@@ -352,7 +353,7 @@ func splitCertKeyPEM(bytes []byte) (certPem []byte, keyPem []byte) {
 func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP, nodePasswordFile string, info *clientaccess.Info) error {
 	csr, err := getCSRBytes(keyFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create certificate request %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to create certificate request %s", certFile)
 	}
 
 	basename := filepath.Base(certFile)
@@ -366,11 +367,11 @@ func getKubeletClientCert(certFile, keyFile, nodeName string, nodeIPs []net.IP,
 	// must be used instead of the one we signed the CSR with.
 	certBytes, keyBytes := splitCertKeyPEM(body)
 	if err := os.WriteFile(certFile, certBytes, 0600); err != nil {
-		return errors.Wrapf(err, "failed to write cert %s", certFile)
+		return pkgerrors.WithMessagef(err, "failed to write cert %s", certFile)
 	}
 	if len(keyBytes) > 0 {
 		if err := os.WriteFile(keyFile, keyBytes, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write key %s", keyFile)
+			return pkgerrors.WithMessagef(err, "failed to write key %s", keyFile)
 		}
 	}
 	return nil
@@ -437,13 +438,13 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 
 	controlConfig, err := getConfig(info)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to retrieve configuration from server")
+		return nil, pkgerrors.WithMessage(err, "failed to retrieve configuration from server")
 	}
 	// If the supervisor and externally-facing apiserver are not on the same port, tell the proxy where to find the apiserver.
 	if controlConfig.SupervisorPort != controlConfig.HTTPSPort {
 		isIPv6 := utilsnet.IsIPv6(net.ParseIP(util.GetFirstValidIPString(envInfo.NodeIP)))
 		if err := proxy.SetAPIServerPort(controlConfig.HTTPSPort, isIPv6); err != nil {
-			return nil, errors.Wrapf(err, "failed to set apiserver port to %d", controlConfig.HTTPSPort)
+			return nil, pkgerrors.WithMessagef(err, "failed to set apiserver port to %d", controlConfig.HTTPSPort)
 		}
 	}
 	apiServerURL := proxy.APIServerURL()
@@ -452,7 +453,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	if controlConfig.FlannelBackend != config.FlannelBackendNone && len(envInfo.FlannelIface) > 0 {
 		flannelIface, err = net.InterfaceByName(envInfo.FlannelIface)
 		if err != nil {
-			return nil, errors.Wrapf(err, "unable to find interface %s", envInfo.FlannelIface)
+			return nil, pkgerrors.WithMessagef(err, "unable to find interface %s", envInfo.FlannelIface)
 		}
 	}
 
@@ -508,7 +509,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 				vpnIPs = append(vpnIPs, vpnInfo.IPv4Address)
 			}
 		} else {
-			return nil, errors.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
+			return nil, fmt.Errorf("address family mismatch when assigning VPN addresses to node: node=%v, VPN ipv4=%v ipv6=%v", nodeIPs, vpnInfo.IPv4Address, vpnInfo.IPv6Address)
 		}
 
 		// Overwrite nodeip and flannel interface and throw a warning if user explicitly set those parameters
@@ -523,7 +524,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 			nodeIPs = vpnIPs
 			flannelIface, err = net.InterfaceByName(vpnInfo.VPNInterface)
 			if err != nil {
-				return nil, errors.Wrapf(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface)
+				return nil, pkgerrors.WithMessagef(err, "unable to find vpn interface: %s", vpnInfo.VPNInterface)
 			}
 		}
 	}
@@ -558,12 +559,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 
 	// Ask the server to sign our kubelet server cert.
 	if err := getKubeletServingCert(nodeName, nodeExternalAndInternalIPs, servingKubeletCert, servingKubeletKey, newNodePasswordFile, info); err != nil {
-		return nil, errors.Wrap(err, servingKubeletCert)
+		return nil, pkgerrors.WithMessage(err, servingKubeletCert)
 	}
 
 	// Ask the server to sign our kubelet client cert.
 	if err := getKubeletClientCert(clientKubeletCert, clientKubeletKey, nodeName, nodeIPs, newNodePasswordFile, info); err != nil {
-		return nil, errors.Wrap(err, clientKubeletCert)
+		return nil, pkgerrors.WithMessage(err, clientKubeletCert)
 	}
 
 	// Generate a kubeconfig for the kubelet.
@@ -577,7 +578,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 
 	// Ask the server to sign our kube-proxy client cert.
 	if err := getClientCert(clientKubeProxyCert, clientKubeProxyKey, info); err != nil {
-		return nil, errors.Wrap(err, clientKubeProxyCert)
+		return nil, pkgerrors.WithMessage(err, clientKubeProxyCert)
 	}
 
 	// Generate a kubeconfig for kube-proxy.
@@ -591,7 +592,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 
 	// Ask the server to sign our agent controller client cert.
 	if err := getClientCert(clientK3sControllerCert, clientK3sControllerKey, info); err != nil {
-		return nil, errors.Wrap(err, clientK3sControllerCert)
+		return nil, pkgerrors.WithMessage(err, clientK3sControllerCert)
 	}
 
 	// Generate a kubeconfig for the agent controller.
@@ -653,7 +654,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	} else {
 		listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
+			return nil, pkgerrors.WithMessage(err, "cannot configure IPv4/IPv6 node-ip")
 		}
 		nodeConfig.AgentConfig.ListenAddress = listenAddress
 	}
@@ -684,7 +685,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	if !nodeConfig.NoFlannel {
 		hostLocal, err := exec.LookPath("host-local")
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to find host-local")
+			return nil, pkgerrors.WithMessagef(err, "failed to find host-local")
 		}
 
 		if envInfo.FlannelConf == "" {
@@ -851,7 +852,7 @@ func getKubeProxyDisabled(ctx context.Context, node *config.Node, proxy proxy.Pr
 
 	controlConfig, err := getConfig(info)
 	if err != nil {
-		return false, errors.Wrap(err, "failed to retrieve configuration from server")
+		return false, pkgerrors.WithMessage(err, "failed to retrieve configuration from server")
 	}
 
 	return controlConfig.DisableKubeProxy, nil

pkg/agent/config/config_linux.go

@@ -4,13 +4,14 @@
 package config
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 
 	"github.com/k3s-io/k3s/pkg/agent/containerd"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -24,17 +25,17 @@ func applyContainerdOSSpecificConfig(nodeConfig *config.Node) error {
 	switch nodeConfig.AgentConfig.Snapshotter {
 	case "overlayfs":
 		if err := containerd.OverlaySupported(nodeConfig.Containerd.Root); err != nil {
-			return errors.Wrapf(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
+			return pkgerrors.WithMessagef(err, "\"overlayfs\" snapshotter cannot be enabled for %q, try using \"fuse-overlayfs\" or \"native\"",
 				nodeConfig.Containerd.Root)
 		}
 	case "fuse-overlayfs":
 		if err := containerd.FuseoverlayfsSupported(nodeConfig.Containerd.Root); err != nil {
-			return errors.Wrapf(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
+			return pkgerrors.WithMessagef(err, "\"fuse-overlayfs\" snapshotter cannot be enabled for %q, try using \"native\"",
 				nodeConfig.Containerd.Root)
 		}
 	case "stargz":
 		if err := containerd.StargzSupported(nodeConfig.Containerd.Root); err != nil {
-			return errors.Wrapf(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
+			return pkgerrors.WithMessagef(err, "\"stargz\" snapshotter cannot be enabled for %q, try using \"overlayfs\" or \"native\"",
 				nodeConfig.Containerd.Root)
 		}
 		nodeConfig.AgentConfig.ImageServiceSocket = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"

pkg/agent/config/config_windows.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/permissions/pkg/access"
 	"github.com/rancher/permissions/pkg/acl"
 	"github.com/rancher/permissions/pkg/sid"
@@ -46,7 +46,7 @@ func configureACL(file string) error {
 		access.GrantSid(windows.GENERIC_ALL, sid.LocalSystem()),
 		access.GrantSid(windows.GENERIC_ALL, sid.BuiltinAdministrators()),
 	}...); err != nil {
-		return errors.Wrapf(err, "failed to configure Access Control List For %s", file)
+		return pkgerrors.WithMessagef(err, "failed to configure Access Control List For %s", file)
 	}
 
 	return nil

pkg/agent/containerd/config_linux.go

@@ -4,6 +4,7 @@
 package containerd
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/containerd/containerd"
@@ -16,7 +17,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/opencontainers/runc/libcontainer/userns"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/kubelet/util"
@@ -70,7 +71,7 @@ func SetupContainerdConfig(cfg *config.Node) error {
 
 	// Verifies if the DefaultRuntime can be found
 	if _, ok := extraRuntimes[cfg.DefaultRuntime]; !ok && cfg.DefaultRuntime != "" {
-		return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime)
+		return fmt.Errorf("default runtime %s was not found", cfg.DefaultRuntime)
 	}
 
 	containerdConfig := templates.ContainerdConfig{
@@ -88,7 +89,7 @@ func SetupContainerdConfig(cfg *config.Node) error {
 
 	selEnabled, selConfigured, err := selinuxStatus()
 	if err != nil {
-		return errors.Wrap(err, "failed to detect selinux")
+		return pkgerrors.WithMessage(err, "failed to detect selinux")
 	}
 	switch {
 	case !cfg.SELinux && selEnabled:

pkg/agent/containerd/config_windows.go

@@ -10,7 +10,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/templates"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	util3 "github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/kubernetes/pkg/kubelet/util"
 )
@@ -64,13 +64,13 @@ func Client(address string) (*containerd.Client, error) {
 }
 
 func OverlaySupported(root string) error {
-	return errors.Wrapf(util3.ErrUnsupportedPlatform, "overlayfs is not supported")
+	return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "overlayfs is not supported")
 }
 
 func FuseoverlayfsSupported(root string) error {
-	return errors.Wrapf(util3.ErrUnsupportedPlatform, "fuse-overlayfs is not supported")
+	return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "fuse-overlayfs is not supported")
 }
 
 func StargzSupported(root string) error {
-	return errors.Wrapf(util3.ErrUnsupportedPlatform, "stargz is not supported")
+	return pkgerrors.WithMessagef(util3.ErrUnsupportedPlatform, "stargz is not supported")
 }

pkg/agent/containerd/containerd.go

@@ -3,6 +3,7 @@ package containerd
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -24,7 +25,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/natefinch/lumberjack"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wharfie/pkg/tarfile"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/sirupsen/logrus"
@@ -137,12 +138,12 @@ func PreloadImages(ctx context.Context, cfg *config.Node) error {
 
 	// At startup all leases from k3s are cleared; we no longer use leases to lock content
 	if err := clearLeases(ctx, client); err != nil {
-		return errors.Wrap(err, "failed to clear leases")
+		return pkgerrors.WithMessage(err, "failed to clear leases")
 	}
 
 	// Clear the pinned labels on all images previously pinned by k3s
 	if err := clearLabels(ctx, client); err != nil {
-		return errors.Wrap(err, "failed to clear pinned labels")
+		return pkgerrors.WithMessage(err, "failed to clear pinned labels")
 	}
 
 	go watchImages(ctx, cfg)
@@ -199,7 +200,7 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien
 		logrus.Infof("Pulling images from %s", filePath)
 		images, err = prePullImages(ctx, client, imageClient, file)
 		if err != nil {
-			return errors.Wrap(err, "failed to pull images from "+filePath)
+			return pkgerrors.WithMessage(err, "failed to pull images from "+filePath)
 		}
 	} else {
 		opener, err := tarfile.GetOpener(filePath)
@@ -216,15 +217,15 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien
 		logrus.Infof("Importing images from %s", filePath)
 		images, err = client.Import(ctx, imageReader, containerd.WithAllPlatforms(true), containerd.WithSkipMissing())
 		if err != nil {
-			return errors.Wrap(err, "failed to import images from "+filePath)
+			return pkgerrors.WithMessage(err, "failed to import images from "+filePath)
 		}
 	}
 
 	if err := labelImages(ctx, client, images, filepath.Base(filePath)); err != nil {
-		return errors.Wrap(err, "failed to add pinned label to images")
+		return pkgerrors.WithMessage(err, "failed to add pinned label to images")
 	}
 	if err := retagImages(ctx, client, images, cfg.AgentConfig.AirgapExtraRegistry); err != nil {
-		return errors.Wrap(err, "failed to retag images")
+		return pkgerrors.WithMessage(err, "failed to retag images")
 	}
 
 	for _, image := range images {
@@ -263,7 +264,7 @@ func clearLabels(ctx context.Context, client *containerd.Client) error {
 		delete(image.Labels, k3sPinnedImageLabelKey)
 		delete(image.Labels, labels.PinnedImageLabelKey)
 		if _, err := imageService.Update(ctx, image, "labels"); err != nil {
-			errs = append(errs, errors.Wrap(err, "failed to delete labels from image "+image.Name))
+			errs = append(errs, pkgerrors.WithMessage(err, "failed to delete labels from image "+image.Name))
 		}
 	}
 	return merr.NewErrors(errs...)
@@ -288,7 +289,7 @@ func labelImages(ctx context.Context, client *containerd.Client, images []images
 		image.Labels[labels.PinnedImageLabelKey] = labels.PinnedImageLabelValue
 		updatedImage, err := imageService.Update(ctx, image, "labels")
 		if err != nil {
-			errs = append(errs, errors.Wrap(err, "failed to add labels to image "+image.Name))
+			errs = append(errs, pkgerrors.WithMessage(err, "failed to add labels to image "+image.Name))
 		} else {
 			images[i] = updatedImage
 		}
@@ -305,7 +306,7 @@ func retagImages(ctx context.Context, client *containerd.Client, images []images
 	for _, image := range images {
 		name, err := parseNamedTagged(image.Name)
 		if err != nil {
-			errs = append(errs, errors.Wrap(err, "failed to parse tags for image "+image.Name))
+			errs = append(errs, pkgerrors.WithMessage(err, "failed to parse tags for image "+image.Name))
 			continue
 		}
 		for _, registry := range registries {
@@ -317,15 +318,15 @@ func retagImages(ctx context.Context, client *containerd.Client, images []images
 			if _, err = imageService.Create(ctx, image); err != nil {
 				if errdefs.IsAlreadyExists(err) {
 					if err = imageService.Delete(ctx, image.Name); err != nil {
-						errs = append(errs, errors.Wrap(err, "failed to delete existing image "+image.Name))
+						errs = append(errs, pkgerrors.WithMessage(err, "failed to delete existing image "+image.Name))
 						continue
 					}
 					if _, err = imageService.Create(ctx, image); err != nil {
-						errs = append(errs, errors.Wrap(err, "failed to tag after deleting existing image "+image.Name))
+						errs = append(errs, pkgerrors.WithMessage(err, "failed to tag after deleting existing image "+image.Name))
 						continue
 					}
 				} else {
-					errs = append(errs, errors.Wrap(err, "failed to tag image "+image.Name))
+					errs = append(errs, pkgerrors.WithMessage(err, "failed to tag image "+image.Name))
 					continue
 				}
 			}

pkg/agent/containerd/watcher.go

@@ -13,7 +13,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/k3s-io/k3s/pkg/agent/cri"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wharfie/pkg/tarfile"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/sirupsen/logrus"
@@ -52,7 +52,7 @@ func isFileSupported(path string) bool {
 
 func (w *Watcher) HandleWatch(path string) error {
 	if err := w.watcher.Add(path); err != nil {
-		return errors.Wrap(err, fmt.Sprintf("failed to watch from %s directory: %v", path, err))
+		return pkgerrors.WithMessage(err, fmt.Sprintf("failed to watch from %s directory: %v", path, err))
 	}
 
 	return nil

pkg/agent/cridockerd/config_linux.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/docker/docker/client"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 const socketPrefix = "unix://"
@@ -25,11 +25,11 @@ func setupDockerCRIConfig(ctx context.Context, cfg *config.Node) error {
 	}
 	c, err := client.NewClientWithOpts(clientOpts...)
 	if err != nil {
-		return errors.Wrap(err, "failed to create docker client")
+		return pkgerrors.WithMessage(err, "failed to create docker client")
 	}
 	i, err := c.Info(ctx)
 	if err != nil {
-		return errors.Wrap(err, "failed to get docker runtime info")
+		return pkgerrors.WithMessage(err, "failed to get docker runtime info")
 	}
 	// note: this mutatation of the passed agent.Config is later used to set the
 	// kubelet's cgroup-driver flag. This may merit moving to somewhere else in order

pkg/agent/flannel/flannel.go

@@ -15,6 +15,7 @@
 package flannel
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 	"net"
@@ -28,7 +29,7 @@ import (
 	"github.com/flannel-io/flannel/pkg/subnet/kube"
 	"github.com/flannel-io/flannel/pkg/trafficmngr/iptables"
 	"github.com/joho/godotenv"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 
@@ -53,7 +54,7 @@ var (
 func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, flannelIPv6Masq bool, netMode int) error {
 	extIface, err := LookupExtInterface(flannelIface, netMode)
 	if err != nil {
-		return errors.Wrap(err, "failed to find the interface")
+		return pkgerrors.WithMessage(err, "failed to find the interface")
 	}
 
 	sm, err := kube.NewSubnetManager(ctx,
@@ -63,12 +64,12 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 		flannelConf,
 		false)
 	if err != nil {
-		return errors.Wrap(err, "failed to create the SubnetManager")
+		return pkgerrors.WithMessage(err, "failed to create the SubnetManager")
 	}
 
 	config, err := sm.GetNetworkConfig(ctx)
 	if err != nil {
-		return errors.Wrap(err, "failed to get the network config")
+		return pkgerrors.WithMessage(err, "failed to get the network config")
 	}
 
 	// Create a backend manager then use it to create the backend and register the network with it.
@@ -76,17 +77,17 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 
 	be, err := bm.GetBackend(config.BackendType)
 	if err != nil {
-		return errors.Wrap(err, "failed to create the flannel backend")
+		return pkgerrors.WithMessage(err, "failed to create the flannel backend")
 	}
 
 	bn, err := be.RegisterNetwork(ctx, &sync.WaitGroup{}, config)
 	if err != nil {
-		return errors.Wrap(err, "failed to register flannel network")
+		return pkgerrors.WithMessage(err, "failed to register flannel network")
 	}
 	trafficMngr := &iptables.IPTablesManager{}
 	err = trafficMngr.Init(ctx, &sync.WaitGroup{})
 	if err != nil {
-		return errors.Wrap(err, "failed to initialize flannel ipTables manager")
+		return pkgerrors.WithMessage(err, "failed to initialize flannel ipTables manager")
 	}
 
 	if netMode == (ipv4+ipv6) || netMode == ipv4 {
@@ -108,7 +109,7 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 		err = trafficMngr.SetupAndEnsureMasqRules(ctx, config.Network, prevSubnet, prevNetwork, ip.IP6Net{}, prevIPv6Subnet, prevIPv6Network, bn.Lease(), 60)
 	}
 	if err != nil {
-		return errors.Wrap(err, "failed to setup masq rules")
+		return pkgerrors.WithMessage(err, "failed to setup masq rules")
 	}
 
 	//setup forward rules
@@ -136,11 +137,11 @@ func LookupExtInterface(iface *net.Interface, netMode int) (*backend.ExternalInt
 		logrus.Debug("No interface defined for flannel in the config. Fetching the default gateway interface")
 		if netMode == ipv4 || netMode == (ipv4+ipv6) {
 			if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
-				return nil, errors.Wrap(err, "failed to get default interface")
+				return nil, pkgerrors.WithMessage(err, "failed to get default interface")
 			}
 		} else {
 			if iface, err = ip.GetDefaultV6GatewayInterface(); err != nil {
-				return nil, errors.Wrap(err, "failed to get default interface")
+				return nil, pkgerrors.WithMessage(err, "failed to get default interface")
 			}
 		}
 	}
@@ -150,14 +151,14 @@ func LookupExtInterface(iface *net.Interface, netMode int) (*backend.ExternalInt
 	case ipv4:
 		ifaceAddr, err = ip.GetInterfaceIP4Addrs(iface)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to find IPv4 address for interface")
+			return nil, pkgerrors.WithMessage(err, "failed to find IPv4 address for interface")
 		}
 		logrus.Infof("The interface %s with ipv4 address %s will be used by flannel", iface.Name, ifaceAddr[0])
 		ifacev6Addr = append(ifacev6Addr, nil)
 	case ipv6:
 		ifacev6Addr, err = ip.GetInterfaceIP6Addrs(iface)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to find IPv6 address for interface")
+			return nil, pkgerrors.WithMessage(err, "failed to find IPv6 address for interface")
 		}
 		logrus.Infof("The interface %s with ipv6 address %s will be used by flannel", iface.Name, ifacev6Addr[0])
 		ifaceAddr = append(ifaceAddr, nil)
@@ -264,7 +265,6 @@ func ReadCIDRsFromSubnetFile(path string, CIDRKey string) []ip.IP4Net {
 	return prevCIDRs
 }
 
-
 // ReadIP6CIDRFromSubnetFile reads the flannel subnet file and extracts the value of IPv6 network CIDRKey
 func ReadIP6CIDRFromSubnetFile(path string, CIDRKey string) ip.IP6Net {
 	prevCIDRs := ReadIP6CIDRsFromSubnetFile(path, CIDRKey)

pkg/agent/flannel/setup.go

@@ -2,6 +2,7 @@ package flannel
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -12,7 +13,7 @@ import (
 	agentutil "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	v1 "k8s.io/api/core/v1"
@@ -75,11 +76,11 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	// Compatibility code for AuthorizeNodeWithSelectors feature-gate.
 	// If the kubelet cannot list nodes, then wait for the k3s-controller RBAC to become ready, and use that kubeconfig instead.
 	if canListNodes, err := util.CheckRBAC(ctx, kubeConfig, resourceAttrs, ""); err != nil {
-		return errors.Wrap(err, "failed to check if RBAC allows node list")
+		return pkgerrors.WithMessage(err, "failed to check if RBAC allows node list")
 	} else if !canListNodes {
 		kubeConfig = nodeConfig.AgentConfig.KubeConfigK3sController
 		if err := util.WaitForRBACReady(ctx, kubeConfig, util.DefaultAPIServerReadyTimeout, resourceAttrs, ""); err != nil {
-			return errors.Wrap(err, "flannel failed to wait for RBAC")
+			return pkgerrors.WithMessage(err, "flannel failed to wait for RBAC")
 		}
 	}
 
@@ -89,12 +90,12 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	}
 
 	if err := waitForPodCIDR(ctx, nodeConfig.AgentConfig.NodeName, coreClient.CoreV1().Nodes()); err != nil {
-		return errors.Wrap(err, "flannel failed to wait for PodCIDR assignment")
+		return pkgerrors.WithMessage(err, "flannel failed to wait for PodCIDR assignment")
 	}
 
 	netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
 	if err != nil {
-		return errors.Wrap(err, "failed to check netMode for flannel")
+		return pkgerrors.WithMessage(err, "failed to check netMode for flannel")
 	}
 	go func() {
 		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConfFile, kubeConfig, nodeConfig.FlannelIPv6Masq, netMode)
@@ -129,7 +130,7 @@ func waitForPodCIDR(ctx context.Context, nodeName string, nodes typedcorev1.Node
 	}
 
 	if _, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition); err != nil {
-		return errors.Wrap(err, "failed to wait for PodCIDR assignment")
+		return pkgerrors.WithMessage(err, "failed to wait for PodCIDR assignment")
 	}
 
 	logrus.Info("Flannel found PodCIDR assigned for node " + nodeName)

pkg/agent/https/https.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/k3s-io/api/pkg/generated/clientset/versioned/scheme"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"k8s.io/apiserver/pkg/authentication/authenticator"

pkg/agent/loadbalancer/httpproxy.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/version"
 	http_dialer "github.com/mwitkow/go-http-dialer"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/http/httpproxy"
 	"golang.org/x/net/proxy"
@@ -32,14 +32,14 @@ func SetHTTPProxy(address string) error {
 
 	serverURL, err := url.Parse(address)
 	if err != nil {
-		return errors.Wrapf(err, "failed to parse address %s", address)
+		return pkgerrors.WithMessagef(err, "failed to parse address %s", address)
 	}
 
 	// Call this directly instead of using the cached environment used by http.ProxyFromEnvironment to allow for testing
 	proxyFromEnvironment := httpproxy.FromEnvironment().ProxyFunc()
 	proxyURL, err := proxyFromEnvironment(serverURL)
 	if err != nil {
-		return errors.Wrapf(err, "failed to get proxy for address %s", address)
+		return pkgerrors.WithMessagef(err, "failed to get proxy for address %s", address)
 	}
 	if proxyURL == nil {
 		logrus.Debug(version.ProgramUpper + "_AGENT_HTTP_PROXY_ALLOWED is true but no proxy is configured for URL " + serverURL.String())
@@ -48,7 +48,7 @@ func SetHTTPProxy(address string) error {
 
 	dialer, err := proxyDialer(proxyURL, defaultDialer)
 	if err != nil {
-		return errors.Wrapf(err, "failed to create proxy dialer for %s", proxyURL)
+		return pkgerrors.WithMessagef(err, "failed to create proxy dialer for %s", proxyURL)
 	}
 
 	defaultDialer = dialer

pkg/agent/loadbalancer/loadbalancer.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/inetaf/tcpproxy"
+	"github.com/k3s-io/k3s/pkg/util/metrics"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/sirupsen/logrus"
 )
@@ -99,13 +100,8 @@ func New(ctx context.Context, dataDir, serviceName, defaultServerURL string, lbS
 		OnDialError: onDialError,
 		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
 			start := time.Now()
-			status := "success"
 			conn, err := lb.servers.dialContext(ctx, network, address)
-			latency := time.Since(start)
-			if err != nil {
-				status = "error"
-			}
-			loadbalancerDials.WithLabelValues(serviceName, status).Observe(latency.Seconds())
+			metrics.ObserveWithStatus(loadbalancerDials, start, err, serviceName)
 			return conn, err
 		},
 	})

pkg/agent/netpol/netpol.go

@@ -27,7 +27,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/metrics"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1core "k8s.io/api/core/v1"
 	"k8s.io/client-go/informers"
@@ -86,7 +86,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 		}
 		return true, nil
 	}); err != nil {
-		return errors.Wrapf(err, "network policy controller failed to wait for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName)
+		return pkgerrors.WithMessagef(err, "network policy controller failed to wait for %s taint to be removed from Node %s", cloudproviderapi.TaintExternalCloudProvider, nodeConfig.AgentConfig.NodeName)
 	}
 
 	krConfig := options.NewKubeRouterConfig()
@@ -123,13 +123,13 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	if nodeConfig.AgentConfig.EnableIPv4 {
 		iptHandler, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
 		if err != nil {
-			return errors.Wrap(err, "failed to create iptables handler")
+			return pkgerrors.WithMessage(err, "failed to create iptables handler")
 		}
 		iptablesCmdHandlers[v1core.IPv4Protocol] = iptHandler
 
 		ipset, err := utils.NewIPSet(false)
 		if err != nil {
-			return errors.Wrap(err, "failed to create ipset handler")
+			return pkgerrors.WithMessage(err, "failed to create ipset handler")
 		}
 		ipSetHandlers[v1core.IPv4Protocol] = ipset
 	}
@@ -137,13 +137,13 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	if nodeConfig.AgentConfig.EnableIPv6 {
 		ipt6Handler, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
 		if err != nil {
-			return errors.Wrap(err, "failed to create iptables handler")
+			return pkgerrors.WithMessage(err, "failed to create iptables handler")
 		}
 		iptablesCmdHandlers[v1core.IPv6Protocol] = ipt6Handler
 
 		ipset, err := utils.NewIPSet(true)
 		if err != nil {
-			return errors.Wrap(err, "failed to create ipset handler")
+			return pkgerrors.WithMessage(err, "failed to create ipset handler")
 		}
 		ipSetHandlers[v1core.IPv6Protocol] = ipset
 	}
@@ -172,7 +172,7 @@ func Run(ctx context.Context, nodeConfig *config.Node) error {
 	npc, err := netpol.NewNetworkPolicyController(client, krConfig, podInformer, npInformer, nsInformer, &sync.Mutex{},
 		iptablesCmdHandlers, ipSetHandlers)
 	if err != nil {
-		return errors.Wrap(err, "unable to initialize network policy controller")
+		return pkgerrors.WithMessage(err, "unable to initialize network policy controller")
 	}
 
 	podInformer.AddEventHandler(npc.PodEventHandler)

pkg/agent/proxy/apiproxy.go

@@ -10,7 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 type Proxy interface {
@@ -58,7 +58,7 @@ func NewSupervisorProxy(ctx context.Context, lbEnabled bool, dataDir, supervisor
 
 	u, err := url.Parse(p.initialSupervisorURL)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to parse %s", p.initialSupervisorURL)
+		return nil, pkgerrors.WithMessagef(err, "failed to parse %s", p.initialSupervisorURL)
 	}
 	p.fallbackSupervisorAddress = u.Host
 	p.supervisorPort = u.Port()
@@ -140,7 +140,7 @@ func (p *proxy) SetAPIServerPort(port int, isIPv6 bool) error {
 
 	u, err := url.Parse(p.initialSupervisorURL)
 	if err != nil {
-		return errors.Wrapf(err, "failed to parse server URL %s", p.initialSupervisorURL)
+		return pkgerrors.WithMessagef(err, "failed to parse server URL %s", p.initialSupervisorURL)
 	}
 	p.apiServerPort = strconv.Itoa(port)
 	u.Host = sysnet.JoinHostPort(u.Hostname(), p.apiServerPort)

pkg/agent/run.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -34,7 +35,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/spegel"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -57,20 +58,20 @@ import (
 func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	nodeConfig, err := config.Get(ctx, cfg, proxy)
 	if err != nil {
-		return errors.Wrap(err, "failed to retrieve agent configuration")
+		return pkgerrors.WithMessage(err, "failed to retrieve agent configuration")
 	}
 
 	dualCluster, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ClusterCIDRs)
 	if err != nil {
-		return errors.Wrap(err, "failed to validate cluster-cidr")
+		return pkgerrors.WithMessage(err, "failed to validate cluster-cidr")
 	}
 	dualService, err := utilsnet.IsDualStackCIDRs(nodeConfig.AgentConfig.ServiceCIDRs)
 	if err != nil {
-		return errors.Wrap(err, "failed to validate service-cidr")
+		return pkgerrors.WithMessage(err, "failed to validate service-cidr")
 	}
 	dualNode, err := utilsnet.IsDualStackIPs(nodeConfig.AgentConfig.NodeIPs)
 	if err != nil {
-		return errors.Wrap(err, "failed to validate node-ip")
+		return pkgerrors.WithMessage(err, "failed to validate node-ip")
 	}
 	serviceIPv4 := utilsnet.IsIPv4CIDR(nodeConfig.AgentConfig.ServiceCIDR)
 	clusterIPv4 := utilsnet.IsIPv4CIDR(nodeConfig.AgentConfig.ClusterCIDR)
@@ -99,7 +100,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 
 	conntrackConfig, err := getConntrackConfig(nodeConfig)
 	if err != nil {
-		return errors.Wrap(err, "failed to validate kube-proxy conntrack configuration")
+		return pkgerrors.WithMessage(err, "failed to validate kube-proxy conntrack configuration")
 	}
 	syssetup.Configure(enableIPv6, conntrackConfig)
 	nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
@@ -111,19 +112,19 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 		}
 
 		if err := spegel.DefaultRegistry.Start(ctx, nodeConfig); err != nil {
-			return errors.Wrap(err, "failed to start embedded registry")
+			return pkgerrors.WithMessage(err, "failed to start embedded registry")
 		}
 	}
 
 	if nodeConfig.SupervisorMetrics {
 		if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil {
-			return errors.Wrap(err, "failed to serve metrics")
+			return pkgerrors.WithMessage(err, "failed to serve metrics")
 		}
 	}
 
 	if nodeConfig.EnablePProf {
 		if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil {
-			return errors.Wrap(err, "failed to serve pprof")
+			return pkgerrors.WithMessage(err, "failed to serve pprof")
 		}
 	}
 
@@ -174,7 +175,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	}
 
 	if err := util.WaitForAPIServerReady(ctx, nodeConfig.AgentConfig.KubeConfigKubelet, util.DefaultAPIServerReadyTimeout); err != nil {
-		return errors.Wrap(err, "failed to wait for apiserver ready")
+		return pkgerrors.WithMessage(err, "failed to wait for apiserver ready")
 	}
 
 	// Use the kubelet kubeconfig to update annotations on the local node
@@ -267,7 +268,7 @@ func RunStandalone(ctx context.Context, cfg cmds.Agent) error {
 
 	nodeConfig, err := config.Get(ctx, cfg, proxy)
 	if err != nil {
-		return errors.Wrap(err, "failed to retrieve agent configuration")
+		return pkgerrors.WithMessage(err, "failed to retrieve agent configuration")
 	}
 
 	if err := executor.Bootstrap(ctx, nodeConfig, cfg); err != nil {
@@ -287,13 +288,13 @@ func RunStandalone(ctx context.Context, cfg cmds.Agent) error {
 
 	if nodeConfig.SupervisorMetrics {
 		if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil {
-			return errors.Wrap(err, "failed to serve metrics")
+			return pkgerrors.WithMessage(err, "failed to serve metrics")
 		}
 	}
 
 	if nodeConfig.EnablePProf {
 		if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil {
-			return errors.Wrap(err, "failed to serve pprof")
+			return pkgerrors.WithMessage(err, "failed to serve pprof")
 		}
 	}
 
@@ -429,7 +430,7 @@ func configureNode(ctx context.Context, nodeConfig *daemonconfig.Node, nodes typ
 	}
 
 	if _, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition); err != nil {
-		return errors.Wrap(err, "failed to configure node")
+		return pkgerrors.WithMessage(err, "failed to configure node")
 	}
 	return nil
 }

pkg/agent/util/file.go

 package util
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 func WriteFile(name string, content string) error {
 	os.MkdirAll(filepath.Dir(name), 0755)
 	err := os.WriteFile(name, []byte(content), 0644)
 	if err != nil {
-		return errors.Wrapf(err, "writing %s", name)
+		return pkgerrors.WithMessagef(err, "writing %s", name)
 	}
 	return nil
 }
@@ -22,11 +23,11 @@ func CopyFile(sourceFile string, destinationFile string, ignoreNotExist bool) er
 	if errors.Is(err, os.ErrNotExist) && ignoreNotExist {
 		return nil
 	} else if err != nil {
-		return errors.Wrapf(err, "copying %s to %s", sourceFile, destinationFile)
+		return pkgerrors.WithMessagef(err, "copying %s to %s", sourceFile, destinationFile)
 	}
 	err = os.WriteFile(destinationFile, input, 0644)
 	if err != nil {
-		return errors.Wrapf(err, "copying %s to %s", sourceFile, destinationFile)
+		return pkgerrors.WithMessagef(err, "copying %s to %s", sourceFile, destinationFile)
 	}
 	return nil
 }

pkg/apis/k3s.cattle.io/v1/doc.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// +k8s:deepcopy-gen=package
-// +groupName=k3s.cattle.io
-package v1

pkg/apis/k3s.cattle.io/v1/types.go

-package v1
-
-import (
-	"k8s.io/apimachinery/pkg/api/resource"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// +genclient
-// +genclient:noStatus
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-// Addon is used to track application of a manifest file on disk. It mostly exists so that the wrangler DesiredSet
-// Apply controller has an object to track as the owner, and ensure that all created resources are tracked when the
-// manifest is modified or removed.
-type Addon struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	// Spec provides information about the on-disk manifest backing this resource.
-	Spec AddonSpec `json:"spec,omitempty"`
-}
-
-type AddonSpec struct {
-	// Source is the Path on disk to the manifest file that this Addon tracks.
-	Source string `json:"source,omitempty" column:""`
-	// Checksum is the SHA256 checksum of the most recently successfully applied manifest file.
-	Checksum string `json:"checksum,omitempty" column:""`
-}
-
-// +genclient
-// +genclient:nonNamespaced
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-// ETCDSnapshot tracks a point-in-time snapshot of the etcd datastore.
-type ETCDSnapshotFile struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	// Spec defines properties of an etcd snapshot file
-	Spec ETCDSnapshotSpec `json:"spec,omitempty"`
-	// Status represents current information about a snapshot.
-	Status ETCDSnapshotStatus `json:"status,omitempty"`
-}
-
-// ETCDSnapshotSpec desribes an etcd snapshot file
-type ETCDSnapshotSpec struct {
-	// SnapshotName contains the base name of the snapshot file. CLI actions that act
-	// on snapshots stored locally or within a pre-configured S3 bucket and
-	// prefix usually take the snapshot name as their argument.
-	SnapshotName string `json:"snapshotName" column:""`
-	// NodeName contains the name of the node that took the snapshot.
-	NodeName string `json:"nodeName" column:"name=Node"`
-	// Location is the absolute file:// or s3:// URI address of the snapshot.
-	Location string `json:"location" column:""`
-	// Metadata contains point-in-time snapshot of the contents of the
-	// k3s-etcd-snapshot-extra-metadata ConfigMap's data field, at the time the
-	// snapshot was taken. This is intended to contain data about cluster state
-	// that may be important for an external system to have available when restoring
-	// the snapshot.
-	Metadata map[string]string `json:"metadata,omitempty"`
-	// S3 contains extra metadata about the S3 storage system holding the
-	// snapshot. This is guaranteed to be set for all snapshots uploaded to S3.
-	// If not specified, the snapshot was not uploaded to S3.
-	S3 *ETCDSnapshotS3 `json:"s3,omitempty"`
-}
-
-// ETCDSnapshotS3 holds information about the S3 storage system holding the snapshot.
-type ETCDSnapshotS3 struct {
-	// Endpoint is the host or host:port of the S3 service
-	Endpoint string `json:"endpoint,omitempty"`
-	// EndpointCA is the path on disk to the S3 service's trusted CA list. Leave empty to use the OS CA bundle.
-	EndpointCA string `json:"endpointCA,omitempty"`
-	// SkipSSLVerify is true if TLS certificate verification is disabled
-	SkipSSLVerify bool `json:"skipSSLVerify,omitempty"`
-	// Bucket is the bucket holding the snapshot
-	Bucket string `json:"bucket,omitempty"`
-	// Region is the region of the S3 service
-	Region string `json:"region,omitempty"`
-	// Prefix is the prefix in which the snapshot file is stored.
-	Prefix string `json:"prefix,omitempty"`
-	// Insecure is true if the S3 service uses HTTP instead of HTTPS
-	Insecure bool `json:"insecure,omitempty"`
-}
-
-// ETCDSnapshotStatus is the status of the ETCDSnapshotFile object.
-type ETCDSnapshotStatus struct {
-	// Size is the size of the snapshot file, in bytes. If not specified, the snapshot failed.
-	Size *resource.Quantity `json:"size,omitempty" column:""`
-	// CreationTime is the timestamp when the snapshot was taken by etcd.
-	CreationTime *metav1.Time `json:"creationTime,omitempty" column:""`
-	// ReadyToUse indicates that the snapshot is available to be restored.
-	ReadyToUse *bool `json:"readyToUse,omitempty"`
-	// Error is the last observed error during snapshot creation, if any.
-	// If the snapshot is retried, this field will be cleared on success.
-	Error *ETCDSnapshotError `json:"error,omitempty"`
-}
-
-// ETCDSnapshotError describes an error encountered during snapshot creation.
-type ETCDSnapshotError struct {
-	// Time is the timestamp when the error was encountered.
-	Time *metav1.Time `json:"time,omitempty"`
-	// Message is a string detailing the encountered error during snapshot creation if specified.
-	// NOTE: message may be logged, and it should not contain sensitive information.
-	Message *string `json:"message,omitempty"`
-}

pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go

-//go:build !ignore_autogenerated
-// +build !ignore_autogenerated
-
-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	runtime "k8s.io/apimachinery/pkg/runtime"
-)
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *Addon) DeepCopyInto(out *Addon) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Addon.
-func (in *Addon) DeepCopy() *Addon {
-	if in == nil {
-		return nil
-	}
-	out := new(Addon)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *Addon) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *AddonList) DeepCopyInto(out *AddonList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]Addon, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonList.
-func (in *AddonList) DeepCopy() *AddonList {
-	if in == nil {
-		return nil
-	}
-	out := new(AddonList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *AddonList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *AddonSpec) DeepCopyInto(out *AddonSpec) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonSpec.
-func (in *AddonSpec) DeepCopy() *AddonSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(AddonSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotError) DeepCopyInto(out *ETCDSnapshotError) {
-	*out = *in
-	if in.Time != nil {
-		in, out := &in.Time, &out.Time
-		*out = (*in).DeepCopy()
-	}
-	if in.Message != nil {
-		in, out := &in.Message, &out.Message
-		*out = new(string)
-		**out = **in
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotError.
-func (in *ETCDSnapshotError) DeepCopy() *ETCDSnapshotError {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotError)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotFile) DeepCopyInto(out *ETCDSnapshotFile) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	in.Spec.DeepCopyInto(&out.Spec)
-	in.Status.DeepCopyInto(&out.Status)
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotFile.
-func (in *ETCDSnapshotFile) DeepCopy() *ETCDSnapshotFile {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotFile)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *ETCDSnapshotFile) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotFileList) DeepCopyInto(out *ETCDSnapshotFileList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]ETCDSnapshotFile, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotFileList.
-func (in *ETCDSnapshotFileList) DeepCopy() *ETCDSnapshotFileList {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotFileList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *ETCDSnapshotFileList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotS3) DeepCopyInto(out *ETCDSnapshotS3) {
-	*out = *in
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotS3.
-func (in *ETCDSnapshotS3) DeepCopy() *ETCDSnapshotS3 {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotS3)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotSpec) DeepCopyInto(out *ETCDSnapshotSpec) {
-	*out = *in
-	if in.Metadata != nil {
-		in, out := &in.Metadata, &out.Metadata
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-	if in.S3 != nil {
-		in, out := &in.S3, &out.S3
-		*out = new(ETCDSnapshotS3)
-		**out = **in
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotSpec.
-func (in *ETCDSnapshotSpec) DeepCopy() *ETCDSnapshotSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ETCDSnapshotStatus) DeepCopyInto(out *ETCDSnapshotStatus) {
-	*out = *in
-	if in.Size != nil {
-		in, out := &in.Size, &out.Size
-		x := (*in).DeepCopy()
-		*out = &x
-	}
-	if in.CreationTime != nil {
-		in, out := &in.CreationTime, &out.CreationTime
-		*out = (*in).DeepCopy()
-	}
-	if in.ReadyToUse != nil {
-		in, out := &in.ReadyToUse, &out.ReadyToUse
-		*out = new(bool)
-		**out = **in
-	}
-	if in.Error != nil {
-		in, out := &in.Error, &out.Error
-		*out = new(ETCDSnapshotError)
-		(*in).DeepCopyInto(*out)
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETCDSnapshotStatus.
-func (in *ETCDSnapshotStatus) DeepCopy() *ETCDSnapshotStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(ETCDSnapshotStatus)
-	in.DeepCopyInto(out)
-	return out
-}

pkg/apis/k3s.cattle.io/v1/zz_generated_list_types.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// +k8s:deepcopy-gen=package
-// +groupName=k3s.cattle.io
-package v1
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-// AddonList is a list of Addon resources
-type AddonList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata"`
-
-	Items []Addon `json:"items"`
-}
-
-func NewAddon(namespace, name string, obj Addon) *Addon {
-	obj.APIVersion, obj.Kind = SchemeGroupVersion.WithKind("Addon").ToAPIVersionAndKind()
-	obj.Name = name
-	obj.Namespace = namespace
-	return &obj
-}
-
-// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-
-// ETCDSnapshotFileList is a list of ETCDSnapshotFile resources
-type ETCDSnapshotFileList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata"`
-
-	Items []ETCDSnapshotFile `json:"items"`
-}
-
-func NewETCDSnapshotFile(namespace, name string, obj ETCDSnapshotFile) *ETCDSnapshotFile {
-	obj.APIVersion, obj.Kind = SchemeGroupVersion.WithKind("ETCDSnapshotFile").ToAPIVersionAndKind()
-	obj.Name = name
-	obj.Namespace = namespace
-	return &obj
-}

pkg/apis/k3s.cattle.io/v1/zz_generated_register.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// +k8s:deepcopy-gen=package
-// +groupName=k3s.cattle.io
-package v1
-
-import (
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-var (
-	AddonResourceName            = "addons"
-	ETCDSnapshotFileResourceName = "etcdsnapshotfiles"
-)
-
-// SchemeGroupVersion is group version used to register these objects
-var SchemeGroupVersion = schema.GroupVersion{Group: k3s.GroupName, Version: "v1"}
-
-// Kind takes an unqualified kind and returns back a Group qualified GroupKind
-func Kind(kind string) schema.GroupKind {
-	return SchemeGroupVersion.WithKind(kind).GroupKind()
-}
-
-// Resource takes an unqualified resource and returns a Group qualified GroupResource
-func Resource(resource string) schema.GroupResource {
-	return SchemeGroupVersion.WithResource(resource).GroupResource()
-}
-
-var (
-	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
-	AddToScheme   = SchemeBuilder.AddToScheme
-)
-
-// Adds the list of known types to Scheme.
-func addKnownTypes(scheme *runtime.Scheme) error {
-	scheme.AddKnownTypes(SchemeGroupVersion,
-		&Addon{},
-		&AddonList{},
-		&ETCDSnapshotFile{},
-		&ETCDSnapshotFileList{},
-	)
-	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
-	return nil
-}

pkg/apis/k3s.cattle.io/zz_generated_register.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package k3s
-
-const (
-	// Package-wide consts from generator "zz_generated_register".
-	GroupName = "k3s.cattle.io"
-)

pkg/bootstrap/bootstrap.go

@@ -3,23 +3,15 @@ package bootstrap
 import (
 	"encoding/json"
 	"io"
-	"net/http"
 	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-func Handler(bootstrap *config.ControlRuntimeBootstrap) http.Handler {
-	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
-		rw.Header().Set("Content-Type", "application/json")
-		ReadFromDisk(rw, bootstrap)
-	})
-}
-
 // ReadFromDisk reads the bootstrap data from the files on disk and
 // writes their content in JSON form to the given io.Writer.
 func ReadFromDisk(w io.Writer, bootstrap *config.ControlRuntimeBootstrap) error {
@@ -79,13 +71,13 @@ func WriteToDiskFromStorage(files PathsDataformat, bootstrap *config.ControlRunt
 		}
 
 		if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
-			return errors.Wrapf(err, "failed to mkdir %s", filepath.Dir(path))
+			return pkgerrors.WithMessagef(err, "failed to mkdir %s", filepath.Dir(path))
 		}
 		if err := os.WriteFile(path, bsf.Content, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write to %s", path)
+			return pkgerrors.WithMessagef(err, "failed to write to %s", path)
 		}
 		if err := os.Chtimes(path, bsf.Timestamp, bsf.Timestamp); err != nil {
-			return errors.Wrapf(err, "failed to update modified time on %s", path)
+			return pkgerrors.WithMessagef(err, "failed to update modified time on %s", path)
 		}
 	}
 

pkg/cli/agent/agent.go

@@ -21,7 +21,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util/permissions"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -48,7 +48,7 @@ func Run(ctx *cli.Context) error {
 
 	if !cmds.AgentConfig.Rootless {
 		if err := permissions.IsPrivileged(); err != nil {
-			return errors.Wrap(err, "agent requires additional privilege if not run with --rootless")
+			return pkgerrors.WithMessage(err, "agent requires additional privilege if not run with --rootless")
 		}
 	}
 

pkg/cli/cert/cert.go

@@ -2,6 +2,7 @@ package cert
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -21,7 +22,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util/services"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/otiai10/copy"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -302,7 +303,7 @@ func rotateCA(app *cli.Context, cfg *cmds.Server, sync *cmds.CertRotateCA) error
 
 	url := fmt.Sprintf("/v1-%s/cert/cacerts?force=%t", version.Program, sync.Force)
 	if err = info.Put(url, buf.Bytes()); err != nil {
-		return errors.Wrap(err, "see server log for details")
+		return pkgerrors.WithMessage(err, "see server log for details")
 	}
 
 	fmt.Println("certificates saved to datastore")

pkg/cli/cmds/init_linux.go

@@ -6,7 +6,7 @@ import (
 	"os"
 
 	"github.com/containerd/containerd/pkg/userns"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rootless-containers/rootlesskit/pkg/parent/cgrouputil"
 )
 
@@ -17,7 +17,7 @@ func EvacuateCgroup2() error {
 		// The root cgroup has to be empty to enable subtree_control, so evacuate it by placing
 		// ourselves in the init cgroup.
 		if err := cgrouputil.EvacuateCgroup2("init"); err != nil {
-			return errors.Wrap(err, "failed to evacuate root cgroup")
+			return pkgerrors.WithMessage(err, "failed to evacuate root cgroup")
 		}
 	}
 	return nil

pkg/cli/cmds/log_linux.go

@@ -14,7 +14,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/natefinch/lumberjack"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -46,7 +46,7 @@ func forkIfLoggingOrReaping() error {
 
 		pwd, err := os.Getwd()
 		if err != nil {
-			return errors.Wrap(err, "failed to get working directory")
+			return pkgerrors.WithMessage(err, "failed to get working directory")
 		}
 
 		if enableReaping {

pkg/cli/cmds/server.go

@@ -92,6 +92,7 @@ type Server struct {
 	EtcdExposeMetrics        bool
 	EtcdSnapshotDir          string
 	EtcdSnapshotCron         string
+	EtcdSnapshotReconcile    time.Duration
 	EtcdSnapshotRetention    int
 	EtcdSnapshotCompress     bool
 	EtcdListFormat           string
@@ -390,6 +391,12 @@ var ServerFlags = []cli.Flag{
 		Destination: &ServerConfig.EtcdSnapshotCron,
 		Value:       "0 */12 * * *",
 	},
+	&cli.DurationFlag{
+		Name:        "etcd-snapshot-reconcile-interval",
+		Usage:       "(db) Snapshot reconcile interval",
+		Destination: &ServerConfig.EtcdSnapshotReconcile,
+		Value:       10 * time.Minute,
+	},
 	&cli.IntFlag{
 		Name:        "etcd-snapshot-retention",
 		Usage:       "(db) Number of snapshots to retain",

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -12,7 +13,7 @@ import (
 	"text/tabwriter"
 	"time"
 
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	k3s "github.com/k3s-io/api/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
@@ -21,7 +22,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
 	util2 "github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -92,7 +93,7 @@ func wrapServerError(err error) error {
 		// since the operation may have actualy succeeded despite the client timing out the request.
 		return err
 	}
-	return errors.Wrap(err, "see server log for details")
+	return pkgerrors.WithMessage(err, "see server log for details")
 }
 
 // Save triggers an on-demand etcd snapshot operation

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -17,7 +17,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/server"
 	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"k8s.io/utils/ptr"
 )
@@ -44,7 +44,7 @@ func commandPrep(cfg *cmds.Server) (*clientaccess.Info, error) {
 }
 
 func wrapServerError(err error) error {
-	return errors.Wrap(err, "see server log for details")
+	return pkgerrors.WithMessage(err, "see server log for details")
 }
 
 func Enable(app *cli.Context) error {

pkg/cli/server/server.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -29,7 +30,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util/permissions"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/k3s/pkg/vpn"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -75,7 +76,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 	if !cfg.DisableAgent && !cfg.Rootless {
 		if err := permissions.IsPrivileged(); err != nil {
-			return errors.Wrap(err, "server requires additional privilege when not run with --rootless and/or --disable-agent")
+			return pkgerrors.WithMessage(err, "server requires additional privilege when not run with --rootless and/or --disable-agent")
 		}
 	}
 
@@ -183,12 +184,19 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule
 
 	if !cfg.EtcdDisableSnapshots || cfg.ClusterReset {
+		if cfg.EtcdSnapshotReconcile <= 0 {
+			return errors.New("etcd-snapshot-reconcile-interval must be greater than 0s")
+		}
 		serverConfig.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress
 		serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
 		serverConfig.ControlConfig.EtcdSnapshotCron = cfg.EtcdSnapshotCron
 		serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
+		serverConfig.ControlConfig.EtcdSnapshotReconcile = metav1.Duration{Duration: cfg.EtcdSnapshotReconcile}
 		serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
 		if cfg.EtcdS3 {
+			if cfg.EtcdS3Timeout <= 0 {
+				return errors.New("etcd-s3-timeout must be greater than 0s")
+			}
 			serverConfig.ControlConfig.EtcdS3 = &config.EtcdS3{
 				AccessKey:     cfg.EtcdS3AccessKey,
 				Bucket:        cfg.EtcdS3BucketName,
@@ -327,7 +335,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ClusterCIDR) {
 		_, parsed, err := net.ParseCIDR(cidr)
 		if err != nil {
-			return errors.Wrapf(err, "invalid cluster-cidr %s", cidr)
+			return pkgerrors.WithMessagef(err, "invalid cluster-cidr %s", cidr)
 		}
 		serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed)
 	}
@@ -342,7 +350,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ServiceCIDR) {
 		_, parsed, err := net.ParseCIDR(cidr)
 		if err != nil {
-			return errors.Wrapf(err, "invalid service-cidr %s", cidr)
+			return pkgerrors.WithMessagef(err, "invalid service-cidr %s", cidr)
 		}
 		serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed)
 	}
@@ -352,7 +360,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 	serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange)
 	if err != nil {
-		return errors.Wrapf(err, "invalid port range %s", cfg.ServiceNodePortRange)
+		return pkgerrors.WithMessagef(err, "invalid port range %s", cfg.ServiceNodePortRange)
 	}
 
 	// the apiserver service does not yet support dual-stack operation
@@ -370,7 +378,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		for _, svcCIDR := range serverConfig.ControlConfig.ServiceIPRanges {
 			clusterDNS, err := utilsnet.GetIndexedIP(svcCIDR, 10)
 			if err != nil {
-				return errors.Wrap(err, "cannot configure default cluster-dns address")
+				return pkgerrors.WithMessage(err, "cannot configure default cluster-dns address")
 			}
 			serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, clusterDNS)
 		}
@@ -420,7 +428,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.MinTLSVersion = tlsMinVersionArg
 	serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg)
 	if err != nil {
-		return errors.Wrap(err, "invalid tls-min-version")
+		return pkgerrors.WithMessage(err, "invalid tls-min-version")
 	}
 
 	serverConfig.StartupHooks = append(serverConfig.StartupHooks, cfg.StartupHooks...)
@@ -450,7 +458,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.CipherSuites = tlsCipherSuites
 	serverConfig.ControlConfig.TLSCipherSuites, err = kubeapiserverflag.TLSCipherSuites(tlsCipherSuites)
 	if err != nil {
-		return errors.Wrap(err, "invalid tls-cipher-suites")
+		return pkgerrors.WithMessage(err, "invalid tls-cipher-suites")
 	}
 
 	// If performing a cluster reset, make sure control-plane components are

pkg/cli/token/token.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -19,7 +20,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"gopkg.in/yaml.v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -121,7 +122,7 @@ func delete(app *cli.Context, cfg *cmds.Token) error {
 		}
 		secretName := bootstraputil.BootstrapTokenSecretName(token)
 		if err := client.CoreV1().Secrets(metav1.NamespaceSystem).Delete(context.TODO(), secretName, metav1.DeleteOptions{}); err != nil {
-			return errors.Wrapf(err, "failed to delete bootstrap token %q", err)
+			return pkgerrors.WithMessagef(err, "failed to delete bootstrap token %q", err)
 		}
 
 		fmt.Printf("bootstrap token %q deleted\n", token)
@@ -218,7 +219,7 @@ func list(app *cli.Context, cfg *cmds.Token) error {
 
 	secrets, err := client.CoreV1().Secrets(metav1.NamespaceSystem).List(context.TODO(), listOptions)
 	if err != nil {
-		return errors.Wrapf(err, "failed to list bootstrap tokens")
+		return pkgerrors.WithMessagef(err, "failed to list bootstrap tokens")
 	}
 
 	tokens := make([]*kubeadm.BootstrapToken, len(secrets.Items))

pkg/clientaccess/kubeconfig.go

@@ -3,7 +3,7 @@ package clientaccess
 import (
 	"os"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
@@ -12,17 +12,17 @@ import (
 func WriteClientKubeConfig(destFile, url, serverCAFile, clientCertFile, clientKeyFile string) error {
 	serverCA, err := os.ReadFile(serverCAFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to read %s", serverCAFile)
+		return pkgerrors.WithMessagef(err, "failed to read %s", serverCAFile)
 	}
 
 	clientCert, err := os.ReadFile(clientCertFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to read %s", clientCertFile)
+		return pkgerrors.WithMessagef(err, "failed to read %s", clientCertFile)
 	}
 
 	clientKey, err := os.ReadFile(clientKeyFile)
 	if err != nil {
-		return errors.Wrapf(err, "failed to read %s", clientKeyFile)
+		return pkgerrors.WithMessagef(err, "failed to read %s", clientKeyFile)
 	}
 
 	config := clientcmdapi.NewConfig()

pkg/clientaccess/token.go

@@ -7,6 +7,7 @@ import (
 	"crypto/x509"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -16,7 +17,7 @@ import (
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/kubeadm"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -67,8 +68,29 @@ type Info struct {
 // ValidationOption is a callback to mutate the token prior to use
 type ValidationOption func(*Info)
 
+// WithCACertificate overrides the CA cert and hash with certs loaded from the
+// provided file. It is not an error if the file doesn't exist; the client
+// will just follow the normal hash validation steps if so.
+func WithCACertificate(certFile string) ValidationOption {
+	return func(i *Info) {
+		cacerts, err := os.ReadFile(certFile)
+		if err != nil {
+			return
+		}
+
+		digest, _ := hashCA(cacerts)
+		if i.caHash != "" && i.caHash != digest {
+			return
+		}
+
+		i.caHash = digest
+		i.CACerts = cacerts
+	}
+}
+
 // WithClientCertificate configures certs and keys to be used
-// to authenticate the request.
+// to authenticate the request. It is not an error if the files do not
+// exist, client cert auth will not be attempted if so.
 func WithClientCertificate(certFile, keyFile string) ValidationOption {
 	return func(i *Info) {
 		i.CertFile = certFile
@@ -338,11 +360,12 @@ func (i *Info) Post(path string, body []byte, options ...any) ([]byte, error) {
 }
 
 // setServer sets the BaseURL and CACerts fields of the Info by connecting to the server
-// and storing the CA bundle.
+// and storing the CA bundle. If CACerts has already been set via ValidationOption,
+// retrieval is skipped.
 func (i *Info) setServer(server string) error {
 	url, err := url.Parse(server)
 	if err != nil {
-		return errors.Wrapf(err, "Invalid server url, failed to parse: %s", server)
+		return pkgerrors.WithMessagef(err, "Invalid server url, failed to parse: %s", server)
 	}
 
 	if url.Scheme != "https" {
@@ -353,13 +376,15 @@ func (i *Info) setServer(server string) error {
 		url.Path = url.Path[:len(url.Path)-1]
 	}
 
-	cacerts, err := getCACerts(*url)
-	if err != nil {
-		return err
+	if len(i.CACerts) == 0 {
+		cacerts, err := getCACerts(*url)
+		if err != nil {
+			return err
+		}
+		i.CACerts = cacerts
 	}
 
 	i.BaseURL = url.String()
-	i.CACerts = cacerts
 	return nil
 }
 
@@ -400,7 +425,7 @@ func getCACerts(u url.URL) ([]byte, error) {
 	// Download the CA bundle using a client that does not validate certs.
 	cacerts, err := get(url, insecureClient, "", "", "")
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get CA certs")
+		return nil, pkgerrors.WithMessage(err, "failed to get CA certs")
 	}
 
 	// Request the CA bundle again, validating that the CA bundle can be loaded
@@ -408,7 +433,7 @@ func getCACerts(u url.URL) ([]byte, error) {
 	// get an empty CA bundle. or if the dynamiclistener cert is incorrectly signed.
 	_, err = get(url, GetHTTPClient(cacerts, "", ""), "", "", "")
 	if err != nil {
-		return nil, errors.Wrap(err, "CA cert validation failed")
+		return nil, pkgerrors.WithMessage(err, "CA cert validation failed")
 	}
 
 	return cacerts, nil

pkg/cloudprovider/instances.go

@@ -2,22 +2,22 @@ package cloudprovider
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	cloudprovider "k8s.io/cloud-provider"
 )
 
 var (
-	InternalIPKey = version.Program + ".io/internal-ip"
-	ExternalIPKey = version.Program + ".io/external-ip"
+	InternalIPKey  = version.Program + ".io/internal-ip"
+	ExternalIPKey  = version.Program + ".io/external-ip"
 	InternalDNSKey = version.Program + ".io/internal-dns"
 	ExternalDNSKey = version.Program + ".io/external-dns"
-	HostnameKey   = version.Program + ".io/hostname"
+	HostnameKey    = version.Program + ".io/hostname"
 )
 
 var _ cloudprovider.InstancesV2 = &k3s{}

pkg/cluster/bootstrap.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -24,7 +26,7 @@ import (
 	"github.com/k3s-io/kine/pkg/client"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/otiai10/copy"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -33,56 +35,65 @@ import (
 // ControlRuntimeBootstrap struct, either via HTTP or from the datastore.
 func (c *Cluster) Bootstrap(ctx context.Context, clusterReset bool) error {
 	if err := c.assignManagedDriver(ctx); err != nil {
-		return err
+		return pkgerrors.WithMessage(err, "failed to set datastore driver")
 	}
 
+	// Check if we need to bootstrap, and whether or not the managed database has already
+	// been initialized (created or joined an existing cluster). Note that nodes without
+	// a local datastore always need to bootstrap and never count as initialized.
+	// This also sets c.clientAccessInfo if c.config.JoinURL and c.config.Token are set.
 	shouldBootstrap, isInitialized, err := c.shouldBootstrapLoad(ctx)
 	if err != nil {
-		return err
+		return pkgerrors.WithMessage(err, "failed to check if bootstrap data has been initialized")
 	}
-	c.shouldBootstrap = shouldBootstrap
 
 	if c.managedDB != nil {
-		if !clusterReset {
-			isHTTP := c.config.JoinURL != "" && c.config.Token != ""
-			// For secondary servers, we attempt to connect and reconcile with the datastore.
-			// If that fails we fallback to the local etcd cluster start
-			if isInitialized && isHTTP && c.clientAccessInfo != nil {
-				if err := c.httpBootstrap(ctx); err == nil {
-					logrus.Info("Successfully reconciled with datastore")
+		if c.config.DisableETCD {
+			// secondary server with etcd disabled, start the etcd proxy so that we can attempt to use it
+			// when reconciling.
+			if err := c.startEtcdProxy(ctx); err != nil {
+				return pkgerrors.WithMessage(err, "failed to start etcd proxy")
+			}
+		} else if isInitialized && !clusterReset {
+			// For secondary servers with etcd, first attempt to connect and reconcile using the join URL.
+			// This saves on having to start up a temporary etcd just to extract bootstrap data.
+			if c.clientAccessInfo != nil {
+				if err := c.httpBootstrap(ctx); err != nil {
+					logrus.Warnf("Unable to reconcile with remote datastore: %v", err)
+				} else {
+					logrus.Info("Successfully reconciled with remote datastore")
 					return nil
 				}
-				logrus.Warnf("Unable to reconcile with datastore: %v", err)
 			}
-			// In the case of etcd, if the database has been initialized, it doesn't
-			// need to be bootstrapped however we still need to check the database
-			// and reconcile the bootstrap data. Below we're starting a temporary
-			// instance of etcd in the event that etcd certificates are unavailable,
-			// reading the data, and comparing that to the data on disk, all the while
-			// starting normal etcd.
-			if isInitialized {
-				if err := c.reconcileEtcd(ctx); err != nil {
-					logrus.Fatalf("Failed to reconcile with temporary etcd: %v", err)
-				}
+			// Not a secondary server or failed to reconcile via join URL, start up a temporary etcd
+			// with the local datastore and use that to reconcile.
+			if err := c.reconcileEtcd(ctx); err != nil {
+				logrus.Fatalf("Failed to reconcile with temporary etcd: %v", err)
 			}
 		}
 	}
 
-	if c.shouldBootstrap {
+	if shouldBootstrap {
 		return c.bootstrap(ctx)
 	}
 
 	return nil
 }
 
-// shouldBootstrapLoad returns true if we need to load ControlRuntimeBootstrap data again and a second boolean
-// indicating that the server has or has not been initialized, if etcd. This is controlled by a stamp file on
-// disk that records successful bootstrap using a hash of the join token.
+// shouldBootstrapLoad returns true if we need to load ControlRuntimeBootstrap data again and a
+// second boolean indicating that the server has or has not been initialized, if etcd. This is
+// controlled by a stamp file on disk that records successful bootstrap using a hash of the join
+// token. This function also sets up the HTTP Bootstrap request handler and sets
+// c.clientAccessInfo if join url and token are set.
 func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
+	opts := []clientaccess.ValidationOption{
+		clientaccess.WithUser("server"),
+		clientaccess.WithCACertificate(c.config.Runtime.ServerCA),
+	}
+
 	// Non-nil managedDB indicates that the database is either initialized, initializing, or joining
 	if c.managedDB != nil {
-		c.config.Runtime.HTTPBootstrap = true
-
+		c.config.Runtime.HTTPBootstrap = c.serveBootstrap()
 		isInitialized, err := c.managedDB.IsInitialized()
 		if err != nil {
 			return false, false, err
@@ -95,7 +106,7 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 			// etcd is promoted from learner. Odds are we won't need this info, and we don't want to fail startup
 			// due to failure to retrieve it as this will break cold cluster restart, so we ignore any errors.
 			if c.config.JoinURL != "" && c.config.Token != "" {
-				c.clientAccessInfo, _ = clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
+				c.clientAccessInfo, _ = clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, opts...)
 			}
 			return false, true, nil
 		} else if c.config.JoinURL == "" {
@@ -104,19 +115,24 @@ func (c *Cluster) shouldBootstrapLoad(ctx context.Context) (bool, bool, error) {
 			return false, false, nil
 		} else {
 			// Not initialized, but have a Join URL - fail if there's no token; if there is then validate it.
+			// Note that this is the path taken by control-plane-only nodes every startup, as they have a non-nil managedDB that is never initialized.
 			if c.config.Token == "" {
-				return false, false, errors.New(version.ProgramUpper + "_TOKEN is required to join a cluster")
+				return false, false, errors.New("token is required to join a cluster")
 			}
 
 			// Fail if the token isn't syntactically valid, or if the CA hash on the remote server doesn't match
 			// the hash in the token. The password isn't actually checked until later when actually bootstrapping.
-			info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, clientaccess.WithUser("server"))
+			info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, c.config.Token, opts...)
 			if err != nil {
-				return false, false, err
+				return false, false, pkgerrors.WithMessage(err, "failed to validate token")
 			}
-
-			logrus.Infof("Managed %s cluster not yet initialized", c.managedDB.EndpointName())
 			c.clientAccessInfo = info
+
+			if c.config.DisableETCD {
+				logrus.Infof("Managed %s disabled on this node", c.managedDB.EndpointName())
+			} else {
+				logrus.Infof("Managed %s cluster not yet initialized", c.managedDB.EndpointName())
+			}
 		}
 	}
 
@@ -318,7 +334,7 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 
 		updated, newer, err := isNewerFile(path, fileData)
 		if err != nil {
-			return errors.Wrapf(err, "failed to get update status of %s", pathKey)
+			return pkgerrors.WithMessagef(err, "failed to get update status of %s", pathKey)
 		}
 		if newer {
 			newerOnDisk = append(newerOnDisk, path)
@@ -335,10 +351,10 @@ func (c *Cluster) ReconcileBootstrapData(ctx context.Context, buf io.ReadSeeker,
 		logrus.Infof("Cluster reset: backing up certificates directory to " + tlsBackupDir)
 
 		if _, err := os.Stat(serverTLSDir); err != nil {
-			return errors.Wrap(err, "cluster reset failed to stat server TLS dir")
+			return pkgerrors.WithMessage(err, "cluster reset failed to stat server TLS dir")
 		}
 		if err := copy.Copy(serverTLSDir, tlsBackupDir); err != nil {
-			return errors.Wrap(err, "cluster reset failed to back up server TLS dir")
+			return pkgerrors.WithMessage(err, "cluster reset failed to back up server TLS dir")
 		}
 	} else if len(newerOnDisk) > 0 {
 		logrus.Fatal(strings.Join(newerOnDisk, ", ") + " newer than datastore and could cause a cluster outage. Remove the file(s) from disk and restart to be recreated from datastore.")
@@ -361,13 +377,13 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo
 			logrus.Warn(path + " doesn't exist. continuing...")
 			return true, false, nil
 		}
-		return false, false, errors.Wrapf(err, "reconcile failed to open")
+		return false, false, pkgerrors.WithMessagef(err, "reconcile failed to open")
 	}
 	defer f.Close()
 
 	data, err := io.ReadAll(f)
 	if err != nil {
-		return false, false, errors.Wrapf(err, "reconcile failed to read")
+		return false, false, pkgerrors.WithMessagef(err, "reconcile failed to read")
 	}
 
 	if bytes.Equal(file.Content, data) {
@@ -376,7 +392,7 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo
 
 	info, err := f.Stat()
 	if err != nil {
-		return false, false, errors.Wrapf(err, "reconcile failed to stat")
+		return false, false, pkgerrors.WithMessagef(err, "reconcile failed to stat")
 	}
 
 	if info.ModTime().Unix()-file.Timestamp.Unix() >= systemTimeSkew {
@@ -387,11 +403,30 @@ func isNewerFile(path string, file bootstrap.File) (updated bool, newerOnDisk bo
 	return true, false, nil
 }
 
+// serveBootstrap sends bootstrap data to the client, a server that is joining the cluster and
+// has only a server token, and cannot use CA certs/keys to access the datastore directly.
+func (c *Cluster) serveBootstrap() http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		rw.Header().Set("Content-Type", "application/json")
+		// Try getting data from the datastore first. Token has already been validated by the request handler.
+		_, token, _ := req.BasicAuth()
+		data, err := c.getBootstrapData(req.Context(), token)
+		if err != nil {
+			// If we failed to read data from the datastore, just send data from disk.
+			logrus.Warnf("Failed to retrieve HTTP bootstrap data from datastore; falling back to disk for %s: %v", req.RemoteAddr, err)
+			bootstrap.ReadFromDisk(rw, &c.config.Runtime.ControlRuntimeBootstrap)
+			return
+		}
+		logrus.Infof("Serving HTTP bootstrap from datastore for %s", req.RemoteAddr)
+		rw.Write(data)
+	})
+}
+
 // httpBootstrap retrieves bootstrap data (certs and keys, etc) from the remote server via HTTP
 // and loads it into the ControlRuntimeBootstrap struct. Unlike the storage bootstrap path,
 // this data does not need to be decrypted since it is generated on-demand by an existing server.
 func (c *Cluster) httpBootstrap(ctx context.Context) error {
-	content, err := c.clientAccessInfo.Get("/v1-" + version.Program + "/server-bootstrap")
+	content, err := c.clientAccessInfo.Get("/v1-"+version.Program+"/server-bootstrap", clientaccess.WithTimeout(15*time.Second))
 	if err != nil {
 		return err
 	}
@@ -399,7 +434,8 @@ func (c *Cluster) httpBootstrap(ctx context.Context) error {
 	return c.ReconcileBootstrapData(ctx, bytes.NewReader(content), &c.config.Runtime.ControlRuntimeBootstrap, true)
 }
 
-func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer, error) {
+// readBootstrapFromDisk returns a buffer holding the JSON-serialized bootstrap data read from disk.
+func (c *Cluster) readBootstrapFromDisk() (*bytes.Buffer, error) {
 	var buf bytes.Buffer
 	if err := bootstrap.ReadFromDisk(&buf, &c.config.Runtime.ControlRuntimeBootstrap); err != nil {
 		return nil, err
@@ -408,16 +444,28 @@ func (c *Cluster) retrieveInitializedDBdata(ctx context.Context) (*bytes.Buffer,
 	return &buf, nil
 }
 
-// bootstrap performs cluster bootstrapping, either via HTTP (for managed databases) or direct load from datastore.
+// bootstrap retrieves cluster bootstrap data: CA certs and other common config. This uses HTTP
+// for etcd (as this node does not yet have CA data available), and direct load from datastore
+// when using kine.
 func (c *Cluster) bootstrap(ctx context.Context) error {
 	c.joining = true
 
-	// bootstrap managed database via HTTPS
-	if c.config.Runtime.HTTPBootstrap {
-		// Assuming we should just compare on managed databases
+	if c.managedDB != nil {
+		// Try to compare local config against the server we're joining.
 		if err := c.compareConfig(); err != nil {
-			return errors.Wrap(err, "failed to validate server configuration")
+			return pkgerrors.WithMessage(err, "failed to validate server configuration")
+		}
+		// Try to bootstrap from the datastore using the local etcd proxy.
+		if data, err := c.getBootstrapData(ctx, c.clientAccessInfo.Password); err != nil {
+			logrus.Debugf("Failed to get bootstrap data from etcd proxy: %v", err)
+		} else {
+			if err := c.ReconcileBootstrapData(ctx, bytes.NewReader(data), &c.config.Runtime.ControlRuntimeBootstrap, false); err != nil {
+				logrus.Debugf("Failed to reconcile bootstrap data from etcd proxy: %v", err)
+			} else {
+				return nil
+			}
 		}
+		// fall back to bootstrapping from the join URL
 		return c.httpBootstrap(ctx)
 	}
 
@@ -427,17 +475,23 @@ func (c *Cluster) bootstrap(ctx context.Context) error {
 
 // compareConfig verifies that the config of the joining control plane node coincides with the cluster's config
 func (c *Cluster) compareConfig() error {
+	opts := []clientaccess.ValidationOption{
+		clientaccess.WithUser("node"),
+		clientaccess.WithCACertificate(c.config.Runtime.ServerCA),
+	}
+
 	token := c.config.AgentToken
 	if token == "" {
 		token = c.config.Token
 	}
-	agentClientAccessInfo, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token, clientaccess.WithUser("node"))
+	agentClientAccessInfo, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token, opts...)
 	if err != nil {
 		return err
 	}
 	serverConfig, err := agentClientAccessInfo.Get("/v1-" + version.Program + "/config")
 	if err != nil {
-		return err
+		logrus.Warnf("Skipping cluster configuration validation: %v", err)
+		return nil
 	}
 	clusterControl := &config.Control{}
 	if err := json.Unmarshal(serverConfig, clusterControl); err != nil {
@@ -517,7 +571,7 @@ func (c *Cluster) reconcileEtcd(ctx context.Context) error {
 		}
 	}
 
-	data, err := c.retrieveInitializedDBdata(reconcileCtx)
+	data, err := c.readBootstrapFromDisk()
 	if err != nil {
 		return err
 	}

pkg/cluster/cluster.go

@@ -3,7 +3,6 @@ package cluster
 import (
 	"context"
 	"net/url"
-	"runtime"
 	"strings"
 	"time"
 
@@ -12,7 +11,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/kine/pkg/endpoint"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/wait"
 	utilsnet "k8s.io/utils/net"
@@ -25,7 +24,6 @@ type Cluster struct {
 	joining          bool
 	storageStarted   bool
 	saveBootstrap    bool
-	shouldBootstrap  bool
 	cnFilterFunc     func(...string) []string
 }
 
@@ -36,104 +34,70 @@ type Cluster struct {
 func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 	// Set up the dynamiclistener and http request handlers
 	if err := c.initClusterAndHTTPS(ctx); err != nil {
-		return nil, errors.Wrap(err, "init cluster datastore and https")
+		return nil, pkgerrors.WithMessage(err, "init cluster datastore and https")
 	}
 
 	if c.config.DisableETCD {
 		ready := make(chan struct{})
 		defer close(ready)
-
-		// try to get /db/info urls first, for a current list of etcd cluster member client URLs
-		clientURLs, _, err := etcd.ClientURLs(ctx, c.clientAccessInfo, c.config.PrivateIP)
-		if err != nil {
-			return nil, err
-		}
-		// If we somehow got no error but also no client URLs, just use the address of the server we're joining
-		if len(clientURLs) == 0 {
-			clientURL, err := url.Parse(c.config.JoinURL)
-			if err != nil {
-				return nil, err
-			}
-			clientURL.Host = clientURL.Hostname() + ":2379"
-			clientURLs = append(clientURLs, clientURL.String())
-			logrus.Warnf("Got empty etcd ClientURL list; using server URL %s", clientURL)
-		}
-		etcdProxy, err := etcd.NewETCDProxy(ctx, c.config.SupervisorPort, c.config.DataDir, clientURLs[0], utilsnet.IsIPv6CIDR(c.config.ServiceIPRanges[0]))
-		if err != nil {
-			return nil, err
-		}
-		// immediately update the load balancer with all etcd addresses
-		// client URLs are a full URI, but the proxy only wants host:port
-		for i, c := range clientURLs {
-			u, err := url.Parse(c)
-			if err != nil {
-				return nil, errors.Wrap(err, "failed to parse etcd ClientURL")
-			}
-			clientURLs[i] = u.Host
-		}
-		etcdProxy.Update(clientURLs)
-
-		// start periodic endpoint sync goroutine
-		c.setupEtcdProxy(ctx, etcdProxy)
-
-		// remove etcd member if it exists
-		if err := c.managedDB.RemoveSelf(ctx); err != nil {
-			logrus.Warnf("Failed to remove this node from etcd members")
-		}
-
-		c.config.Runtime.EtcdConfig.Endpoints = strings.Split(c.config.Datastore.Endpoint, ",")
-		c.config.Runtime.EtcdConfig.TLSConfig = c.config.Datastore.BackendTLSConfig
-
 		return ready, nil
 	}
 
-	// start managed database (if necessary)
+	// start managed etcd database; when kine is in use this is a no-op.
 	if err := c.start(ctx); err != nil {
-		return nil, errors.Wrap(err, "start managed database")
+		return nil, pkgerrors.WithMessage(err, "start managed database")
 	}
 
-	// get the wait channel for testing managed database readiness
-	ready, err := c.testClusterDB(ctx)
-	if err != nil {
-		return nil, err
-	}
+	// get the wait channel for testing etcd server readiness; when kine is in
+	// use the channel is closed immediately.
+	ready := c.testClusterDB(ctx)
 
+	// set c.config.Datastore and c.config.Runtime.EtcdConfig with values
+	// necessary to build etcd clients, and start kine listener if necessary.
 	if err := c.startStorage(ctx, false); err != nil {
 		return nil, err
 	}
 
-	// if necessary, store bootstrap data to datastore
+	// if necessary, store bootstrap data to datastore. saveBootstrap is only set
+	// when using kine, so this can be done before the ready channel has been closed.
 	if c.saveBootstrap {
 		if err := Save(ctx, c.config, false); err != nil {
 			return nil, err
 		}
 	}
 
-	// at this point, if etcd is in use, it's bootstrapping is complete
-	// so save the bootstrap data. We will need for etcd to be up. If
-	// the save call returns an error, we panic since subsequent etcd
-	// snapshots will be empty.
 	if c.managedDB != nil {
 		go func() {
 			for {
 				select {
 				case <-ready:
+					// always save to managed etcd, to ensure that any file modified locally are in sync with the datastore.
+					// this will panic if multiple keys exist, to prevent nodes from running with different bootstrap data.
 					if err := Save(ctx, c.config, false); err != nil {
 						panic(err)
 					}
 
 					if !c.config.EtcdDisableSnapshots {
-						_ = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
-							err := c.managedDB.ReconcileSnapshotData(ctx)
-							if err != nil {
+						// do an initial reconcile of snapshots with a fast retry until it succeeds
+						wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
+							if err := c.managedDB.ReconcileSnapshotData(ctx); err != nil {
 								logrus.Errorf("Failed to record snapshots for cluster: %v", err)
+								return false, nil
 							}
-							return err == nil, nil
+							return true, nil
 						})
+
+						// continue reconciling snapshots in the background at the configured interval.
+						// the interval is jittered by 5% to avoid all nodes reconciling at the same time.
+						wait.JitterUntilWithContext(ctx, func(ctx context.Context) {
+							if err := c.managedDB.ReconcileSnapshotData(ctx); err != nil {
+								logrus.Errorf("Failed to record snapshots for cluster: %v", err)
+							}
+						}, c.config.EtcdSnapshotReconcile.Duration, 0.05, false)
 					}
 					return
-				default:
-					runtime.Gosched()
+				case <-ctx.Done():
+					return
 				}
 			}
 		}()
@@ -142,6 +106,49 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 	return ready, nil
 }
 
+// startEtcdProxy starts an etcd load-balancer proxy, for control-plane-only nodes
+// without a local datastore.
+func (c *Cluster) startEtcdProxy(ctx context.Context) error {
+	defaultURL, err := url.Parse(c.config.JoinURL)
+	if err != nil {
+		return err
+	}
+	defaultURL.Host = defaultURL.Hostname() + ":2379"
+	etcdProxy, err := etcd.NewETCDProxy(ctx, c.config.SupervisorPort, c.config.DataDir, defaultURL.String(), utilsnet.IsIPv6CIDR(c.config.ServiceIPRanges[0]))
+	if err != nil {
+		return err
+	}
+
+	// immediately update the load balancer with all etcd addresses
+	// from /db/info, for a current list of etcd cluster member client URLs.
+	// client URLs are a full URI, but the proxy only wants host:port
+	if clientURLs, _, err := etcd.ClientURLs(ctx, c.clientAccessInfo, c.config.PrivateIP); err != nil || len(clientURLs) == 0 {
+		logrus.Warnf("Failed to get etcd ClientURLs: %v", err)
+	} else {
+		for i, c := range clientURLs {
+			u, err := url.Parse(c)
+			if err != nil {
+				return pkgerrors.WithMessage(err, "failed to parse etcd ClientURL")
+			}
+			clientURLs[i] = u.Host
+		}
+		etcdProxy.Update(clientURLs)
+	}
+
+	// start periodic endpoint sync goroutine
+	c.setupEtcdProxy(ctx, etcdProxy)
+
+	// remove etcd member if it exists
+	if err := c.managedDB.RemoveSelf(ctx); err != nil {
+		logrus.Warnf("Failed to remove this node from etcd members: %v", err)
+	}
+
+	c.config.Runtime.EtcdConfig.Endpoints = strings.Split(c.config.Datastore.Endpoint, ",")
+	c.config.Runtime.EtcdConfig.TLSConfig = c.config.Datastore.BackendTLSConfig
+
+	return nil
+}
+
 // startStorage starts the kine listener and configures the endpoints, if necessary.
 // This calls into the kine endpoint code, which sets up the database client
 // and unix domain socket listener if using an external database. In the case of an etcd
@@ -162,7 +169,7 @@ func (c *Cluster) startStorage(ctx context.Context, bootstrap bool) error {
 	// start listening on the kine socket as an etcd endpoint, or return the external etcd endpoints
 	etcdConfig, err := endpoint.Listen(ctx, c.config.Datastore)
 	if err != nil {
-		return errors.Wrap(err, "creating storage endpoint")
+		return pkgerrors.WithMessage(err, "creating storage endpoint")
 	}
 
 	// Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers

pkg/cluster/managed.go

@@ -5,14 +5,13 @@ package cluster
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
 	"os"
 	"time"
 
-	"github.com/pkg/errors"
-
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/etcd"
@@ -26,11 +25,11 @@ import (
 
 // testClusterDB returns a channel that will be closed when the datastore connection is available.
 // The datastore is tested for readiness every 5 seconds until the test succeeds.
-func (c *Cluster) testClusterDB(ctx context.Context) (<-chan struct{}, error) {
+func (c *Cluster) testClusterDB(ctx context.Context) <-chan struct{} {
 	result := make(chan struct{})
 	if c.managedDB == nil {
 		close(result)
-		return result, nil
+		return result
 	}
 
 	go func() {
@@ -51,7 +50,7 @@ func (c *Cluster) testClusterDB(ctx context.Context) (<-chan struct{}, error) {
 		}
 	}()
 
-	return result, nil
+	return result
 }
 
 // start starts the database, unless a cluster reset has been requested, in which case

pkg/cluster/storage.go

@@ -198,8 +198,12 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 
 	attempts := 0
 	tokenKey := storageKey(normalizedToken)
-	return wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) {
+	return wait.PollUntilContextCancel(ctx, 5*time.Second, true, func(ctx context.Context) (bool, error) {
 		attempts++
+
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+
 		value, saveBootstrap, err := getBootstrapKeyFromStorage(ctx, storageClient, normalizedToken, token)
 		c.saveBootstrap = saveBootstrap
 		if err != nil {
@@ -246,6 +250,25 @@ func (c *Cluster) storageBootstrap(ctx context.Context) error {
 	})
 }
 
+// getBootstrapData makes a single attempt to retrieve and decrypt bootstrap data from the datastore.
+func (c *Cluster) getBootstrapData(ctx context.Context, token string) ([]byte, error) {
+	storageClient, err := client.New(c.config.Runtime.EtcdConfig)
+	if err != nil {
+		return nil, err
+	}
+	defer storageClient.Close()
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	value, err := storageClient.Get(ctx, storageKey(token))
+	if err != nil {
+		return nil, err
+	}
+
+	return decrypt(token, value.Data)
+}
+
 // getBootstrapKeyFromStorage will list all keys that has prefix /bootstrap and will check for key that is
 // hashed with empty string and will check for any key that is hashed by different token than the one
 // passed to it, it will return error if it finds a key that is hashed with different token and will return

pkg/codegen/cleanup/main.go

-package main
-
-import (
-	"github.com/rancher/wrangler/pkg/cleanup"
-	"github.com/sirupsen/logrus"
-)
-
-func main() {
-	if err := cleanup.Cleanup("./pkg/apis"); err != nil {
-		logrus.Fatal(err)
-	}
-}

pkg/codegen/main.go

@@ -4,16 +4,9 @@ import (
 	"os"
 
 	bindata "github.com/go-bindata/go-bindata"
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	controllergen "github.com/rancher/wrangler/pkg/controller-gen"
-	"github.com/rancher/wrangler/pkg/controller-gen/args"
 	"github.com/sirupsen/logrus"
 )
 
-var (
-	basePackage = "github.com/k3s-io/k3s/types"
-)
-
 func main() {
 	os.Unsetenv("GOPATH")
 	bc := &bindata.Config{
@@ -66,19 +59,4 @@ func main() {
 	if err := bindata.Translate(bc); err != nil {
 		logrus.Fatal(err)
 	}
-
-	controllergen.Run(args.Options{
-		OutputPackage: "github.com/k3s-io/k3s/pkg/generated",
-		Boilerplate:   "scripts/boilerplate.go.txt",
-		Groups: map[string]args.Group{
-			"k3s.cattle.io": {
-				Types: []interface{}{
-					v1.Addon{},
-					v1.ETCDSnapshotFile{},
-				},
-				GenerateTypes:   true,
-				GenerateClients: true,
-			},
-		},
-	})
 }

pkg/containerd/utility_windows.go

@@ -5,17 +5,17 @@ package containerd
 
 import (
 	util2 "github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 )
 
 func OverlaySupported(root string) error {
-	return errors.Wrapf(util2.ErrUnsupportedPlatform, "overlayfs is not supported")
+	return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "overlayfs is not supported")
 }
 
 func FuseoverlayfsSupported(root string) error {
-	return errors.Wrapf(util2.ErrUnsupportedPlatform, "fuse-overlayfs is not supported")
+	return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "fuse-overlayfs is not supported")
 }
 
 func StargzSupported(root string) error {
-	return errors.Wrapf(util2.ErrUnsupportedPlatform, "stargz is not supported")
+	return pkgerrors.WithMessagef(util2.ErrUnsupportedPlatform, "stargz is not supported")
 }

pkg/crd/crds.go

 package crd
 
 import (
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	v1 "github.com/k3s-io/api/k3s.cattle.io/v1"
 	"github.com/rancher/wrangler/pkg/crd"
 )
 

pkg/daemons/agent/agent.go

@@ -10,6 +10,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/proxy"
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/component-base/logs"
 	logsapi "k8s.io/component-base/logs/api/v1"
@@ -24,7 +25,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy
 	defer logs.FlushLogs()
 
 	if err := startKubelet(ctx, &nodeConfig.AgentConfig); err != nil {
-		return err
+		return pkgerrors.WithMessage(err, "failed to start kubelet")
 	}
 
 	go func() {

pkg/daemons/config/types.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io"
+	"github.com/k3s-io/api/pkg/generated/controllers/k3s.cattle.io"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/wharfie/pkg/registries"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
@@ -233,17 +233,18 @@ type Control struct {
 	ClusterResetRestorePath  string
 	MinTLSVersion            string
 	CipherSuites             []string
-	TLSMinVersion            uint16   `json:"-"`
-	TLSCipherSuites          []uint16 `json:"-"`
-	EtcdSnapshotName         string   `json:"-"`
-	EtcdDisableSnapshots     bool     `json:"-"`
-	EtcdExposeMetrics        bool     `json:"-"`
-	EtcdSnapshotDir          string   `json:"-"`
-	EtcdSnapshotCron         string   `json:"-"`
-	EtcdSnapshotRetention    int      `json:"-"`
-	EtcdSnapshotCompress     bool     `json:"-"`
-	EtcdListFormat           string   `json:"-"`
-	EtcdS3                   *EtcdS3  `json:"-"`
+	TLSMinVersion            uint16          `json:"-"`
+	TLSCipherSuites          []uint16        `json:"-"`
+	EtcdSnapshotName         string          `json:"-"`
+	EtcdDisableSnapshots     bool            `json:"-"`
+	EtcdExposeMetrics        bool            `json:"-"`
+	EtcdSnapshotDir          string          `json:"-"`
+	EtcdSnapshotCron         string          `json:"-"`
+	EtcdSnapshotReconcile    metav1.Duration `json:"-"`
+	EtcdSnapshotRetention    int             `json:"-"`
+	EtcdSnapshotCompress     bool            `json:"-"`
+	EtcdListFormat           string          `json:"-"`
+	EtcdS3                   *EtcdS3         `json:"-"`
 	ServerNodeName           string
 	VLevel                   int
 	VModule                  string
@@ -310,7 +311,6 @@ type ControlRuntimeBootstrap struct {
 type ControlRuntime struct {
 	ControlRuntimeBootstrap
 
-	HTTPBootstrap                        bool
 	APIServerReady                       <-chan struct{}
 	ContainerRuntimeReady                <-chan struct{}
 	ETCDReady                            <-chan struct{}
@@ -340,6 +340,7 @@ type ControlRuntime struct {
 	AgentToken         string
 	APIServer          http.Handler
 	Handler            http.Handler
+	HTTPBootstrap      http.Handler
 	Tunnel             http.Handler
 	Authenticator      authenticator.Request
 

pkg/daemons/control/proxy/proxy.go

 package proxy
 
 import (
+	"errors"
 	"io"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -43,12 +44,12 @@ func (p *proxy) pipe(src, dst io.ReadWriter) {
 	for {
 		n, err := src.Read(buff)
 		if err != nil {
-			p.err(errors.Wrap(err, "read failed"))
+			p.err(pkgerrors.WithMessage(err, "read failed"))
 			return
 		}
 		_, err = dst.Write(buff[:n])
 		if err != nil {
-			p.err(errors.Wrap(err, "write failed"))
+			p.err(pkgerrors.WithMessage(err, "write failed"))
 			return
 		}
 	}

pkg/daemons/control/server.go

@@ -2,6 +2,7 @@ package control
 
 import (
 	"context"
+	"errors"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -16,7 +17,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	authorizationv1 "k8s.io/api/authorization/v1"
 	v1 "k8s.io/api/core/v1"
@@ -40,12 +41,12 @@ func Server(ctx context.Context, cfg *config.Control) error {
 
 	logsapi.ReapplyHandling = logsapi.ReapplyHandlingIgnoreUnchanged
 	if err := prepare(ctx, cfg); err != nil {
-		return errors.Wrap(err, "preparing server")
+		return pkgerrors.WithMessage(err, "preparing server")
 	}
 
 	tunnel, err := setupTunnel(ctx, cfg)
 	if err != nil {
-		return errors.Wrap(err, "setup tunnel server")
+		return pkgerrors.WithMessage(err, "setup tunnel server")
 	}
 	cfg.Runtime.Tunnel = tunnel
 
@@ -287,17 +288,16 @@ func defaults(config *config.Control) {
 }
 
 func prepare(ctx context.Context, config *config.Control) error {
-	var err error
-
 	defaults(config)
 
 	if err := os.MkdirAll(config.DataDir, 0700); err != nil {
 		return err
 	}
 
-	config.DataDir, err = filepath.Abs(config.DataDir)
-	if err != nil {
+	if dataDir, err := filepath.Abs(config.DataDir); err != nil {
 		return err
+	} else {
+		config.DataDir = dataDir
 	}
 
 	os.MkdirAll(filepath.Join(config.DataDir, "etc"), 0700)
@@ -308,19 +308,19 @@ func prepare(ctx context.Context, config *config.Control) error {
 
 	cluster := cluster.New(config)
 	if err := cluster.Bootstrap(ctx, config.ClusterReset); err != nil {
-		return err
+		return pkgerrors.WithMessage(err, "failed to bootstrap cluster data")
 	}
 
 	if err := deps.GenServerDeps(config); err != nil {
-		return err
+		return pkgerrors.WithMessage(err, "failed to generate server dependencies")
 	}
 
-	ready, err := cluster.Start(ctx)
-	if err != nil {
-		return err
+	if ready, err := cluster.Start(ctx); err != nil {
+		return pkgerrors.WithMessage(err, "failed to start cluster")
+	} else {
+		config.Runtime.ETCDReady = ready
 	}
 
-	config.Runtime.ETCDReady = ready
 	return nil
 }
 
@@ -515,7 +515,7 @@ func waitForUntaintedNode(ctx context.Context, kubeConfig string) error {
 	}
 
 	if _, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition); err != nil {
-		return errors.Wrap(err, "failed to wait for untainted node")
+		return pkgerrors.WithMessage(err, "failed to wait for untainted node")
 	}
 	return nil
 }

pkg/daemons/control/tunnel.go

@@ -3,6 +3,7 @@ package control
 import (
 	"bufio"
 	"context"
+	"errors"
 	"io"
 	"net"
 	"net/http"
@@ -15,7 +16,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/nodeconfig"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
 	"github.com/rancher/remotedialer"
 	"github.com/sirupsen/logrus"
 	"github.com/yl2chen/cidranger"

pkg/daemons/executor/embed.go

@@ -5,6 +5,7 @@ package executor
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"net/http"
 	"os"
@@ -18,7 +19,6 @@ import (
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	cloudprovider "k8s.io/cloud-provider"

pkg/datadir/datadir.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/util/permissions"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/resolvehome"
 )
 
@@ -31,7 +31,7 @@ func LocalHome(dataDir string, forceLocal bool) (string, error) {
 
 	dataDir, err := resolvehome.Resolve(dataDir)
 	if err != nil {
-		return "", errors.Wrapf(err, "resolving %s", dataDir)
+		return "", pkgerrors.WithMessagef(err, "resolving %s", dataDir)
 	}
 
 	return filepath.Abs(dataDir)

pkg/deploy/controller.go

@@ -15,11 +15,11 @@ import (
 	"sync"
 	"time"
 
+	apisv1 "github.com/k3s-io/api/k3s.cattle.io/v1"
+	controllersv1 "github.com/k3s-io/api/pkg/generated/controllers/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/agent/util"
-	apisv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	controllersv1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
 	pkgutil "github.com/k3s-io/k3s/pkg/util"
-	errors2 "github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/apply"
 	"github.com/rancher/wrangler/pkg/kv"
 	"github.com/rancher/wrangler/pkg/merr"
@@ -165,7 +165,7 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 		// Disabled files are not just skipped, but actively deleted from the filesystem
 		if shouldDisableFile(base, path, w.disables) {
 			if err := w.delete(path); err != nil {
-				errs = append(errs, errors2.Wrapf(err, "failed to delete %s", path))
+				errs = append(errs, pkgerrors.WithMessagef(err, "failed to delete %s", path))
 			}
 			continue
 		}
@@ -178,7 +178,7 @@ func (w *watcher) listFilesIn(base string, force bool) error {
 			continue
 		}
 		if err := w.deploy(path, !force); err != nil {
-			errs = append(errs, errors2.Wrapf(err, "failed to process %s", path))
+			errs = append(errs, pkgerrors.WithMessagef(err, "failed to process %s", path))
 		} else {
 			w.modTime[path] = modTime
 		}

pkg/deploy/stage.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -38,7 +38,7 @@ staging:
 		os.MkdirAll(filepath.Dir(p), 0700)
 		logrus.Info("Writing manifest: ", p)
 		if err := os.WriteFile(p, content, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write to %s", name)
+			return pkgerrors.WithMessagef(err, "failed to write to %s", name)
 		}
 	}
 

pkg/etcd/etcd.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/fs"
 	"net"
@@ -33,7 +34,7 @@ import (
 	"github.com/k3s-io/kine/pkg/client"
 	endpoint2 "github.com/k3s-io/kine/pkg/endpoint"
 	cp "github.com/otiai10/copy"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/start"
@@ -215,7 +216,7 @@ func (e *ETCD) Test(ctx context.Context) error {
 
 	status, err := e.status(ctx)
 	if err != nil {
-		return errors.Wrap(err, "failed to get etcd status")
+		return pkgerrors.WithMessage(err, "failed to get etcd status")
 	} else if status.IsLearner {
 		return errors.New("this server has not yet been promoted from learner to voting member")
 	} else if status.Leader == 0 {
@@ -229,19 +230,19 @@ func (e *ETCD) Test(ctx context.Context) error {
 
 	// defrag this node to reclaim freed space from compacted revisions
 	if err := e.defragment(ctx); err != nil {
-		return errors.Wrap(err, "failed to defragment etcd database")
+		return pkgerrors.WithMessage(err, "failed to defragment etcd database")
 	}
 
 	// clear alarms on this node
 	if err := e.clearAlarms(ctx, status.Header.MemberId); err != nil {
-		return errors.Wrap(err, "failed to disarm etcd alarms")
+		return pkgerrors.WithMessage(err, "failed to disarm etcd alarms")
 	}
 
 	// refresh status - note that errors may remain on other nodes, but this
 	// should not prevent us from continuing with startup.
 	status, err = e.status(ctx)
 	if err != nil {
-		return errors.Wrap(err, "failed to get etcd status")
+		return pkgerrors.WithMessage(err, "failed to get etcd status")
 	}
 
 	logrus.Infof("Datastore using %d of %d bytes after defragment", status.DbSizeInUse, status.DbSize)
@@ -334,7 +335,7 @@ func (e *ETCD) IsInitialized() (bool, error) {
 	} else if os.IsNotExist(err) {
 		return false, nil
 	} else {
-		return false, errors.Wrap(err, "invalid state for wal directory "+dir)
+		return false, pkgerrors.WithMessage(err, "invalid state for wal directory "+dir)
 	}
 }
 
@@ -410,16 +411,16 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 				if errors.Is(err, s3.ErrNoConfigSecret) {
 					return errors.New("cannot use S3 config secret when restoring snapshot; configuration must be set in CLI or config file")
 				} else {
-					return errors.Wrap(err, "failed to initialize S3 client")
+					return pkgerrors.WithMessage(err, "failed to initialize S3 client")
 				}
 			}
 			dir, err := snapshotDir(e.config, true)
 			if err != nil {
-				return errors.Wrap(err, "failed to get the snapshot dir")
+				return pkgerrors.WithMessage(err, "failed to get the snapshot dir")
 			}
 			path, err := s3client.Download(ctx, e.config.ClusterResetRestorePath, dir)
 			if err != nil {
-				return errors.Wrap(err, "failed to download snapshot from S3")
+				return pkgerrors.WithMessage(err, "failed to download snapshot from S3")
 			}
 			e.config.ClusterResetRestorePath = path
 			logrus.Infof("S3 download complete for %s", e.config.ClusterResetRestorePath)
@@ -452,7 +453,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) error {
 	isInitialized, err := e.IsInitialized()
 	if err != nil {
-		return errors.Wrapf(err, "failed to check for initialized etcd datastore")
+		return pkgerrors.WithMessagef(err, "failed to check for initialized etcd datastore")
 	}
 
 	if err := e.startClient(ctx); err != nil {
@@ -649,6 +650,13 @@ func (e *ETCD) Register(handler http.Handler) (http.Handler, error) {
 	// is being removed from the cluster.
 	if !e.config.DisableAPIServer {
 		e.config.Runtime.LeaderElectedClusterControllerStarts[version.Program+"-etcd"] = func(ctx context.Context) {
+			// ensure client is started, as etcd startup may not have handled this if this is a control-plane-only node
+			if e.client == nil {
+				if err := e.startClient(ctx); err != nil {
+					panic(pkgerrors.WithMessage(err, "failed to start etcd client"))
+				}
+			}
+
 			registerEndpointsHandlers(ctx, e)
 			registerMemberHandlers(ctx, e)
 			registerSnapshotHandlers(ctx, e)
@@ -656,7 +664,7 @@ func (e *ETCD) Register(handler http.Handler) (http.Handler, error) {
 			// Re-run informer factory startup after core and leader-elected controllers have started.
 			// Additional caches may need to start for the newly added OnChange/OnRemove callbacks.
 			if err := start.All(ctx, 5, e.config.Runtime.K3s, e.config.Runtime.Core); err != nil {
-				panic(errors.Wrap(err, "failed to start wrangler controllers"))
+				panic(pkgerrors.WithMessage(err, "failed to start wrangler controllers"))
 			}
 		}
 	}
@@ -731,7 +739,7 @@ func (e *ETCD) infoHandler() http.Handler {
 
 		members, err := e.client.MemberList(ctx)
 		if err != nil {
-			util.SendError(errors.Wrap(err, "failed to get etcd MemberList"), rw, req, http.StatusInternalServerError)
+			util.SendError(pkgerrors.WithMessage(err, "failed to get etcd MemberList"), rw, req, http.StatusInternalServerError)
 			return
 		}
 
@@ -1314,7 +1322,7 @@ func (e *ETCD) trackLearnerProgress(ctx context.Context, progress *learnerProgre
 func (e *ETCD) getETCDStatus(ctx context.Context, url string) (*clientv3.StatusResponse, error) {
 	resp, err := e.client.Status(ctx, url)
 	if err != nil {
-		return resp, errors.Wrap(err, "failed to check etcd member status")
+		return resp, pkgerrors.WithMessage(err, "failed to check etcd member status")
 	}
 	if len(resp.Errors) != 0 {
 		return resp, errors.New("etcd member has status errors: " + strings.Join(resp.Errors, ","))
@@ -1547,7 +1555,7 @@ func (e *ETCD) Restore(ctx context.Context) error {
 	if strings.HasSuffix(e.config.ClusterResetRestorePath, snapshot.CompressedExtension) {
 		dir, err := snapshotDir(e.config, true)
 		if err != nil {
-			return errors.Wrap(err, "failed to get the snapshot dir")
+			return pkgerrors.WithMessage(err, "failed to get the snapshot dir")
 		}
 
 		decompressSnapshot, err := e.decompressSnapshot(dir, e.config.ClusterResetRestorePath)
@@ -1648,6 +1656,12 @@ func GetAPIServerURLsFromETCD(ctx context.Context, cfg *config.Control) ([]strin
 // GetMembersClientURLs will list through the member lists in etcd and return
 // back a combined list of client urls for each member in the cluster
 func (e *ETCD) GetMembersClientURLs(ctx context.Context) ([]string, error) {
+	if e.client == nil {
+		if err := e.startClient(ctx); err != nil {
+			return nil, err
+		}
+	}
+
 	members, err := e.client.MemberList(ctx)
 	if err != nil {
 		return nil, err

pkg/etcd/etcd_linux_test.go

@@ -31,6 +31,7 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/grpc/reflection"
 	"google.golang.org/grpc/status"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
@@ -67,6 +68,7 @@ func generateTestConfig() *config.Control {
 		DataDir:               "/tmp/k3s/", // Different than the default value
 		EtcdSnapshotName:      "etcd-snapshot",
 		EtcdSnapshotCron:      "0 */12 * * *",
+		EtcdSnapshotReconcile: metav1.Duration{Duration: 10 * time.Minute},
 		EtcdSnapshotRetention: 5,
 		EtcdS3: &config.EtcdS3{
 			Endpoint: "s3.amazonaws.com",

pkg/etcd/member_controller.go

@@ -2,12 +2,12 @@ package etcd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"

pkg/etcd/s3/s3.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/base64"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -26,7 +27,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -99,13 +100,13 @@ func Start(ctx context.Context, config *config.Control) (*Controller, error) {
 				// cluster id hack: see https://groups.google.com/forum/#!msg/kubernetes-sig-architecture/mVGobfD4TpY/nkdbkX1iBwAJ
 				ns, err := c.core.V1().Namespace().Get(metav1.NamespaceSystem, metav1.GetOptions{})
 				if err != nil {
-					return false, errors.Wrap(err, "failed to set S3 snapshot cluster ID")
+					return false, pkgerrors.WithMessage(err, "failed to set S3 snapshot cluster ID")
 				}
 				c.clusterID = string(ns.UID)
 
 				tokenHash, err := util.GetTokenHash(config)
 				if err != nil {
-					return false, errors.Wrap(err, "failed to set S3 snapshot server token hash")
+					return false, pkgerrors.WithMessage(err, "failed to set S3 snapshot server token hash")
 				}
 				c.tokenHash = tokenHash
 
@@ -137,7 +138,7 @@ func (c *Controller) GetClient(ctx context.Context, etcdS3 *config.EtcdS3) (*Cli
 		if isDefault {
 			e, err := c.getConfigFromSecret(etcdS3.ConfigSecret)
 			if err != nil {
-				return nil, errors.Wrapf(err, "failed to get config from etcd-s3-config-secret %q", etcdS3.ConfigSecret)
+				return nil, pkgerrors.WithMessagef(err, "failed to get config from etcd-s3-config-secret %q", etcdS3.ConfigSecret)
 			}
 			logrus.Infof("Using etcd s3 configuration from etcd-s3-config-secret %q", etcdS3.ConfigSecret)
 			etcdS3 = e
@@ -196,7 +197,7 @@ func (c *Controller) GetClient(ctx context.Context, etcdS3 *config.EtcdS3) (*Cli
 		if etcdS3.Proxy != "none" {
 			u, err = url.Parse(etcdS3.Proxy)
 			if err != nil {
-				return nil, errors.Wrap(err, "failed to parse etcd-s3-proxy value as URL")
+				return nil, pkgerrors.WithMessage(err, "failed to parse etcd-s3-proxy value as URL")
 			}
 			if u.Scheme == "" || u.Host == "" {
 				return nil, fmt.Errorf("proxy URL must include scheme and host")
@@ -219,7 +220,7 @@ func (c *Controller) GetClient(ctx context.Context, etcdS3 *config.EtcdS3) (*Cli
 	})
 
 	if _, err := creds.Get(); err != nil {
-		return nil, errors.Wrap(err, "failed to get credentials")
+		return nil, pkgerrors.WithMessage(err, "failed to get credentials")
 	}
 
 	opt := minio.Options{
@@ -241,7 +242,7 @@ func (c *Controller) GetClient(ctx context.Context, etcdS3 *config.EtcdS3) (*Cli
 
 	exists, err := mc.BucketExists(ctx, etcdS3.Bucket)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to test for existence of bucket %s", etcdS3.Bucket)
+		return nil, pkgerrors.WithMessagef(err, "failed to test for existence of bucket %s", etcdS3.Bucket)
 	}
 	if !exists {
 		return nil, fmt.Errorf("bucket %s does not exist", etcdS3.Bucket)

pkg/etcd/snapshot.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -17,14 +18,15 @@ import (
 	"strings"
 	"time"
 
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	k3s "github.com/k3s-io/api/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd/s3"
 	"github.com/k3s-io/k3s/pkg/etcd/snapshot"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/k3s-io/k3s/pkg/util/metrics"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 	snapshotv3 "go.etcd.io/etcd/client/v3/snapshot"
@@ -191,6 +193,20 @@ func (e *ETCD) decompressSnapshot(snapshotDir, snapshotFile string) (string, err
 // subcommand for prune that can be run manually if the user wants to remove old snapshots.
 // Returns metadata about the new and pruned snapshots.
 func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
+	res, err := e.snapshot(ctx)
+	if err != nil {
+		return res, err
+	}
+	return res, e.reconcileSnapshotData(ctx, res)
+}
+
+// snapshot is the actual snapshot save/upload implementation.
+// This is not inline in the Snapshot function so that the save and reconcile operation
+// metrics do not overlap.
+func (e *ETCD) snapshot(ctx context.Context) (_ *managed.SnapshotResult, rerr error) {
+	snapshotStart := time.Now()
+	defer metrics.ObserveWithStatus(snapshotSaveCount, snapshotStart, rerr)
+
 	if !e.snapshotMu.TryLock() {
 		return nil, errors.New("snapshot save already in progress")
 	}
@@ -212,7 +228,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 	endpoints := getEndpoints(e.config)
 	status, err := e.client.Status(ctx, endpoints[0])
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to check etcd status for snapshot")
+		return nil, pkgerrors.WithMessage(err, "failed to check etcd status for snapshot")
 	}
 
 	if status.IsLearner {
@@ -222,17 +238,17 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 
 	snapshotDir, err := snapshotDir(e.config, true)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get etcd-snapshot-dir")
+		return nil, pkgerrors.WithMessage(err, "failed to get etcd-snapshot-dir")
 	}
 
 	cfg, err := getClientConfig(ctx, e.config)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get config for etcd snapshot")
+		return nil, pkgerrors.WithMessage(err, "failed to get config for etcd snapshot")
 	}
 
 	tokenHash, err := util.GetTokenHash(e.config)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get server token hash for etcd snapshot")
+		return nil, pkgerrors.WithMessage(err, "failed to get server token hash for etcd snapshot")
 	}
 
 	nodeName := os.Getenv("NODE_NAME")
@@ -243,7 +259,11 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 
 	var sf *snapshot.File
 
-	if err := snapshotv3.Save(ctx, e.client.GetLogger(), *cfg, snapshotPath); err != nil {
+	saveStart := time.Now()
+	err = snapshotv3.Save(ctx, e.client.GetLogger(), *cfg, snapshotPath)
+	metrics.ObserveWithStatus(snapshotSaveLocalCount, saveStart, err)
+
+	if err != nil {
 		sf = &snapshot.File{
 			Name:     snapshotName,
 			Location: "",
@@ -258,7 +278,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 		}
 		logrus.Errorf("Failed to take etcd snapshot: %v", err)
 		if err := e.addSnapshotData(*sf); err != nil {
-			return nil, errors.Wrap(err, "failed to sync ETCDSnapshotFile")
+			return nil, pkgerrors.WithMessage(err, "failed to sync ETCDSnapshotFile")
 		}
 	}
 
@@ -274,7 +294,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 			}
 
 			if err != nil {
-				return nil, errors.Wrap(err, "failed to compress snapshot")
+				return nil, pkgerrors.WithMessage(err, "failed to compress snapshot")
 			}
 			snapshotPath = zipPath
 			logrus.Info("Compressed snapshot: " + snapshotPath)
@@ -282,7 +302,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 
 		f, err := os.Stat(snapshotPath)
 		if err != nil {
-			return nil, errors.Wrap(err, "unable to retrieve snapshot information from local snapshot")
+			return nil, pkgerrors.WithMessage(err, "unable to retrieve snapshot information from local snapshot")
 		}
 
 		sf = &snapshot.File{
@@ -319,10 +339,12 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 		res.Deleted = append(res.Deleted, deleted...)
 
 		if e.config.EtcdS3 != nil {
+			s3Start := time.Now()
 			if s3client, err := e.getS3Client(ctx); err != nil {
 				logrus.Warnf("Unable to initialize S3 client: %v", err)
 				if !errors.Is(err, s3.ErrNoConfigSecret) {
-					err = errors.Wrap(err, "failed to initialize S3 client")
+					metrics.ObserveWithStatus(snapshotSaveS3Count, s3Start, err)
+					err = pkgerrors.WithMessage(err, "failed to initialize S3 client")
 					sf = &snapshot.File{
 						Name:     f.Name(),
 						NodeName: "s3",
@@ -341,6 +363,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 				// upload will return a snapshot.File even on error - if there was an
 				// error, it will be reflected in the status and message.
 				sf, err = s3client.Upload(ctx, snapshotPath, extraMetadata, now)
+				metrics.ObserveWithStatus(snapshotSaveS3Count, s3Start, err)
 				if err != nil {
 					logrus.Errorf("Error received during snapshot upload to S3: %s", err)
 				} else {
@@ -365,7 +388,7 @@ func (e *ETCD) Snapshot(ctx context.Context) (*managed.SnapshotResult, error) {
 		}
 	}
 
-	return res, e.reconcileSnapshotData(ctx, res)
+	return res, nil
 }
 
 // listLocalSnapshots provides a list of the currently stored
@@ -376,7 +399,7 @@ func (e *ETCD) listLocalSnapshots() (map[string]snapshot.File, error) {
 	snapshots := make(map[string]snapshot.File)
 	snapshotDir, err := snapshotDir(e.config, true)
 	if err != nil {
-		return snapshots, errors.Wrap(err, "failed to get etcd-snapshot-dir")
+		return snapshots, pkgerrors.WithMessage(err, "failed to get etcd-snapshot-dir")
 	}
 
 	if err := filepath.Walk(snapshotDir, func(path string, file os.FileInfo, err error) error {
@@ -444,7 +467,7 @@ func (e *ETCD) getS3Client(ctx context.Context) (*s3.Client, error) {
 func (e *ETCD) PruneSnapshots(ctx context.Context) (*managed.SnapshotResult, error) {
 	snapshotDir, err := snapshotDir(e.config, false)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get etcd-snapshot-dir")
+		return nil, pkgerrors.WithMessage(err, "failed to get etcd-snapshot-dir")
 	}
 
 	res := &managed.SnapshotResult{}
@@ -482,7 +505,7 @@ func (e *ETCD) ListSnapshots(ctx context.Context) (*k3s.ETCDSnapshotFileList, er
 		if s3client, err := e.getS3Client(ctx); err != nil {
 			logrus.Warnf("Unable to initialize S3 client: %v", err)
 			if !errors.Is(err, s3.ErrNoConfigSecret) {
-				return nil, errors.Wrap(err, "failed to initialize S3 client")
+				return nil, pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			}
 		} else {
 			sfs, err := s3client.ListSnapshots(ctx)
@@ -516,7 +539,7 @@ func (e *ETCD) ListSnapshots(ctx context.Context) (*k3s.ETCDSnapshotFileList, er
 func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) (*managed.SnapshotResult, error) {
 	snapshotDir, err := snapshotDir(e.config, false)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get etcd-snapshot-dir")
+		return nil, pkgerrors.WithMessage(err, "failed to get etcd-snapshot-dir")
 	}
 
 	var s3client *s3.Client
@@ -525,7 +548,7 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) (*manage
 		if err != nil {
 			logrus.Warnf("Unable to initialize S3 client: %v", err)
 			if !errors.Is(err, s3.ErrNoConfigSecret) {
-				return nil, errors.Wrap(err, "failed to initialize S3 client")
+				return nil, pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			}
 		}
 	}
@@ -659,7 +682,10 @@ func (e *ETCD) ReconcileSnapshotData(ctx context.Context) error {
 // It will reconcile snapshot data from disk locally always, and if S3 is enabled, will attempt to
 // list S3 snapshots and reconcile snapshots from S3. Any snapshots listed in the Deleted field of
 // the provided SnapshotResult are deleted, even if they are within a retention window.
-func (e *ETCD) reconcileSnapshotData(ctx context.Context, res *managed.SnapshotResult) error {
+func (e *ETCD) reconcileSnapshotData(ctx context.Context, res *managed.SnapshotResult) (rerr error) {
+	reconcileStart := time.Now()
+	defer metrics.ObserveWithStatus(snapshotReconcileCount, reconcileStart, rerr)
+
 	// make sure the core.Factory is initialized. There can
 	// be a race between this core code startup.
 	for e.config.Runtime.Core == nil {
@@ -670,7 +696,9 @@ func (e *ETCD) reconcileSnapshotData(ctx context.Context, res *managed.SnapshotR
 	defer logrus.Infof("Reconciliation of ETCDSnapshotFile resources complete")
 
 	// Get snapshots from local filesystem
+	localStart := time.Now()
 	snapshotFiles, err := e.listLocalSnapshots()
+	metrics.ObserveWithStatus(snapshotReconcileLocalCount, localStart, err)
 	if err != nil {
 		return err
 	}
@@ -679,13 +707,17 @@ func (e *ETCD) reconcileSnapshotData(ctx context.Context, res *managed.SnapshotR
 
 	// Get snapshots from S3
 	if e.config.EtcdS3 != nil {
+		s3Start := time.Now()
 		if s3client, err := e.getS3Client(ctx); err != nil {
 			logrus.Warnf("Unable to initialize S3 client: %v", err)
 			if !errors.Is(err, s3.ErrNoConfigSecret) {
-				return errors.Wrap(err, "failed to initialize S3 client")
+				metrics.ObserveWithStatus(snapshotReconcileS3Count, s3Start, err)
+				return pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			}
 		} else {
-			if s3Snapshots, err := s3client.ListSnapshots(ctx); err != nil {
+			s3Snapshots, err := s3client.ListSnapshots(ctx)
+			metrics.ObserveWithStatus(snapshotReconcileS3Count, s3Start, err)
+			if err != nil {
 				logrus.Errorf("Error retrieving S3 snapshots for reconciliation: %v", err)
 			} else {
 				for k, v := range s3Snapshots {

pkg/etcd/snapshot/types.go

@@ -11,7 +11,7 @@ import (
 	"regexp"
 	"strings"
 
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	k3s "github.com/k3s-io/api/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/minio/minio-go/v7"

pkg/etcd/snapshot_controller.go

@@ -2,19 +2,20 @@ package etcd
 
 import (
 	"context"
+	"errors"
 	"os"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	apisv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	apisv1 "github.com/k3s-io/api/k3s.cattle.io/v1"
+	k3s "github.com/k3s-io/api/k3s.cattle.io/v1"
+	controllersv1 "github.com/k3s-io/api/pkg/generated/controllers/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/etcd/snapshot"
-	controllersv1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	controllerv1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -90,14 +91,14 @@ func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*a
 	sfKey := sf.GenerateConfigMapKey()
 	m, err := sf.Marshal()
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to marshal snapshot ConfigMap data")
+		return nil, pkgerrors.WithMessage(err, "failed to marshal snapshot ConfigMap data")
 	}
 	marshalledSnapshot := string(m)
 
 	snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{})
 	if err != nil {
 		if !apierrors.IsNotFound(err) {
-			return nil, errors.Wrap(err, "failed to get snapshot ConfigMap")
+			return nil, pkgerrors.WithMessage(err, "failed to get snapshot ConfigMap")
 		}
 		snapshotConfigMap = &v1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{
@@ -142,7 +143,7 @@ func (e *etcdSnapshotHandler) sync(key string, esf *apisv1.ETCDSnapshotFile) (*a
 	}
 
 	if err != nil {
-		err = errors.Wrap(err, "failed to sync snapshot to ConfigMap")
+		err = pkgerrors.WithMessage(err, "failed to sync snapshot to ConfigMap")
 	}
 
 	return nil, err
@@ -157,14 +158,14 @@ func (e *etcdSnapshotHandler) onRemove(key string, esf *apisv1.ETCDSnapshotFile)
 		if apierrors.IsNotFound(err) {
 			return nil, nil
 		}
-		return nil, errors.Wrap(err, "failed to get snapshot ConfigMap")
+		return nil, pkgerrors.WithMessage(err, "failed to get snapshot ConfigMap")
 	}
 
 	sfKey := generateETCDSnapshotFileConfigMapKey(*esf)
 	if _, ok := snapshotConfigMap.Data[sfKey]; ok {
 		delete(snapshotConfigMap.Data, sfKey)
 		if _, err := e.configmaps.Update(snapshotConfigMap); err != nil {
-			return nil, errors.Wrap(err, "failed to remove snapshot from ConfigMap")
+			return nil, pkgerrors.WithMessage(err, "failed to remove snapshot from ConfigMap")
 		}
 	}
 	e.etcd.emitEvent(esf)
@@ -243,7 +244,7 @@ func (e *etcdSnapshotHandler) reconcile() error {
 	snapshotConfigMap, err := e.configmaps.Get(metav1.NamespaceSystem, snapshotConfigMapName, metav1.GetOptions{})
 	if err != nil {
 		if !apierrors.IsNotFound(err) {
-			return errors.Wrap(err, "failed to get snapshot ConfigMap")
+			return pkgerrors.WithMessage(err, "failed to get snapshot ConfigMap")
 		}
 		snapshotConfigMap = &v1.ConfigMap{
 			ObjectMeta: metav1.ObjectMeta{

pkg/etcd/snapshot_handler.go

@@ -7,11 +7,11 @@ import (
 	"io"
 	"net/http"
 
-	k3s "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
+	k3s "github.com/k3s-io/api/k3s.cattle.io/v1"
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -70,7 +70,7 @@ func (e *ETCD) snapshotHandler() http.Handler {
 func (e *ETCD) handleList(rw http.ResponseWriter, req *http.Request) error {
 	if e.config.EtcdS3 != nil {
 		if _, err := e.getS3Client(req.Context()); err != nil {
-			err = errors.Wrap(err, "failed to initialize S3 client")
+			err = pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			util.SendError(err, rw, req, http.StatusBadRequest)
 			return nil
 		}
@@ -87,7 +87,7 @@ func (e *ETCD) handleList(rw http.ResponseWriter, req *http.Request) error {
 func (e *ETCD) handleSave(rw http.ResponseWriter, req *http.Request) error {
 	if e.config.EtcdS3 != nil {
 		if _, err := e.getS3Client(req.Context()); err != nil {
-			err = errors.Wrap(err, "failed to initialize S3 client")
+			err = pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			util.SendError(err, rw, req, http.StatusBadRequest)
 			return nil
 		}
@@ -104,7 +104,7 @@ func (e *ETCD) handleSave(rw http.ResponseWriter, req *http.Request) error {
 func (e *ETCD) handlePrune(rw http.ResponseWriter, req *http.Request) error {
 	if e.config.EtcdS3 != nil {
 		if _, err := e.getS3Client(req.Context()); err != nil {
-			err = errors.Wrap(err, "failed to initialize S3 client")
+			err = pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			util.SendError(err, rw, req, http.StatusBadRequest)
 			return nil
 		}
@@ -121,7 +121,7 @@ func (e *ETCD) handlePrune(rw http.ResponseWriter, req *http.Request) error {
 func (e *ETCD) handleDelete(rw http.ResponseWriter, req *http.Request, snapshots []string) error {
 	if e.config.EtcdS3 != nil {
 		if _, err := e.getS3Client(req.Context()); err != nil {
-			err = errors.Wrap(err, "failed to initialize S3 client")
+			err = pkgerrors.WithMessage(err, "failed to initialize S3 client")
 			util.SendError(err, rw, req, http.StatusBadRequest)
 			return nil
 		}

pkg/etcd/snapshot_metrics.go

+package etcd
+
+import (
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/component-base/metrics"
+)
+
+var (
+	snapshotSaveCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_save_duration_seconds",
+		Help:    "Total time taken to complete the etcd snapshot process",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+
+	snapshotSaveLocalCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_save_local_duration_seconds",
+		Help:    "Total time taken to save a local snapshot file",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+
+	snapshotSaveS3Count = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_save_s3_duration_seconds",
+		Help:    "Total time taken to upload a snapshot file to S3",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+
+	snapshotReconcileCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_reconcile_duration_seconds",
+		Help:    "Total time taken to sync the list of etcd snapshots",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+
+	snapshotReconcileLocalCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_reconcile_local_duration_seconds",
+		Help:    "Total time taken to list local snapshot files",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+
+	snapshotReconcileS3Count = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    version.Program + "_etcd_snapshot_reconcile_s3_duration_seconds",
+		Help:    "Total time taken to list S3 snapshot files",
+		Buckets: metrics.ExponentialBuckets(0.008, 2, 15),
+	}, []string{"status"})
+)
+
+// MustRegister registers etcd snapshot metrics
+func MustRegister(registerer prometheus.Registerer) {
+	registerer.MustRegister(
+		snapshotSaveCount,
+		snapshotSaveLocalCount,
+		snapshotSaveS3Count,
+		snapshotReconcileCount,
+		snapshotReconcileLocalCount,
+		snapshotReconcileS3Count,
+	)
+}

pkg/generated/clientset/versioned/clientset.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package versioned
-
-import (
-	"fmt"
-	"net/http"
-
-	k3sv1 "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1"
-	discovery "k8s.io/client-go/discovery"
-	rest "k8s.io/client-go/rest"
-	flowcontrol "k8s.io/client-go/util/flowcontrol"
-)
-
-type Interface interface {
-	Discovery() discovery.DiscoveryInterface
-	K3sV1() k3sv1.K3sV1Interface
-}
-
-// Clientset contains the clients for groups.
-type Clientset struct {
-	*discovery.DiscoveryClient
-	k3sV1 *k3sv1.K3sV1Client
-}
-
-// K3sV1 retrieves the K3sV1Client
-func (c *Clientset) K3sV1() k3sv1.K3sV1Interface {
-	return c.k3sV1
-}
-
-// Discovery retrieves the DiscoveryClient
-func (c *Clientset) Discovery() discovery.DiscoveryInterface {
-	if c == nil {
-		return nil
-	}
-	return c.DiscoveryClient
-}
-
-// NewForConfig creates a new Clientset for the given config.
-// If config's RateLimiter is not set and QPS and Burst are acceptable,
-// NewForConfig will generate a rate-limiter in configShallowCopy.
-// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
-// where httpClient was generated with rest.HTTPClientFor(c).
-func NewForConfig(c *rest.Config) (*Clientset, error) {
-	configShallowCopy := *c
-
-	if configShallowCopy.UserAgent == "" {
-		configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent()
-	}
-
-	// share the transport between all clients
-	httpClient, err := rest.HTTPClientFor(&configShallowCopy)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewForConfigAndClient(&configShallowCopy, httpClient)
-}
-
-// NewForConfigAndClient creates a new Clientset for the given config and http client.
-// Note the http client provided takes precedence over the configured transport values.
-// If config's RateLimiter is not set and QPS and Burst are acceptable,
-// NewForConfigAndClient will generate a rate-limiter in configShallowCopy.
-func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) {
-	configShallowCopy := *c
-	if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 {
-		if configShallowCopy.Burst <= 0 {
-			return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0")
-		}
-		configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst)
-	}
-
-	var cs Clientset
-	var err error
-	cs.k3sV1, err = k3sv1.NewForConfigAndClient(&configShallowCopy, httpClient)
-	if err != nil {
-		return nil, err
-	}
-
-	cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient)
-	if err != nil {
-		return nil, err
-	}
-	return &cs, nil
-}
-
-// NewForConfigOrDie creates a new Clientset for the given config and
-// panics if there is an error in the config.
-func NewForConfigOrDie(c *rest.Config) *Clientset {
-	cs, err := NewForConfig(c)
-	if err != nil {
-		panic(err)
-	}
-	return cs
-}
-
-// New creates a new Clientset for the given RESTClient.
-func New(c rest.Interface) *Clientset {
-	var cs Clientset
-	cs.k3sV1 = k3sv1.New(c)
-
-	cs.DiscoveryClient = discovery.NewDiscoveryClient(c)
-	return &cs
-}

pkg/generated/clientset/versioned/fake/clientset_generated.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	clientset "github.com/k3s-io/k3s/pkg/generated/clientset/versioned"
-	k3sv1 "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1"
-	fakek3sv1 "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/discovery"
-	fakediscovery "k8s.io/client-go/discovery/fake"
-	"k8s.io/client-go/testing"
-)
-
-// NewSimpleClientset returns a clientset that will respond with the provided objects.
-// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
-// without applying any validations and/or defaults. It shouldn't be considered a replacement
-// for a real clientset and is mostly useful in simple unit tests.
-func NewSimpleClientset(objects ...runtime.Object) *Clientset {
-	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
-	for _, obj := range objects {
-		if err := o.Add(obj); err != nil {
-			panic(err)
-		}
-	}
-
-	cs := &Clientset{tracker: o}
-	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
-	cs.AddReactor("*", "*", testing.ObjectReaction(o))
-	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
-		gvr := action.GetResource()
-		ns := action.GetNamespace()
-		watch, err := o.Watch(gvr, ns)
-		if err != nil {
-			return false, nil, err
-		}
-		return true, watch, nil
-	})
-
-	return cs
-}
-
-// Clientset implements clientset.Interface. Meant to be embedded into a
-// struct to get a default implementation. This makes faking out just the method
-// you want to test easier.
-type Clientset struct {
-	testing.Fake
-	discovery *fakediscovery.FakeDiscovery
-	tracker   testing.ObjectTracker
-}
-
-func (c *Clientset) Discovery() discovery.DiscoveryInterface {
-	return c.discovery
-}
-
-func (c *Clientset) Tracker() testing.ObjectTracker {
-	return c.tracker
-}
-
-var (
-	_ clientset.Interface = &Clientset{}
-	_ testing.FakeClient  = &Clientset{}
-)
-
-// K3sV1 retrieves the K3sV1Client
-func (c *Clientset) K3sV1() k3sv1.K3sV1Interface {
-	return &fakek3sv1.FakeK3sV1{Fake: &c.Fake}
-}

pkg/generated/clientset/versioned/fake/doc.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// This package has the automatically generated fake clientset.
-package fake

pkg/generated/clientset/versioned/fake/register.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	k3sv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-)
-
-var scheme = runtime.NewScheme()
-var codecs = serializer.NewCodecFactory(scheme)
-
-var localSchemeBuilder = runtime.SchemeBuilder{
-	k3sv1.AddToScheme,
-}
-
-// AddToScheme adds all types of this clientset into the given scheme. This allows composition
-// of clientsets, like in:
-//
-//	import (
-//	  "k8s.io/client-go/kubernetes"
-//	  clientsetscheme "k8s.io/client-go/kubernetes/scheme"
-//	  aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
-//	)
-//
-//	kclientset, _ := kubernetes.NewForConfig(c)
-//	_ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
-//
-// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
-// correctly.
-var AddToScheme = localSchemeBuilder.AddToScheme
-
-func init() {
-	v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"})
-	utilruntime.Must(AddToScheme(scheme))
-}

pkg/generated/clientset/versioned/scheme/doc.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// This package contains the scheme of the automatically generated clientset.
-package scheme

pkg/generated/clientset/versioned/scheme/register.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package scheme
-
-import (
-	k3sv1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-	serializer "k8s.io/apimachinery/pkg/runtime/serializer"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-)
-
-var Scheme = runtime.NewScheme()
-var Codecs = serializer.NewCodecFactory(Scheme)
-var ParameterCodec = runtime.NewParameterCodec(Scheme)
-var localSchemeBuilder = runtime.SchemeBuilder{
-	k3sv1.AddToScheme,
-}
-
-// AddToScheme adds all types of this clientset into the given scheme. This allows composition
-// of clientsets, like in:
-//
-//	import (
-//	  "k8s.io/client-go/kubernetes"
-//	  clientsetscheme "k8s.io/client-go/kubernetes/scheme"
-//	  aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
-//	)
-//
-//	kclientset, _ := kubernetes.NewForConfig(c)
-//	_ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme)
-//
-// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types
-// correctly.
-var AddToScheme = localSchemeBuilder.AddToScheme
-
-func init() {
-	v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})
-	utilruntime.Must(AddToScheme(Scheme))
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/addon.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"context"
-	"time"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	scheme "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	rest "k8s.io/client-go/rest"
-)
-
-// AddonsGetter has a method to return a AddonInterface.
-// A group's client should implement this interface.
-type AddonsGetter interface {
-	Addons(namespace string) AddonInterface
-}
-
-// AddonInterface has methods to work with Addon resources.
-type AddonInterface interface {
-	Create(ctx context.Context, addon *v1.Addon, opts metav1.CreateOptions) (*v1.Addon, error)
-	Update(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (*v1.Addon, error)
-	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
-	DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
-	Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.Addon, error)
-	List(ctx context.Context, opts metav1.ListOptions) (*v1.AddonList, error)
-	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
-	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Addon, err error)
-	AddonExpansion
-}
-
-// addons implements AddonInterface
-type addons struct {
-	client rest.Interface
-	ns     string
-}
-
-// newAddons returns a Addons
-func newAddons(c *K3sV1Client, namespace string) *addons {
-	return &addons{
-		client: c.RESTClient(),
-		ns:     namespace,
-	}
-}
-
-// Get takes name of the addon, and returns the corresponding addon object, and an error if there is any.
-func (c *addons) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Addon, err error) {
-	result = &v1.Addon{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("addons").
-		Name(name).
-		VersionedParams(&options, scheme.ParameterCodec).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// List takes label and field selectors, and returns the list of Addons that match those selectors.
-func (c *addons) List(ctx context.Context, opts metav1.ListOptions) (result *v1.AddonList, err error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	result = &v1.AddonList{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("addons").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Watch returns a watch.Interface that watches the requested addons.
-func (c *addons) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	opts.Watch = true
-	return c.client.Get().
-		Namespace(c.ns).
-		Resource("addons").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Watch(ctx)
-}
-
-// Create takes the representation of a addon and creates it.  Returns the server's representation of the addon, and an error, if there is any.
-func (c *addons) Create(ctx context.Context, addon *v1.Addon, opts metav1.CreateOptions) (result *v1.Addon, err error) {
-	result = &v1.Addon{}
-	err = c.client.Post().
-		Namespace(c.ns).
-		Resource("addons").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(addon).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Update takes the representation of a addon and updates it. Returns the server's representation of the addon, and an error, if there is any.
-func (c *addons) Update(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (result *v1.Addon, err error) {
-	result = &v1.Addon{}
-	err = c.client.Put().
-		Namespace(c.ns).
-		Resource("addons").
-		Name(addon.Name).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(addon).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Delete takes name of the addon and deletes it. Returns an error if one occurs.
-func (c *addons) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("addons").
-		Name(name).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *addons) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
-	var timeout time.Duration
-	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
-	}
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("addons").
-		VersionedParams(&listOpts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// Patch applies the patch and returns the patched addon.
-func (c *addons) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Addon, err error) {
-	result = &v1.Addon{}
-	err = c.client.Patch(pt).
-		Namespace(c.ns).
-		Resource("addons").
-		Name(name).
-		SubResource(subresources...).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(data).
-		Do(ctx).
-		Into(result)
-	return
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/doc.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// This package has the automatically generated typed clients.
-package v1

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/etcdsnapshotfile.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"context"
-	"time"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	scheme "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	rest "k8s.io/client-go/rest"
-)
-
-// ETCDSnapshotFilesGetter has a method to return a ETCDSnapshotFileInterface.
-// A group's client should implement this interface.
-type ETCDSnapshotFilesGetter interface {
-	ETCDSnapshotFiles() ETCDSnapshotFileInterface
-}
-
-// ETCDSnapshotFileInterface has methods to work with ETCDSnapshotFile resources.
-type ETCDSnapshotFileInterface interface {
-	Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (*v1.ETCDSnapshotFile, error)
-	Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error)
-	UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error)
-	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
-	DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
-	Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.ETCDSnapshotFile, error)
-	List(ctx context.Context, opts metav1.ListOptions) (*v1.ETCDSnapshotFileList, error)
-	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
-	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error)
-	ETCDSnapshotFileExpansion
-}
-
-// eTCDSnapshotFiles implements ETCDSnapshotFileInterface
-type eTCDSnapshotFiles struct {
-	client rest.Interface
-}
-
-// newETCDSnapshotFiles returns a ETCDSnapshotFiles
-func newETCDSnapshotFiles(c *K3sV1Client) *eTCDSnapshotFiles {
-	return &eTCDSnapshotFiles{
-		client: c.RESTClient(),
-	}
-}
-
-// Get takes name of the eTCDSnapshotFile, and returns the corresponding eTCDSnapshotFile object, and an error if there is any.
-func (c *eTCDSnapshotFiles) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ETCDSnapshotFile, err error) {
-	result = &v1.ETCDSnapshotFile{}
-	err = c.client.Get().
-		Resource("etcdsnapshotfiles").
-		Name(name).
-		VersionedParams(&options, scheme.ParameterCodec).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// List takes label and field selectors, and returns the list of ETCDSnapshotFiles that match those selectors.
-func (c *eTCDSnapshotFiles) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ETCDSnapshotFileList, err error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	result = &v1.ETCDSnapshotFileList{}
-	err = c.client.Get().
-		Resource("etcdsnapshotfiles").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Watch returns a watch.Interface that watches the requested eTCDSnapshotFiles.
-func (c *eTCDSnapshotFiles) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	opts.Watch = true
-	return c.client.Get().
-		Resource("etcdsnapshotfiles").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Watch(ctx)
-}
-
-// Create takes the representation of a eTCDSnapshotFile and creates it.  Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any.
-func (c *eTCDSnapshotFiles) Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (result *v1.ETCDSnapshotFile, err error) {
-	result = &v1.ETCDSnapshotFile{}
-	err = c.client.Post().
-		Resource("etcdsnapshotfiles").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(eTCDSnapshotFile).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Update takes the representation of a eTCDSnapshotFile and updates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any.
-func (c *eTCDSnapshotFiles) Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) {
-	result = &v1.ETCDSnapshotFile{}
-	err = c.client.Put().
-		Resource("etcdsnapshotfiles").
-		Name(eTCDSnapshotFile.Name).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(eTCDSnapshotFile).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *eTCDSnapshotFiles) UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) {
-	result = &v1.ETCDSnapshotFile{}
-	err = c.client.Put().
-		Resource("etcdsnapshotfiles").
-		Name(eTCDSnapshotFile.Name).
-		SubResource("status").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(eTCDSnapshotFile).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Delete takes name of the eTCDSnapshotFile and deletes it. Returns an error if one occurs.
-func (c *eTCDSnapshotFiles) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
-	return c.client.Delete().
-		Resource("etcdsnapshotfiles").
-		Name(name).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *eTCDSnapshotFiles) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
-	var timeout time.Duration
-	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
-	}
-	return c.client.Delete().
-		Resource("etcdsnapshotfiles").
-		VersionedParams(&listOpts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// Patch applies the patch and returns the patched eTCDSnapshotFile.
-func (c *eTCDSnapshotFiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error) {
-	result = &v1.ETCDSnapshotFile{}
-	err = c.client.Patch(pt).
-		Resource("etcdsnapshotfiles").
-		Name(name).
-		SubResource(subresources...).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(data).
-		Do(ctx).
-		Into(result)
-	return
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/doc.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-// Package fake has the automatically generated clients.
-package fake

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_addon.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	"context"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	labels "k8s.io/apimachinery/pkg/labels"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	testing "k8s.io/client-go/testing"
-)
-
-// FakeAddons implements AddonInterface
-type FakeAddons struct {
-	Fake *FakeK3sV1
-	ns   string
-}
-
-var addonsResource = v1.SchemeGroupVersion.WithResource("addons")
-
-var addonsKind = v1.SchemeGroupVersion.WithKind("Addon")
-
-// Get takes name of the addon, and returns the corresponding addon object, and an error if there is any.
-func (c *FakeAddons) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.Addon, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewGetAction(addonsResource, c.ns, name), &v1.Addon{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.Addon), err
-}
-
-// List takes label and field selectors, and returns the list of Addons that match those selectors.
-func (c *FakeAddons) List(ctx context.Context, opts metav1.ListOptions) (result *v1.AddonList, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewListAction(addonsResource, addonsKind, c.ns, opts), &v1.AddonList{})
-
-	if obj == nil {
-		return nil, err
-	}
-
-	label, _, _ := testing.ExtractFromListOptions(opts)
-	if label == nil {
-		label = labels.Everything()
-	}
-	list := &v1.AddonList{ListMeta: obj.(*v1.AddonList).ListMeta}
-	for _, item := range obj.(*v1.AddonList).Items {
-		if label.Matches(labels.Set(item.Labels)) {
-			list.Items = append(list.Items, item)
-		}
-	}
-	return list, err
-}
-
-// Watch returns a watch.Interface that watches the requested addons.
-func (c *FakeAddons) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
-	return c.Fake.
-		InvokesWatch(testing.NewWatchAction(addonsResource, c.ns, opts))
-
-}
-
-// Create takes the representation of a addon and creates it.  Returns the server's representation of the addon, and an error, if there is any.
-func (c *FakeAddons) Create(ctx context.Context, addon *v1.Addon, opts metav1.CreateOptions) (result *v1.Addon, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewCreateAction(addonsResource, c.ns, addon), &v1.Addon{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.Addon), err
-}
-
-// Update takes the representation of a addon and updates it. Returns the server's representation of the addon, and an error, if there is any.
-func (c *FakeAddons) Update(ctx context.Context, addon *v1.Addon, opts metav1.UpdateOptions) (result *v1.Addon, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewUpdateAction(addonsResource, c.ns, addon), &v1.Addon{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.Addon), err
-}
-
-// Delete takes name of the addon and deletes it. Returns an error if one occurs.
-func (c *FakeAddons) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
-	_, err := c.Fake.
-		Invokes(testing.NewDeleteActionWithOptions(addonsResource, c.ns, name, opts), &v1.Addon{})
-
-	return err
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *FakeAddons) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
-	action := testing.NewDeleteCollectionAction(addonsResource, c.ns, listOpts)
-
-	_, err := c.Fake.Invokes(action, &v1.AddonList{})
-	return err
-}
-
-// Patch applies the patch and returns the patched addon.
-func (c *FakeAddons) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.Addon, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewPatchSubresourceAction(addonsResource, c.ns, name, pt, data, subresources...), &v1.Addon{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.Addon), err
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_etcdsnapshotfile.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	"context"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	labels "k8s.io/apimachinery/pkg/labels"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	testing "k8s.io/client-go/testing"
-)
-
-// FakeETCDSnapshotFiles implements ETCDSnapshotFileInterface
-type FakeETCDSnapshotFiles struct {
-	Fake *FakeK3sV1
-}
-
-var etcdsnapshotfilesResource = v1.SchemeGroupVersion.WithResource("etcdsnapshotfiles")
-
-var etcdsnapshotfilesKind = v1.SchemeGroupVersion.WithKind("ETCDSnapshotFile")
-
-// Get takes name of the eTCDSnapshotFile, and returns the corresponding eTCDSnapshotFile object, and an error if there is any.
-func (c *FakeETCDSnapshotFiles) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.ETCDSnapshotFile, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootGetAction(etcdsnapshotfilesResource, name), &v1.ETCDSnapshotFile{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.ETCDSnapshotFile), err
-}
-
-// List takes label and field selectors, and returns the list of ETCDSnapshotFiles that match those selectors.
-func (c *FakeETCDSnapshotFiles) List(ctx context.Context, opts metav1.ListOptions) (result *v1.ETCDSnapshotFileList, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootListAction(etcdsnapshotfilesResource, etcdsnapshotfilesKind, opts), &v1.ETCDSnapshotFileList{})
-	if obj == nil {
-		return nil, err
-	}
-
-	label, _, _ := testing.ExtractFromListOptions(opts)
-	if label == nil {
-		label = labels.Everything()
-	}
-	list := &v1.ETCDSnapshotFileList{ListMeta: obj.(*v1.ETCDSnapshotFileList).ListMeta}
-	for _, item := range obj.(*v1.ETCDSnapshotFileList).Items {
-		if label.Matches(labels.Set(item.Labels)) {
-			list.Items = append(list.Items, item)
-		}
-	}
-	return list, err
-}
-
-// Watch returns a watch.Interface that watches the requested eTCDSnapshotFiles.
-func (c *FakeETCDSnapshotFiles) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
-	return c.Fake.
-		InvokesWatch(testing.NewRootWatchAction(etcdsnapshotfilesResource, opts))
-}
-
-// Create takes the representation of a eTCDSnapshotFile and creates it.  Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any.
-func (c *FakeETCDSnapshotFiles) Create(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.CreateOptions) (result *v1.ETCDSnapshotFile, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootCreateAction(etcdsnapshotfilesResource, eTCDSnapshotFile), &v1.ETCDSnapshotFile{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.ETCDSnapshotFile), err
-}
-
-// Update takes the representation of a eTCDSnapshotFile and updates it. Returns the server's representation of the eTCDSnapshotFile, and an error, if there is any.
-func (c *FakeETCDSnapshotFiles) Update(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (result *v1.ETCDSnapshotFile, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootUpdateAction(etcdsnapshotfilesResource, eTCDSnapshotFile), &v1.ETCDSnapshotFile{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.ETCDSnapshotFile), err
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *FakeETCDSnapshotFiles) UpdateStatus(ctx context.Context, eTCDSnapshotFile *v1.ETCDSnapshotFile, opts metav1.UpdateOptions) (*v1.ETCDSnapshotFile, error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootUpdateSubresourceAction(etcdsnapshotfilesResource, "status", eTCDSnapshotFile), &v1.ETCDSnapshotFile{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.ETCDSnapshotFile), err
-}
-
-// Delete takes name of the eTCDSnapshotFile and deletes it. Returns an error if one occurs.
-func (c *FakeETCDSnapshotFiles) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
-	_, err := c.Fake.
-		Invokes(testing.NewRootDeleteActionWithOptions(etcdsnapshotfilesResource, name, opts), &v1.ETCDSnapshotFile{})
-	return err
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *FakeETCDSnapshotFiles) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
-	action := testing.NewRootDeleteCollectionAction(etcdsnapshotfilesResource, listOpts)
-
-	_, err := c.Fake.Invokes(action, &v1.ETCDSnapshotFileList{})
-	return err
-}
-
-// Patch applies the patch and returns the patched eTCDSnapshotFile.
-func (c *FakeETCDSnapshotFiles) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ETCDSnapshotFile, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewRootPatchSubresourceAction(etcdsnapshotfilesResource, name, pt, data, subresources...), &v1.ETCDSnapshotFile{})
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1.ETCDSnapshotFile), err
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/fake/fake_k3s.cattle.io_client.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package fake
-
-import (
-	v1 "github.com/k3s-io/k3s/pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1"
-	rest "k8s.io/client-go/rest"
-	testing "k8s.io/client-go/testing"
-)
-
-type FakeK3sV1 struct {
-	*testing.Fake
-}
-
-func (c *FakeK3sV1) Addons(namespace string) v1.AddonInterface {
-	return &FakeAddons{c, namespace}
-}
-
-func (c *FakeK3sV1) ETCDSnapshotFiles() v1.ETCDSnapshotFileInterface {
-	return &FakeETCDSnapshotFiles{c}
-}
-
-// RESTClient returns a RESTClient that is used to communicate
-// with API server by this client implementation.
-func (c *FakeK3sV1) RESTClient() rest.Interface {
-	var ret *rest.RESTClient
-	return ret
-}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/generated_expansion.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-type AddonExpansion interface{}
-
-type ETCDSnapshotFileExpansion interface{}

pkg/generated/clientset/versioned/typed/k3s.cattle.io/v1/k3s.cattle.io_client.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"net/http"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
-	rest "k8s.io/client-go/rest"
-)
-
-type K3sV1Interface interface {
-	RESTClient() rest.Interface
-	AddonsGetter
-	ETCDSnapshotFilesGetter
-}
-
-// K3sV1Client is used to interact with features provided by the k3s.cattle.io group.
-type K3sV1Client struct {
-	restClient rest.Interface
-}
-
-func (c *K3sV1Client) Addons(namespace string) AddonInterface {
-	return newAddons(c, namespace)
-}
-
-func (c *K3sV1Client) ETCDSnapshotFiles() ETCDSnapshotFileInterface {
-	return newETCDSnapshotFiles(c)
-}
-
-// NewForConfig creates a new K3sV1Client for the given config.
-// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient),
-// where httpClient was generated with rest.HTTPClientFor(c).
-func NewForConfig(c *rest.Config) (*K3sV1Client, error) {
-	config := *c
-	if err := setConfigDefaults(&config); err != nil {
-		return nil, err
-	}
-	httpClient, err := rest.HTTPClientFor(&config)
-	if err != nil {
-		return nil, err
-	}
-	return NewForConfigAndClient(&config, httpClient)
-}
-
-// NewForConfigAndClient creates a new K3sV1Client for the given config and http client.
-// Note the http client provided takes precedence over the configured transport values.
-func NewForConfigAndClient(c *rest.Config, h *http.Client) (*K3sV1Client, error) {
-	config := *c
-	if err := setConfigDefaults(&config); err != nil {
-		return nil, err
-	}
-	client, err := rest.RESTClientForConfigAndClient(&config, h)
-	if err != nil {
-		return nil, err
-	}
-	return &K3sV1Client{client}, nil
-}
-
-// NewForConfigOrDie creates a new K3sV1Client for the given config and
-// panics if there is an error in the config.
-func NewForConfigOrDie(c *rest.Config) *K3sV1Client {
-	client, err := NewForConfig(c)
-	if err != nil {
-		panic(err)
-	}
-	return client
-}
-
-// New creates a new K3sV1Client for the given RESTClient.
-func New(c rest.Interface) *K3sV1Client {
-	return &K3sV1Client{c}
-}
-
-func setConfigDefaults(config *rest.Config) error {
-	gv := v1.SchemeGroupVersion
-	config.GroupVersion = &gv
-	config.APIPath = "/apis"
-	config.NegotiatedSerializer = scheme.Codecs.WithoutConversion()
-
-	if config.UserAgent == "" {
-		config.UserAgent = rest.DefaultKubernetesUserAgent()
-	}
-
-	return nil
-}
-
-// RESTClient returns a RESTClient that is used to communicate
-// with API server by this client implementation.
-func (c *K3sV1Client) RESTClient() rest.Interface {
-	if c == nil {
-		return nil
-	}
-	return c.restClient
-}

pkg/generated/controllers/k3s.cattle.io/factory.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package k3s
-
-import (
-	"github.com/rancher/lasso/pkg/controller"
-	"github.com/rancher/wrangler/pkg/generic"
-	"k8s.io/client-go/rest"
-)
-
-type Factory struct {
-	*generic.Factory
-}
-
-func NewFactoryFromConfigOrDie(config *rest.Config) *Factory {
-	f, err := NewFactoryFromConfig(config)
-	if err != nil {
-		panic(err)
-	}
-	return f
-}
-
-func NewFactoryFromConfig(config *rest.Config) (*Factory, error) {
-	return NewFactoryFromConfigWithOptions(config, nil)
-}
-
-func NewFactoryFromConfigWithNamespace(config *rest.Config, namespace string) (*Factory, error) {
-	return NewFactoryFromConfigWithOptions(config, &FactoryOptions{
-		Namespace: namespace,
-	})
-}
-
-type FactoryOptions = generic.FactoryOptions
-
-func NewFactoryFromConfigWithOptions(config *rest.Config, opts *FactoryOptions) (*Factory, error) {
-	f, err := generic.NewFactoryFromConfigWithOptions(config, opts)
-	return &Factory{
-		Factory: f,
-	}, err
-}
-
-func NewFactoryFromConfigWithOptionsOrDie(config *rest.Config, opts *FactoryOptions) *Factory {
-	f, err := NewFactoryFromConfigWithOptions(config, opts)
-	if err != nil {
-		panic(err)
-	}
-	return f
-}
-
-func (c *Factory) K3s() Interface {
-	return New(c.ControllerFactory())
-}
-
-func (c *Factory) WithAgent(userAgent string) Interface {
-	return New(controller.NewSharedControllerFactoryWithAgent(userAgent, c.ControllerFactory()))
-}

pkg/generated/controllers/k3s.cattle.io/interface.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package k3s
-
-import (
-	v1 "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io/v1"
-	"github.com/rancher/lasso/pkg/controller"
-)
-
-type Interface interface {
-	V1() v1.Interface
-}
-
-type group struct {
-	controllerFactory controller.SharedControllerFactory
-}
-
-// New returns a new Interface.
-func New(controllerFactory controller.SharedControllerFactory) Interface {
-	return &group{
-		controllerFactory: controllerFactory,
-	}
-}
-
-func (g *group) V1() v1.Interface {
-	return v1.New(g.controllerFactory)
-}

pkg/generated/controllers/k3s.cattle.io/v1/addon.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/generic"
-)
-
-// AddonController interface for managing Addon resources.
-type AddonController interface {
-	generic.ControllerInterface[*v1.Addon, *v1.AddonList]
-}
-
-// AddonClient interface for managing Addon resources in Kubernetes.
-type AddonClient interface {
-	generic.ClientInterface[*v1.Addon, *v1.AddonList]
-}
-
-// AddonCache interface for retrieving Addon resources in memory.
-type AddonCache interface {
-	generic.CacheInterface[*v1.Addon]
-}

pkg/generated/controllers/k3s.cattle.io/v1/etcdsnapshotfile.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	"context"
-	"time"
-
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/wrangler/pkg/apply"
-	"github.com/rancher/wrangler/pkg/condition"
-	"github.com/rancher/wrangler/pkg/generic"
-	"github.com/rancher/wrangler/pkg/kv"
-	"k8s.io/apimachinery/pkg/api/equality"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-// ETCDSnapshotFileController interface for managing ETCDSnapshotFile resources.
-type ETCDSnapshotFileController interface {
-	generic.NonNamespacedControllerInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList]
-}
-
-// ETCDSnapshotFileClient interface for managing ETCDSnapshotFile resources in Kubernetes.
-type ETCDSnapshotFileClient interface {
-	generic.NonNamespacedClientInterface[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList]
-}
-
-// ETCDSnapshotFileCache interface for retrieving ETCDSnapshotFile resources in memory.
-type ETCDSnapshotFileCache interface {
-	generic.NonNamespacedCacheInterface[*v1.ETCDSnapshotFile]
-}
-
-type ETCDSnapshotFileStatusHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error)
-
-type ETCDSnapshotFileGeneratingHandler func(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) ([]runtime.Object, v1.ETCDSnapshotStatus, error)
-
-func RegisterETCDSnapshotFileStatusHandler(ctx context.Context, controller ETCDSnapshotFileController, condition condition.Cond, name string, handler ETCDSnapshotFileStatusHandler) {
-	statusHandler := &eTCDSnapshotFileStatusHandler{
-		client:    controller,
-		condition: condition,
-		handler:   handler,
-	}
-	controller.AddGenericHandler(ctx, name, generic.FromObjectHandlerToHandler(statusHandler.sync))
-}
-
-func RegisterETCDSnapshotFileGeneratingHandler(ctx context.Context, controller ETCDSnapshotFileController, apply apply.Apply,
-	condition condition.Cond, name string, handler ETCDSnapshotFileGeneratingHandler, opts *generic.GeneratingHandlerOptions) {
-	statusHandler := &eTCDSnapshotFileGeneratingHandler{
-		ETCDSnapshotFileGeneratingHandler: handler,
-		apply:                             apply,
-		name:                              name,
-		gvk:                               controller.GroupVersionKind(),
-	}
-	if opts != nil {
-		statusHandler.opts = *opts
-	}
-	controller.OnChange(ctx, name, statusHandler.Remove)
-	RegisterETCDSnapshotFileStatusHandler(ctx, controller, condition, name, statusHandler.Handle)
-}
-
-type eTCDSnapshotFileStatusHandler struct {
-	client    ETCDSnapshotFileClient
-	condition condition.Cond
-	handler   ETCDSnapshotFileStatusHandler
-}
-
-func (a *eTCDSnapshotFileStatusHandler) sync(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) {
-	if obj == nil {
-		return obj, nil
-	}
-
-	origStatus := obj.Status.DeepCopy()
-	obj = obj.DeepCopy()
-	newStatus, err := a.handler(obj, obj.Status)
-	if err != nil {
-		// Revert to old status on error
-		newStatus = *origStatus.DeepCopy()
-	}
-
-	if a.condition != "" {
-		if errors.IsConflict(err) {
-			a.condition.SetError(&newStatus, "", nil)
-		} else {
-			a.condition.SetError(&newStatus, "", err)
-		}
-	}
-	if !equality.Semantic.DeepEqual(origStatus, &newStatus) {
-		if a.condition != "" {
-			// Since status has changed, update the lastUpdatedTime
-			a.condition.LastUpdated(&newStatus, time.Now().UTC().Format(time.RFC3339))
-		}
-
-		var newErr error
-		obj.Status = newStatus
-		newObj, newErr := a.client.UpdateStatus(obj)
-		if err == nil {
-			err = newErr
-		}
-		if newErr == nil {
-			obj = newObj
-		}
-	}
-	return obj, err
-}
-
-type eTCDSnapshotFileGeneratingHandler struct {
-	ETCDSnapshotFileGeneratingHandler
-	apply apply.Apply
-	opts  generic.GeneratingHandlerOptions
-	gvk   schema.GroupVersionKind
-	name  string
-}
-
-func (a *eTCDSnapshotFileGeneratingHandler) Remove(key string, obj *v1.ETCDSnapshotFile) (*v1.ETCDSnapshotFile, error) {
-	if obj != nil {
-		return obj, nil
-	}
-
-	obj = &v1.ETCDSnapshotFile{}
-	obj.Namespace, obj.Name = kv.RSplit(key, "/")
-	obj.SetGroupVersionKind(a.gvk)
-
-	return nil, generic.ConfigureApplyForObject(a.apply, obj, &a.opts).
-		WithOwner(obj).
-		WithSetID(a.name).
-		ApplyObjects()
-}
-
-func (a *eTCDSnapshotFileGeneratingHandler) Handle(obj *v1.ETCDSnapshotFile, status v1.ETCDSnapshotStatus) (v1.ETCDSnapshotStatus, error) {
-	if !obj.DeletionTimestamp.IsZero() {
-		return status, nil
-	}
-
-	objs, newStatus, err := a.ETCDSnapshotFileGeneratingHandler(obj, status)
-	if err != nil {
-		return newStatus, err
-	}
-
-	return newStatus, generic.ConfigureApplyForObject(a.apply, obj, &a.opts).
-		WithOwner(obj).
-		WithSetID(a.name).
-		ApplyObjects(objs...)
-}

pkg/generated/controllers/k3s.cattle.io/v1/interface.go

-/*
-Copyright The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by main. DO NOT EDIT.
-
-package v1
-
-import (
-	v1 "github.com/k3s-io/k3s/pkg/apis/k3s.cattle.io/v1"
-	"github.com/rancher/lasso/pkg/controller"
-	"github.com/rancher/wrangler/pkg/generic"
-	"github.com/rancher/wrangler/pkg/schemes"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-)
-
-func init() {
-	schemes.Register(v1.AddToScheme)
-}
-
-type Interface interface {
-	Addon() AddonController
-	ETCDSnapshotFile() ETCDSnapshotFileController
-}
-
-func New(controllerFactory controller.SharedControllerFactory) Interface {
-	return &version{
-		controllerFactory: controllerFactory,
-	}
-}
-
-type version struct {
-	controllerFactory controller.SharedControllerFactory
-}
-
-func (v *version) Addon() AddonController {
-	return generic.NewController[*v1.Addon, *v1.AddonList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "Addon"}, "addons", true, v.controllerFactory)
-}
-
-func (v *version) ETCDSnapshotFile() ETCDSnapshotFileController {
-	return generic.NewNonNamespacedController[*v1.ETCDSnapshotFile, *v1.ETCDSnapshotFileList](schema.GroupVersionKind{Group: "k3s.cattle.io", Version: "v1", Kind: "ETCDSnapshotFile"}, "etcdsnapshotfiles", v.controllerFactory)
-}

pkg/kubeadm/token.go

 package kubeadm
 
 import (
+	"errors"
+
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	bootstraputil "k8s.io/cluster-bootstrap/token/util"

pkg/kubeadm/utils.go

 package kubeadm
 
 import (
+	"fmt"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
@@ -32,7 +33,7 @@ func (bts BootstrapTokenString) String() string {
 func NewBootstrapTokenString(token string) (*BootstrapTokenString, error) {
 	substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token)
 	if len(substrs) != 3 {
-		return nil, errors.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
+		return nil, fmt.Errorf("the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
 	}
 
 	return &BootstrapTokenString{ID: substrs[1], Secret: substrs[2]}, nil
@@ -100,24 +101,24 @@ func BootstrapTokenFromSecret(secret *v1.Secret) (*BootstrapToken, error) {
 	// Get the Token ID field from the Secret data
 	tokenID := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenIDKey)
 	if len(tokenID) == 0 {
-		return nil, errors.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name)
+		return nil, fmt.Errorf("bootstrap Token Secret has no token-id data: %s", secret.Name)
 	}
 
 	// Enforce the right naming convention
 	if secret.Name != bootstraputil.BootstrapTokenSecretName(tokenID) {
-		return nil, errors.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q",
+		return nil, fmt.Errorf("bootstrap token name is not of the form '%s(token-id)'. Actual: %q. Expected: %q",
 			bootstrapapi.BootstrapTokenSecretPrefix, secret.Name, bootstraputil.BootstrapTokenSecretName(tokenID))
 	}
 
 	tokenSecret := bootstrapsecretutil.GetData(secret, bootstrapapi.BootstrapTokenSecretKey)
 	if len(tokenSecret) == 0 {
-		return nil, errors.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name)
+		return nil, fmt.Errorf("bootstrap Token Secret has no token-secret data: %s", secret.Name)
 	}
 
 	// Create the BootstrapTokenString object based on the ID and Secret
 	bts, err := NewBootstrapTokenStringFromIDAndSecret(tokenID, tokenSecret)
 	if err != nil {
-		return nil, errors.Wrap(err, "bootstrap Token Secret is invalid and couldn't be parsed")
+		return nil, pkgerrors.WithMessage(err, "bootstrap Token Secret is invalid and couldn't be parsed")
 	}
 
 	// Get the description (if any) from the Secret
@@ -130,7 +131,7 @@ func BootstrapTokenFromSecret(secret *v1.Secret) (*BootstrapToken, error) {
 	if len(secretExpiration) > 0 {
 		expTime, err := time.Parse(time.RFC3339, secretExpiration)
 		if err != nil {
-			return nil, errors.Wrapf(err, "can't parse expiration time of bootstrap token %q", secret.Name)
+			return nil, pkgerrors.WithMessagef(err, "can't parse expiration time of bootstrap token %q", secret.Name)
 		}
 		expires = &metav1.Time{Time: expTime}
 	}

pkg/metrics/metrics.go

@@ -8,6 +8,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/agent/https"
 	"github.com/k3s-io/k3s/pkg/agent/loadbalancer"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	lassometrics "github.com/rancher/lasso/pkg/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
@@ -35,6 +36,8 @@ func init() {
 	lassometrics.MustRegister(DefaultRegisterer)
 	// same for loadbalancer metrics
 	loadbalancer.MustRegister(DefaultRegisterer)
+	// and etcd snapshot metrics
+	etcd.MustRegister(DefaultRegisterer)
 }
 
 // Config holds fields for the metrics listener

pkg/node/controller.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/nodepassword"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	core "k8s.io/api/core/v1"
@@ -72,7 +72,7 @@ func (h *handler) updateHosts(node *core.Node, removed bool) (*core.Node, error)
 	}
 	if removed {
 		if err := h.removeNodePassword(nodeName); err != nil {
-			logrus.Warn(errors.Wrap(err, "Unable to remove node password"))
+			logrus.Warn(pkgerrors.WithMessage(err, "Unable to remove node password"))
 		}
 	}
 	if h.modCoreDNS {
@@ -99,7 +99,7 @@ func (h *handler) updateCoreDNSConfigMap(nodeName, hostName, nodeIPv4, nodeIPv6
 
 	configMap, err := h.configMaps.Get("kube-system", "coredns", metav1.GetOptions{})
 	if err != nil || configMap == nil {
-		logrus.Warn(errors.Wrap(err, "Unable to fetch coredns config map"))
+		logrus.Warn(pkgerrors.WithMessage(err, "Unable to fetch coredns config map"))
 		return nil
 	}
 

pkg/nodeconfig/nodeconfig.go

@@ -11,7 +11,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/configfilearg"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 )
 
@@ -45,7 +45,7 @@ func getNodeArgs() (string, error) {
 	}
 	nodeArgs, err := json.Marshal(nodeArgsList)
 	if err != nil {
-		return "", errors.Wrap(err, "Failed to retrieve argument list for node")
+		return "", pkgerrors.WithMessage(err, "Failed to retrieve argument list for node")
 	}
 	return string(nodeArgs), nil
 }
@@ -65,7 +65,7 @@ func getNodeEnv() (string, error) {
 	}
 	k3sEnvJSON, err := json.Marshal(k3sEnv)
 	if err != nil {
-		return "", errors.Wrap(err, "Failed to retrieve environment map for node")
+		return "", pkgerrors.WithMessage(err, "Failed to retrieve environment map for node")
 	}
 	return string(k3sEnvJSON), nil
 }

pkg/nodepassword/nodepassword.go

 package nodepassword
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -9,7 +10,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/authenticator/hash"
 	"github.com/k3s-io/k3s/pkg/passwd"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -122,7 +123,7 @@ func MigrateFile(secretClient coreclient.SecretController, nodeClient coreclient
 	for _, nodeName := range nodeNames {
 		if pass, ok := passwd.Pass(nodeName); ok {
 			if err := Ensure(secretClient, nodeName, pass); err != nil {
-				logrus.Warn(errors.Wrapf(err, "error migrating node password entry for node '%s'", nodeName))
+				logrus.Warn(pkgerrors.WithMessagef(err, "error migrating node password entry for node '%s'", nodeName))
 			} else {
 				ensured++
 			}

pkg/nodepassword/validate.go

@@ -2,6 +2,7 @@ package nodepassword
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"os"
 	"path"
@@ -13,7 +14,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	coreclient "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
@@ -149,16 +150,16 @@ func verifyLocalPassword(ctx context.Context, control *config.Control, mu *sync.
 
 	passBytes, err := os.ReadFile(nodePasswordFile)
 	if err != nil {
-		return "", http.StatusInternalServerError, errors.Wrap(err, "unable to read node password file")
+		return "", http.StatusInternalServerError, pkgerrors.WithMessage(err, "unable to read node password file")
 	}
 
 	passHash, err := Hasher.CreateHash(strings.TrimSpace(string(passBytes)))
 	if err != nil {
-		return "", http.StatusInternalServerError, errors.Wrap(err, "unable to hash node password file")
+		return "", http.StatusInternalServerError, pkgerrors.WithMessage(err, "unable to hash node password file")
 	}
 
 	if err := Hasher.VerifyHash(passHash, node.Password); err != nil {
-		return "", http.StatusForbidden, errors.Wrap(err, "unable to verify local node password")
+		return "", http.StatusForbidden, pkgerrors.WithMessage(err, "unable to verify local node password")
 	}
 
 	mu.Lock()
@@ -193,7 +194,7 @@ func verifyRemotePassword(ctx context.Context, control *config.Control, mu *sync
 func verifyNode(ctx context.Context, nodeClient coreclient.NodeController, node *nodeInfo) error {
 	if nodeName, isNodeAuth := identifier.NodeIdentity(node.User); isNodeAuth {
 		if _, err := nodeClient.Cache().Get(nodeName); err != nil {
-			return errors.Wrap(err, "unable to verify node identity")
+			return pkgerrors.WithMessage(err, "unable to verify node identity")
 		}
 	}
 	return nil

pkg/rootless/mounts.go

@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
@@ -41,7 +41,7 @@ func setupMounts(stateDir string) error {
 
 	for _, v := range mountMap {
 		if err := setupMount(v[0], v[1]); err != nil {
-			return errors.Wrapf(err, "failed to setup mount %s => %s", v[0], v[1])
+			return pkgerrors.WithMessagef(err, "failed to setup mount %s => %s", v[0], v[1])
 		}
 	}
 
@@ -75,16 +75,16 @@ func setupMount(target, dir string) error {
 	}
 
 	if err := os.MkdirAll(toCreate, 0700); err != nil {
-		return errors.Wrapf(err, "failed to create directory %s", toCreate)
+		return pkgerrors.WithMessagef(err, "failed to create directory %s", toCreate)
 	}
 
 	logrus.Debug("Mounting none ", toCreate, " tmpfs")
 	if err := unix.Mount("none", toCreate, "tmpfs", 0, ""); err != nil {
-		return errors.Wrapf(err, "failed to mount tmpfs to %s", toCreate)
+		return pkgerrors.WithMessagef(err, "failed to mount tmpfs to %s", toCreate)
 	}
 
 	if err := os.MkdirAll(target, 0700); err != nil {
-		return errors.Wrapf(err, "failed to create directory %s", target)
+		return pkgerrors.WithMessagef(err, "failed to create directory %s", target)
 	}
 
 	if dir == "" {
@@ -92,7 +92,7 @@ func setupMount(target, dir string) error {
 	}
 
 	if err := os.MkdirAll(dir, 0700); err != nil {
-		return errors.Wrapf(err, "failed to create directory %s", dir)
+		return pkgerrors.WithMessagef(err, "failed to create directory %s", dir)
 	}
 
 	logrus.Debug("Mounting ", dir, target, " none bind")

pkg/rootless/rootless.go

@@ -4,6 +4,8 @@
 package rootless
 
 import (
+	"errors"
+	"fmt"
 	"net"
 	"os"
 	"os/exec"
@@ -12,7 +14,7 @@ import (
 	"strings"
 
 	"github.com/opencontainers/runc/libcontainer/cgroups"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rootless-containers/rootlesskit/pkg/child"
 	"github.com/rootless-containers/rootlesskit/pkg/copyup/tmpfssymlink"
 	"github.com/rootless-containers/rootlesskit/pkg/network/slirp4netns"
@@ -97,7 +99,7 @@ func validateSysctl() error {
 	for key, expectedValue := range expected {
 		if actualValue, err := readSysctl(key); err == nil {
 			if expectedValue != actualValue {
-				return errors.Errorf("expected sysctl value %q to be %q, got %q; try adding \"%s=%s\" to /etc/sysctl.conf and running `sudo sysctl --system`",
+				return fmt.Errorf("expected sysctl value %q to be %q, got %q; try adding \"%s=%s\" to /etc/sysctl.conf and running `sudo sysctl --system`",
 					key, expectedValue, actualValue, key, expectedValue)
 			}
 		}
@@ -123,14 +125,14 @@ func parseCIDR(s string) (*net.IPNet, error) {
 		return nil, err
 	}
 	if !ip.Equal(ipnet.IP) {
-		return nil, errors.Errorf("cidr must be like 10.0.2.0/24, not like 10.0.2.100/24")
+		return nil, errors.New("host identifier bits must not be set in CIDR prefix")
 	}
 	return ipnet, nil
 }
 
 func createParentOpt(driver portDriver, stateDir string, enableIPv6 bool) (*parent.Opt, error) {
 	if err := os.MkdirAll(stateDir, 0755); err != nil {
-		return nil, errors.Wrapf(err, "failed to mkdir %s", stateDir)
+		return nil, pkgerrors.WithMessagef(err, "failed to mkdir %s", stateDir)
 	}
 
 	driver.SetStateDir(stateDir)

pkg/secretsencrypt/config.go

@@ -16,7 +16,7 @@ import (
 	"github.com/prometheus/common/expfmt"
 	corev1 "k8s.io/api/core/v1"
 
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
+	"github.com/k3s-io/api/pkg/generated/clientset/versioned/scheme"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"

pkg/server/auth/auth.go

 package auth
 
 import (
+	"errors"
 	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apiserver/pkg/endpoints/request"
 )

pkg/server/context.go

@@ -3,13 +3,13 @@ package server
 import (
 	"context"
 
+	"github.com/k3s-io/api/pkg/generated/controllers/k3s.cattle.io"
 	helmcrd "github.com/k3s-io/helm-controller/pkg/crd"
 	"github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io"
 	addoncrd "github.com/k3s-io/k3s/pkg/crd"
-	"github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/crd"
 	"github.com/rancher/wrangler/pkg/generated/controllers/apps"
 	"github.com/rancher/wrangler/pkg/generated/controllers/batch"
@@ -57,7 +57,7 @@ func NewContext(ctx context.Context, config *Config, forServer bool) (*Context,
 	if forServer {
 		recorder = util.BuildControllerEventRecorder(k8s, version.Program+"-supervisor", metav1.NamespaceAll)
 		if err := registerCrds(ctx, config, restConfig); err != nil {
-			return nil, errors.Wrap(err, "failed to register CRDs")
+			return nil, pkgerrors.WithMessage(err, "failed to register CRDs")
 		}
 	}
 

pkg/server/handlers/cert.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -21,7 +22,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/control/deps"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/sirupsen/logrus"
@@ -76,12 +77,12 @@ func caCertReplace(control *config.Control, buf io.ReadCloser, force bool) error
 	}
 
 	if err := defaultBootstrap(control, tmpControl); err != nil {
-		return errors.Wrap(err, "failed to set default bootstrap values")
+		return pkgerrors.WithMessage(err, "failed to set default bootstrap values")
 	}
 
 	if err := validateBootstrap(control, tmpControl); err != nil {
 		if !force {
-			return errors.Wrap(err, "failed to validate new CA certificates and keys")
+			return pkgerrors.WithMessage(err, "failed to validate new CA certificates and keys")
 		}
 		logrus.Warnf("Save of CA certificates and keys forced, ignoring validation errors: %v", err)
 	}
@@ -102,7 +103,7 @@ func defaultBootstrap(oldControl, newControl *config.Control) error {
 		newVal := newMeta.FieldByName(field.Name)
 		info, err := os.Stat(newVal.String())
 		if err != nil && !errors.Is(err, fs.ErrNotExist) {
-			errs = append(errs, errors.Wrap(err, field.Name))
+			errs = append(errs, pkgerrors.WithMessage(err, field.Name))
 			continue
 		}
 
@@ -140,19 +141,19 @@ func validateBootstrap(oldControl, newControl *config.Control) error {
 		// Check CA chain consistency and cert/key agreement
 		if strings.HasSuffix(field.Name, "CA") {
 			if err := validateCA(oldVal.String(), newVal.String()); err != nil {
-				errs = append(errs, errors.Wrap(err, field.Name))
+				errs = append(errs, pkgerrors.WithMessage(err, field.Name))
 			}
 			newKeyVal := newMeta.FieldByName(field.Name + "Key")
 			oldKeyVal := oldMeta.FieldByName(field.Name + "Key")
 			if err := validateCAKey(oldVal.String(), oldKeyVal.String(), newVal.String(), newKeyVal.String()); err != nil {
-				errs = append(errs, errors.Wrap(err, field.Name+"Key"))
+				errs = append(errs, pkgerrors.WithMessage(err, field.Name+"Key"))
 			}
 		}
 
 		// Check signing key rotation
 		if field.Name == "ServiceKey" {
 			if err := validateServiceKey(oldVal.String(), newVal.String()); err != nil {
-				errs = append(errs, errors.Wrap(err, field.Name))
+				errs = append(errs, pkgerrors.WithMessage(err, field.Name))
 			}
 		}
 	}
@@ -204,7 +205,7 @@ func validateCA(oldCAPath, newCAPath string) error {
 	// Verify the first cert in the bundle, using the combined roots and intermediates
 	_, err = newCerts[0].Verify(x509.VerifyOptions{Roots: roots, Intermediates: intermediates})
 	if err != nil {
-		err = errors.Wrap(err, "new CA cert cannot be verified using old CA chain")
+		err = pkgerrors.WithMessage(err, "new CA cert cannot be verified using old CA chain")
 	}
 	return err
 }
@@ -218,7 +219,7 @@ func validateCAKey(oldCAPath, oldCAKeyPath, newCAPath, newCAKeyPath string) erro
 
 	_, err := tls.LoadX509KeyPair(newCAPath, newCAKeyPath)
 	if err != nil {
-		err = errors.Wrap(err, "new CA cert and key cannot be loaded as X590KeyPair")
+		err = pkgerrors.WithMessage(err, "new CA cert and key cannot be loaded as X590KeyPair")
 	}
 	return err
 }

pkg/server/handlers/handlers.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -14,13 +15,12 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/k3s-io/k3s/pkg/bootstrap"
 	"github.com/k3s-io/k3s/pkg/cli/cmds"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/nodepassword"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -134,7 +134,7 @@ func File(fileName ...string) http.Handler {
 		for _, f := range fileName {
 			bytes, err := os.ReadFile(f)
 			if err != nil {
-				util.SendError(errors.Wrapf(err, "failed to read %s", f), resp, req, http.StatusInternalServerError)
+				util.SendError(pkgerrors.WithMessagef(err, "failed to read %s", f), resp, req, http.StatusInternalServerError)
 				return
 			}
 			resp.Write(bytes)
@@ -165,7 +165,7 @@ func APIServers(control *config.Control) http.Handler {
 		endpoints := collectAddresses(ctx)
 		resp.Header().Set("content-type", "application/json")
 		if err := json.NewEncoder(resp).Encode(endpoints); err != nil {
-			util.SendError(errors.Wrap(err, "failed to encode apiserver endpoints"), resp, req, http.StatusInternalServerError)
+			util.SendError(pkgerrors.WithMessage(err, "failed to encode apiserver endpoints"), resp, req, http.StatusInternalServerError)
 		}
 	})
 }
@@ -179,7 +179,7 @@ func Config(control *config.Control, cfg *cmds.Server) http.Handler {
 		control.DisableKubeProxy = cfg.DisableKubeProxy
 		resp.Header().Set("content-type", "application/json")
 		if err := json.NewEncoder(resp).Encode(control); err != nil {
-			util.SendError(errors.Wrap(err, "failed to encode agent config"), resp, req, http.StatusInternalServerError)
+			util.SendError(pkgerrors.WithMessage(err, "failed to encode agent config"), resp, req, http.StatusInternalServerError)
 		}
 	})
 }
@@ -199,8 +199,8 @@ func Readyz(control *config.Control) http.Handler {
 }
 
 func Bootstrap(control *config.Control) http.Handler {
-	if control.Runtime.HTTPBootstrap {
-		return bootstrap.Handler(&control.Runtime.ControlRuntimeBootstrap)
+	if control.Runtime.HTTPBootstrap != nil {
+		return control.Runtime.HTTPBootstrap
 	}
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		logrus.Warnf("Received HTTP bootstrap request from %s, but embedded etcd is not enabled.", req.RemoteAddr)

pkg/server/handlers/secrets-encrypt.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,7 +18,6 @@ import (
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/util"
-	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
@@ -79,6 +79,10 @@ func EncryptionStatus(control *config.Control) http.Handler {
 
 func encryptionStatus(control *config.Control) (EncryptionState, error) {
 	state := EncryptionState{}
+	if control.Runtime.Core == nil {
+		return state, util.ErrCoreNotReady
+	}
+
 	providers, err := secretsencrypt.GetEncryptionProviders(control.Runtime)
 	if os.IsNotExist(err) {
 		return state, nil

pkg/server/server.go

@@ -29,7 +29,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/util/permissions"
 	"github.com/k3s-io/k3s/pkg/version"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/apply"
 	v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/leader"
@@ -55,7 +55,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
 	}
 
 	if err := control.Server(ctx, &config.ControlConfig); err != nil {
-		return errors.Wrap(err, "starting kubernetes")
+		return pkgerrors.WithMessage(err, "starting kubernetes")
 	}
 
 	wg := &sync.WaitGroup{}
@@ -72,7 +72,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
 	}
 	for _, hook := range config.StartupHooks {
 		if err := hook(ctx, wg, shArgs); err != nil {
-			return errors.Wrap(err, "startup hook")
+			return pkgerrors.WithMessage(err, "startup hook")
 		}
 	}
 	go startOnAPIServerReady(ctx, config)
@@ -100,12 +100,12 @@ func runControllers(ctx context.Context, config *Config) error {
 
 	sc, err := NewContext(ctx, config, true)
 	if err != nil {
-		return errors.Wrap(err, "failed to create new server context")
+		return pkgerrors.WithMessage(err, "failed to create new server context")
 	}
 
 	controlConfig.Runtime.StartupHooksWg.Wait()
 	if err := stageFiles(ctx, sc, controlConfig); err != nil {
-		return errors.Wrap(err, "failed to stage files")
+		return pkgerrors.WithMessage(err, "failed to stage files")
 	}
 
 	// run migration before we set controlConfig.Runtime.Core
@@ -113,7 +113,7 @@ func runControllers(ctx context.Context, config *Config) error {
 		sc.Core.Core().V1().Secret(),
 		sc.Core.Core().V1().Node(),
 		controlConfig.Runtime.NodePasswdFile); err != nil {
-		logrus.Warn(errors.Wrap(err, "error migrating node-password file"))
+		logrus.Warn(pkgerrors.WithMessage(err, "error migrating node-password file"))
 	}
 	controlConfig.Runtime.K8s = sc.K8s
 	controlConfig.Runtime.K3s = sc.K3s
@@ -126,12 +126,12 @@ func runControllers(ctx context.Context, config *Config) error {
 
 	for _, controller := range config.Controllers {
 		if err := controller(ctx, sc); err != nil {
-			return errors.Wrapf(err, "failed to start %s controller", util.GetFunctionName(controller))
+			return pkgerrors.WithMessagef(err, "failed to start %s controller", util.GetFunctionName(controller))
 		}
 	}
 
 	if err := sc.Start(ctx); err != nil {
-		return errors.Wrap(err, "failed to start wranger controllers")
+		return pkgerrors.WithMessage(err, "failed to start wranger controllers")
 	}
 
 	if !controlConfig.DisableAPIServer {
@@ -165,14 +165,14 @@ func apiserverControllers(ctx context.Context, sc *Context, config *Config) {
 	}
 	for _, controller := range config.LeaderControllers {
 		if err := controller(ctx, sc); err != nil {
-			panic(errors.Wrapf(err, "failed to start %s leader controller", util.GetFunctionName(controller)))
+			panic(pkgerrors.WithMessagef(err, "failed to start %s leader controller", util.GetFunctionName(controller)))
 		}
 	}
 
 	// Re-run informer factory startup after core and leader-elected controllers have started.
 	// Additional caches may need to start for the newly added OnChange/OnRemove callbacks.
 	if err := sc.Start(ctx); err != nil {
-		panic(errors.Wrap(err, "failed to start wranger controllers"))
+		panic(pkgerrors.WithMessage(err, "failed to start wranger controllers"))
 	}
 }
 
@@ -500,11 +500,11 @@ func setupDataDirAndChdir(config *config.Control) error {
 	dataDir := config.DataDir
 
 	if err := os.MkdirAll(dataDir, 0700); err != nil {
-		return errors.Wrapf(err, "can not mkdir %s", dataDir)
+		return pkgerrors.WithMessagef(err, "can not mkdir %s", dataDir)
 	}
 
 	if err := os.Chdir(dataDir); err != nil {
-		return errors.Wrapf(err, "can not chdir %s", dataDir)
+		return pkgerrors.WithMessagef(err, "can not chdir %s", dataDir)
 	}
 
 	return nil

pkg/spegel/bootstrap.go

@@ -3,6 +3,7 @@ package spegel
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,7 +14,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/libp2p/go-libp2p/core/peer"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/sirupsen/logrus"
 	"github.com/spegel-org/spegel/pkg/routing"
@@ -81,14 +82,14 @@ func (c *agentBootstrapper) Run(ctx context.Context, id string) error {
 		withCert := clientaccess.WithClientCertificate(c.clientCert, c.clientKey)
 		info, err := clientaccess.ParseAndValidateToken(c.server, c.token, withCert)
 		if err != nil {
-			return errors.Wrap(err, "failed to validate join token")
+			return pkgerrors.WithMessage(err, "failed to validate join token")
 		}
 		c.info = info
 	}
 
 	client, err := util.GetClientSet(c.kubeConfig)
 	if err != nil {
-		return errors.Wrap(err, "failed to create kubernetes client")
+		return pkgerrors.WithMessage(err, "failed to create kubernetes client")
 	}
 	nodes := client.CoreV1().Nodes()
 

pkg/spegel/spegel.go

@@ -3,6 +3,7 @@ package spegel
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
 	"net"
@@ -30,7 +31,7 @@ import (
 	"github.com/libp2p/go-libp2p"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoreds"
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spegel-org/spegel/pkg/metrics"
 	"github.com/spegel-org/spegel/pkg/oci"
@@ -141,33 +142,33 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 	ociOpts := []oci.Option{oci.WithContentPath(filepath.Join(nodeConfig.Containerd.Root, "io.containerd.content.v1.content"))}
 	ociClient, err := oci.NewContainerd(nodeConfig.Containerd.Address, registryNamespace, nodeConfig.Containerd.Registry, urls, ociOpts...)
 	if err != nil {
-		return errors.Wrap(err, "failed to create OCI client")
+		return pkgerrors.WithMessage(err, "failed to create OCI client")
 	}
 
 	// create or load persistent private key
 	keyFile := filepath.Join(nodeConfig.Containerd.Opt, "peer.key")
 	keyBytes, _, err := cert.LoadOrGenerateKeyFile(keyFile, false)
 	if err != nil {
-		return errors.Wrap(err, "failed to load or generate p2p private key")
+		return pkgerrors.WithMessage(err, "failed to load or generate p2p private key")
 	}
 	privKey, err := cert.ParsePrivateKeyPEM(keyBytes)
 	if err != nil {
-		return errors.Wrap(err, "failed to parse p2p private key")
+		return pkgerrors.WithMessage(err, "failed to parse p2p private key")
 	}
 	p2pKey, _, err := crypto.KeyPairFromStdKey(privKey)
 	if err != nil {
-		return errors.Wrap(err, "failed to convert p2p private key")
+		return pkgerrors.WithMessage(err, "failed to convert p2p private key")
 	}
 
 	// create a peerstore to allow persisting nodes across restarts
 	peerFile := filepath.Join(nodeConfig.Containerd.Opt, "peerstore.db")
 	ds, err := leveldb.NewDatastore(peerFile, nil)
 	if err != nil {
-		return errors.Wrap(err, "failed to create peerstore datastore")
+		return pkgerrors.WithMessage(err, "failed to create peerstore datastore")
 	}
 	ps, err := pstoreds.NewPeerstore(ctx, ds, pstoreds.DefaultOpts())
 	if err != nil {
-		return errors.Wrap(err, "failed to create peerstore")
+		return pkgerrors.WithMessage(err, "failed to create peerstore")
 	}
 
 	// get latest tag configuration override
@@ -198,13 +199,13 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 	}
 	router, err := routing.NewP2PRouter(ctx, routerAddr, c.Bootstrapper, c.RegistryPort, opts...)
 	if err != nil {
-		return errors.Wrap(err, "failed to create P2P router")
+		return pkgerrors.WithMessage(err, "failed to create P2P router")
 	}
 	go router.Run(ctx)
 
 	caCert, err := os.ReadFile(c.ServerCAFile)
 	if err != nil {
-		return errors.Wrap(err, "failed to read server CA")
+		return pkgerrors.WithMessage(err, "failed to read server CA")
 	}
 	client := clientaccess.GetHTTPClient(caCert, c.ClientCertFile, c.ClientKeyFile)
 	metrics.Register()
@@ -219,7 +220,7 @@ func (c *Config) Start(ctx context.Context, nodeConfig *config.Node) error {
 	reg := registry.NewRegistry(ociClient, router, registryOpts...)
 	regSvr, err := reg.Server(":" + c.RegistryPort)
 	if err != nil {
-		return errors.Wrap(err, "failed to create embedded registry server")
+		return pkgerrors.WithMessage(err, "failed to create embedded registry server")
 	}
 
 	// Track images available in containerd and publish via p2p router

pkg/static/stage.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -20,7 +20,7 @@ func Stage(dataDir string) error {
 		logrus.Info("Writing static file: ", p)
 		os.MkdirAll(filepath.Dir(p), 0700)
 		if err := os.WriteFile(p, content, 0600); err != nil {
-			return errors.Wrapf(err, "failed to write to %s", name)
+			return pkgerrors.WithMessagef(err, "failed to write to %s", name)
 		}
 	}
 

pkg/util/api.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -9,7 +10,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/merr"
 	"github.com/rancher/wrangler/pkg/schemes"
 	"github.com/sirupsen/logrus"
@@ -83,7 +84,7 @@ func WaitForAPIServerReady(ctx context.Context, kubeconfigPath string, timeout t
 		healthStatus := 0
 		result := restClient.Get().AbsPath("/readyz").Do(ctx).StatusCode(&healthStatus)
 		if rerr := result.Error(); rerr != nil {
-			lastErr = errors.Wrap(rerr, "failed to get apiserver /readyz status")
+			lastErr = pkgerrors.WithMessage(rerr, "failed to get apiserver /readyz status")
 			return false, nil
 		}
 		if healthStatus != http.StatusOK {

pkg/util/apierrors.go

@@ -2,12 +2,12 @@ package util
 
 import (
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"math/big"
 	"net/http"
 
-	"github.com/k3s-io/k3s/pkg/generated/clientset/versioned/scheme"
-	"github.com/pkg/errors"
+	"github.com/k3s-io/api/pkg/generated/clientset/versioned/scheme"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

pkg/util/file.go

 package util
 
 import (
+	"errors"
 	"os"
 	"os/user"
 	"path/filepath"
@@ -8,7 +9,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 

pkg/util/metrics/metrics.go

+package metrics
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func ObserveWithStatus(vec *prometheus.HistogramVec, start time.Time, err error, labels ...string) {
+	status := "success"
+	if err != nil {
+		status = "error"
+	}
+	labels = append(labels, status)
+	vec.WithLabelValues(labels...).Observe(time.Since(start).Seconds())
+}

pkg/util/permissions/permissions_windows.go

@@ -6,7 +6,7 @@ package permissions
 import (
 	"fmt"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"golang.org/x/sys/windows"
 )
 
@@ -27,7 +27,7 @@ func IsPrivileged() error {
 		0, 0, 0, 0, 0, 0,
 		&sid)
 	if err != nil {
-		return errors.Wrap(err, "failed to create Windows SID")
+		return pkgerrors.WithMessage(err, "failed to create Windows SID")
 	}
 	defer windows.FreeSid(sid)
 
@@ -36,7 +36,7 @@ func IsPrivileged() error {
 
 	member, err := token.IsMember(sid)
 	if err != nil {
-		return errors.Wrap(err, "failed to check group membership")
+		return pkgerrors.WithMessage(err, "failed to check group membership")
 	}
 
 	if !member {

pkg/util/token.go

@@ -5,12 +5,12 @@ import (
 	cryptorand "crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
+	"errors"
 	"os"
 	"path/filepath"
 
 	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
-	"github.com/pkg/errors"
 )
 
 func Random(size int) (string, error) {

pkg/vpn/vpn.go

@@ -2,6 +2,7 @@ package vpn
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"net/url"
@@ -9,7 +10,7 @@ import (
 
 	"github.com/k3s-io/k3s/pkg/util"
 
-	"github.com/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -60,7 +61,7 @@ func StartVPN(vpnAuthConfigFile string) error {
 		logrus.Debugf("Flags passed to tailscale up: %v", args)
 		output, err := util.ExecCommand("tailscale", args)
 		if err != nil {
-			return errors.Wrap(err, "tailscale up failed: "+output)
+			return pkgerrors.WithMessage(err, "tailscale up failed: "+output)
 		}
 		logrus.Debugf("Output from tailscale up: %v", output)
 		return nil

scripts/download

@@ -24,7 +24,7 @@ mkdir -p ${DATA_DIR}
 
 case ${OS} in
   linux)
-    git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/k3s-io/runc ${RUNC_DIR}
+    git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR}
     curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf -
     ;;
   windows)

scripts/test-mods

 #!/bin/bash
 set -e -x
 
-res=$(go mod edit --json | jq -r '.Replace[] | select(.Old.Path | contains("k8s.io/")) | .New.Path' | grep -vE '^(k8s.io/|github.com/k3s-io/)' | wc -l)
+res=$(go mod edit --json | jq -r '.Replace[] | select(.Old.Path | startswith("k8s.io/")) | .New.Path' | grep -vE '^(k8s.io/|github.com/k3s-io/)' | wc -l)
 if [ $res -gt 0 ];then
   echo "Incorrect kubernetes replacement fork in go.mod"
   exit 1

tests/e2e/rootless/rootless_test.go

@@ -111,7 +111,7 @@ var _ = Describe("Various Startup Configurations", Ordered, func() {
 			_, _ = e2e.ParseNodes(tc.KubeConfigFile, false)
 
 			Eventually(func() error {
-				return tests.AllPodsUp(tc.KubeConfigFile)
+				return tests.CheckDefaultDeployments(tc.KubeConfigFile)
 			}, "360s", "5s").Should(Succeed())
 			e2e.DumpPods(tc.KubeConfigFile)
 		})