Switch to using kubelet config files instead of CLI args

go.mod

@@ -164,6 +164,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kube-proxy v0.0.0
 	k8s.io/kubectl v0.32.0
+	k8s.io/kubelet v0.32.0
 	k8s.io/kubernetes v1.32.0
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
 	sigs.k8s.io/cri-tools v0.0.0-00010101000000-000000000000
@@ -478,7 +479,6 @@ require (
 	k8s.io/kube-controller-manager v0.0.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20241127205056-99599406b04f // indirect
 	k8s.io/kube-scheduler v0.0.0 // indirect
-	k8s.io/kubelet v0.32.0 // indirect
 	k8s.io/metrics v0.0.0 // indirect
 	k8s.io/mount-utils v0.32.0 // indirect
 	k8s.io/pod-security-admission v0.0.0 // indirect

pkg/agent/config/config.go

@@ -543,6 +543,12 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 		return nil, err
 	}
 
+	// Ensure kubelet config dir exists
+	kubeletConfigDir := filepath.Join(envInfo.DataDir, "agent", "etc", "kubelet.conf.d")
+	if err := os.MkdirAll(kubeletConfigDir, 0700); err != nil {
+		return nil, err
+	}
+
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
 		SELinux:                  envInfo.EnableSELinux,
@@ -571,6 +577,7 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain
 	nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo)
 	nodeConfig.AgentConfig.ClientCA = clientCAFile
+	nodeConfig.AgentConfig.KubeletConfigDir = kubeletConfigDir
 	nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.KubeConfigK3sController = kubeconfigK3sController

pkg/daemons/agent/agent.go

@@ -2,19 +2,34 @@ package agent
 
 import (
 	"context"
+	"fmt"
 	"math/rand"
+	"net"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/agent/config"
 	"github.com/k3s-io/k3s/pkg/agent/proxy"
+	"github.com/k3s-io/k3s/pkg/agent/util"
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/daemons/executor"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/otiai10/copy"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/component-base/logs"
 	logsapi "k8s.io/component-base/logs/api/v1"
+	logsv1 "k8s.io/component-base/logs/api/v1"
 	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
 	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
+	kubeletconfig "k8s.io/kubelet/config/v1beta1"
+	"k8s.io/kubernetes/pkg/util/taints"
+	utilsnet "k8s.io/utils/net"
+	utilsptr "k8s.io/utils/ptr"
+	"sigs.k8s.io/yaml"
 )
 
 func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy) error {
@@ -24,7 +39,7 @@ func Agent(ctx context.Context, nodeConfig *daemonconfig.Node, proxy proxy.Proxy
 	defer logs.FlushLogs()
 
 	if err := startKubelet(ctx, &nodeConfig.AgentConfig); err != nil {
-		return err
+		return errors.Wrap(err, "failed to start kubelet")
 	}
 
 	go func() {
@@ -46,9 +61,21 @@ func startKubeProxy(ctx context.Context, cfg *daemonconfig.Agent) error {
 }
 
 func startKubelet(ctx context.Context, cfg *daemonconfig.Agent) error {
-	argsMap := kubeletArgs(cfg)
+	argsMap, defaultConfig, err := kubeletArgsAndConfig(cfg)
+	if err != nil {
+		return errors.Wrap(err, "prepare default configuration drop-in")
+	}
+
+	extraArgs, err := extractConfigArgs(cfg.KubeletConfigDir, cfg.ExtraKubeletArgs, defaultConfig)
+	if err != nil {
+		return errors.Wrap(err, "prepare user configuration drop-ins")
+	}
+
+	if err := writeKubeletConfig(cfg.KubeletConfigDir, defaultConfig); err != nil {
+		return errors.Wrap(err, "generate default kubelet configuration drop-in")
+	}
 
-	args := daemonconfig.GetArgs(argsMap, cfg.ExtraKubeletArgs)
+	args := daemonconfig.GetArgs(argsMap, extraArgs)
 	logrus.Infof("Running kubelet %s", daemonconfig.ArgString(args))
 
 	return executor.Kubelet(ctx, args)
@@ -67,3 +94,170 @@ func ImageCredProvAvailable(cfg *daemonconfig.Agent) bool {
 	}
 	return true
 }
+
+// extractConfigArgs strips out any --config or --config-dir flags from the
+// provided args list, and if set, copies the content of the file or dir into
+// the target drop-in directory.
+func extractConfigArgs(path string, extraArgs []string, config *kubeletconfig.KubeletConfiguration) ([]string, error) {
+	args := make([]string, 0, len(extraArgs))
+	strippedArgs := map[string]string{}
+	var skipVal bool
+	for i := range extraArgs {
+		if skipVal {
+			skipVal = false
+			continue
+		}
+
+		var val string
+		key := strings.TrimPrefix(extraArgs[i], "--")
+		if k, v, ok := strings.Cut(key, "="); ok {
+			// key=val pair
+			key = k
+			val = v
+		} else if len(extraArgs) > i+1 {
+			// key in this arg, value in next arg
+			val = extraArgs[i+1]
+			skipVal = true
+		}
+
+		switch key {
+		case "config", "config-dir":
+			if val == "" {
+				return nil, fmt.Errorf("value required for kubelet-arg --%s", key)
+			}
+			strippedArgs[key] = val
+		default:
+			args = append(args, extraArgs[i])
+		}
+	}
+
+	// copy the config file into our managed config dir, unless its already in there
+	if strippedArgs["config"] != "" && !strings.HasPrefix(strippedArgs["config"], path) {
+		src := strippedArgs["config"]
+		dest := filepath.Join(path, "10-cli-config.conf")
+		if err := util.CopyFile(src, dest, false); err != nil {
+			return nil, errors.Wrapf(err, "copy config %q into managed drop-in dir %q", src, dest)
+		}
+	}
+	// copy the config-dir into our managed config dir, unless its already in there
+	if strippedArgs["config-dir"] != "" && !strings.HasPrefix(strippedArgs["config-dir"], path) {
+		src := strippedArgs["config-dir"]
+		dest := filepath.Join(path, "20-cli-config-dir")
+		if err := copy.Copy(src, dest, copy.Options{PreserveOwner: true}); err != nil {
+			return nil, errors.Wrapf(err, "copy config-dir %q into managed drop-in dir %q", src, dest)
+		}
+	}
+	return args, nil
+}
+
+// writeKubeletConfig marshals the provided KubeletConfiguration object into a
+// drop-in config file in the target drop-in directory.
+func writeKubeletConfig(path string, config *kubeletconfig.KubeletConfiguration) error {
+	b, err := yaml.Marshal(config)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(path, "00-"+version.Program+"-defaults.conf"), b, 0600)
+}
+
+func defaultKubeletConfig(cfg *daemonconfig.Agent) (*kubeletconfig.KubeletConfiguration, error) {
+	bindAddress := "127.0.0.1"
+	isIPv6 := utilsnet.IsIPv6(net.ParseIP([]string{cfg.NodeIP}[0]))
+	if isIPv6 {
+		bindAddress = "::1"
+	}
+
+	defaultConfig := &kubeletconfig.KubeletConfiguration{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "kubelet.config.k8s.io/v1beta1",
+			Kind:       "KubeletConfiguration",
+		},
+		CPUManagerReconcilePeriod:        metav1.Duration{Duration: time.Second * 10},
+		CgroupDriver:                     "cgroupfs",
+		ClusterDomain:                    cfg.ClusterDomain,
+		EvictionPressureTransitionPeriod: metav1.Duration{Duration: time.Minute * 5},
+		FailSwapOn:                       utilsptr.To(false),
+		FileCheckFrequency:               metav1.Duration{Duration: time.Second * 20},
+		HTTPCheckFrequency:               metav1.Duration{Duration: time.Second * 20},
+		HealthzBindAddress:               bindAddress,
+		ImageMinimumGCAge:                metav1.Duration{Duration: time.Minute * 2},
+		NodeStatusReportFrequency:        metav1.Duration{Duration: time.Minute * 5},
+		NodeStatusUpdateFrequency:        metav1.Duration{Duration: time.Second * 10},
+		ProtectKernelDefaults:            cfg.ProtectKernelDefaults,
+		ReadOnlyPort:                     0,
+		RuntimeRequestTimeout:            metav1.Duration{Duration: time.Minute * 2},
+		StreamingConnectionIdleTimeout:   metav1.Duration{Duration: time.Hour * 4},
+		SyncFrequency:                    metav1.Duration{Duration: time.Minute},
+		VolumeStatsAggPeriod:             metav1.Duration{Duration: time.Minute},
+		EvictionHard: map[string]string{
+			"imagefs.available": "5%",
+			"nodefs.available":  "5%",
+		},
+		EvictionMinimumReclaim: map[string]string{
+			"imagefs.available": "10%",
+			"nodefs.available":  "10%",
+		},
+		Authentication: kubeletconfig.KubeletAuthentication{
+			Anonymous: kubeletconfig.KubeletAnonymousAuthentication{
+				Enabled: utilsptr.To(false),
+			},
+			Webhook: kubeletconfig.KubeletWebhookAuthentication{
+				Enabled:  utilsptr.To(true),
+				CacheTTL: metav1.Duration{Duration: time.Minute * 2},
+			},
+		},
+		Authorization: kubeletconfig.KubeletAuthorization{
+			Mode: kubeletconfig.KubeletAuthorizationModeWebhook,
+			Webhook: kubeletconfig.KubeletWebhookAuthorization{
+				CacheAuthorizedTTL:   metav1.Duration{Duration: time.Minute * 5},
+				CacheUnauthorizedTTL: metav1.Duration{Duration: time.Second * 30},
+			},
+		},
+		Logging: logsv1.LoggingConfiguration{
+			Format:    "text",
+			Verbosity: logsv1.VerbosityLevel(cfg.VLevel),
+			FlushFrequency: logsv1.TimeOrMetaDuration{
+				Duration:          metav1.Duration{Duration: time.Second * 5},
+				SerializeAsString: true,
+			},
+		},
+	}
+
+	if cfg.ListenAddress != "" {
+		defaultConfig.Address = cfg.ListenAddress
+	}
+
+	if cfg.ClientCA != "" {
+		defaultConfig.Authentication.X509.ClientCAFile = cfg.ClientCA
+	}
+
+	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
+		defaultConfig.TLSCertFile = cfg.ServingKubeletCert
+		defaultConfig.TLSPrivateKeyFile = cfg.ServingKubeletKey
+	}
+
+	for _, addr := range cfg.ClusterDNSs {
+		defaultConfig.ClusterDNS = append(defaultConfig.ClusterDNS, addr.String())
+	}
+
+	if cfg.ResolvConf != "" {
+		defaultConfig.ResolverConfig = utilsptr.To(cfg.ResolvConf)
+	}
+
+	if cfg.PodManifests != "" && defaultConfig.StaticPodPath == "" {
+		defaultConfig.StaticPodPath = cfg.PodManifests
+	}
+	if err := os.MkdirAll(defaultConfig.StaticPodPath, 0750); err != nil {
+		return nil, errors.Wrapf(err, "failed to create static pod manifest dir %s", defaultConfig.StaticPodPath)
+	}
+
+	if t, _, err := taints.ParseTaints(cfg.NodeTaints); err != nil {
+		return nil, errors.Wrap(err, "failed to parse node taints")
+	} else {
+		defaultConfig.RegisterWithTaints = t
+	}
+
+	logsv1.VModuleConfigurationPflag(&defaultConfig.Logging.VModule).Set(cfg.VModule)
+
+	return defaultConfig, nil
+}

pkg/daemons/agent/agent_linux.go

@@ -5,7 +5,6 @@ package agent
 
 import (
 	"net"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -13,23 +12,25 @@ import (
 	"github.com/k3s-io/k3s/pkg/cgroups"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
-	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	kubeletconfig "k8s.io/kubelet/config/v1beta1"
 	utilsnet "k8s.io/utils/net"
+	utilsptr "k8s.io/utils/ptr"
 )
 
 const socketPrefix = "unix://"
 
-func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) {
+func createRootlessConfig(argsMap map[string]string, controllers map[string]bool) error {
 	argsMap["feature-gates=KubeletInUserNamespace"] = "true"
 	// "/sys/fs/cgroup" is namespaced
 	cgroupfsWritable := unix.Access("/sys/fs/cgroup", unix.W_OK) == nil
 	if controllers["cpu"] && controllers["pids"] && cgroupfsWritable {
 		logrus.Info("cgroup v2 controllers are delegated for rootless.")
-		return
+		return nil
 	}
-	logrus.Fatal("delegated cgroup v2 controllers are required for rootless.")
+	return errors.New("delegated cgroup v2 controllers are required for rootless")
 }
 
 func kubeProxyArgs(cfg *config.Agent) map[string]string {
@@ -64,73 +65,49 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
 	return argsMap
 }
 
-func kubeletArgs(cfg *config.Agent) map[string]string {
-	bindAddress := "127.0.0.1"
-	if utilsnet.IsIPv6(net.ParseIP(cfg.NodeIP)) {
-		bindAddress = "::1"
+// kubeletArgsAndConfig generates default kubelet args and configuration.
+// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them,
+// and a structured configuration file that upstream does not provide a convienent way to initailize with default values.
+// The defaults and our desired config also vary by OS.
+func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) {
+	defaultConfig, err := defaultKubeletConfig(cfg)
+	if err != nil {
+		return nil, nil, err
 	}
 	argsMap := map[string]string{
-		"healthz-bind-address":         bindAddress,
-		"read-only-port":               "0",
-		"cluster-domain":               cfg.ClusterDomain,
+		"config-dir": cfg.KubeletConfigDir,
 		"kubeconfig": cfg.KubeConfigKubelet,
-		"eviction-hard":                "imagefs.available<5%,nodefs.available<5%",
-		"eviction-minimum-reclaim":     "imagefs.available=10%,nodefs.available=10%",
-		"fail-swap-on":                 "false",
-		"cgroup-driver":                "cgroupfs",
-		"authentication-token-webhook": "true",
-		"anonymous-auth":               "false",
-		"authorization-mode":           modes.ModeWebhook,
-	}
-	if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
-		argsMap["pod-manifest-path"] = cfg.PodManifests
-	}
-	if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
-		logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
 	}
+
 	if cfg.RootDir != "" {
 		argsMap["root-dir"] = cfg.RootDir
 		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
 	}
-	if len(cfg.ClusterDNS) > 0 {
-		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
-	}
-	if cfg.ResolvConf != "" {
-		argsMap["resolv-conf"] = cfg.ResolvConf
-	}
 	if cfg.RuntimeSocket != "" {
-		argsMap["serialize-image-pulls"] = "false"
+		defaultConfig.SerializeImagePulls = utilsptr.To(false)
+		// note: this is a legacy cadvisor flag that the kubelet still exposes, but
+		// it must be set in order for cadvisor to pull stats properly.
 		if strings.Contains(cfg.RuntimeSocket, "containerd") {
 			argsMap["containerd"] = cfg.RuntimeSocket
 		}
 		// cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix
 		if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) {
-			argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
+			defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket
 		} else {
-			argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
+			defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket
 		}
 	}
 	if cfg.ImageServiceSocket != "" {
 		if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
-			argsMap["image-service-endpoint"] = cfg.ImageServiceSocket
+			defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket
 		} else {
-			argsMap["image-service-endpoint"] = socketPrefix + cfg.ImageServiceSocket
+			defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket
 		}
 	}
-	if cfg.ListenAddress != "" {
-		argsMap["address"] = cfg.ListenAddress
-	}
-	if cfg.ClientCA != "" {
-		argsMap["anonymous-auth"] = "false"
-		argsMap["client-ca-file"] = cfg.ClientCA
-	}
-	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
-		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
-		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
-	}
 	if cfg.NodeName != "" {
 		argsMap["hostname-override"] = cfg.NodeName
 	}
+
 	// If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe.
 	// When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly.
 	// This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet
@@ -141,33 +118,28 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 			argsMap["node-ip"] = cfg.NodeIP
 		}
 	} else {
+		argsMap["cloud-provider"] = "external"
 		if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" {
 			argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs)
 		}
 	}
+
 	kubeletRoot, runtimeRoot, controllers := cgroups.CheckCgroups()
+	if !controllers["pids"] {
+		return nil, nil, errors.New("pids cgroup controller not found")
+	}
 	if !controllers["cpu"] {
 		logrus.Warn("Disabling CPU quotas due to missing cpu controller or cpu.cfs_period_us")
-		argsMap["cpu-cfs-quota"] = "false"
-	}
-	if !controllers["pids"] {
-		logrus.Fatal("pids cgroup controller not found")
+		defaultConfig.CPUCFSQuota = utilsptr.To(false)
 	}
 	if kubeletRoot != "" {
-		argsMap["kubelet-cgroups"] = kubeletRoot
+		defaultConfig.KubeletCgroups = kubeletRoot
 	}
 	if runtimeRoot != "" {
 		argsMap["runtime-cgroups"] = runtimeRoot
 	}
 
 	argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
-	if len(cfg.NodeTaints) > 0 {
-		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
-	}
-
-	if !cfg.DisableCCM {
-		argsMap["cloud-provider"] = "external"
-	}
 
 	if ImageCredProvAvailable(cfg) {
 		logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
@@ -176,25 +148,18 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 	}
 
 	if cfg.Rootless {
-		createRootlessConfig(argsMap, controllers)
+		if err := createRootlessConfig(argsMap, controllers); err != nil {
+			return nil, nil, err
 		}
-
-	if cfg.Systemd {
-		argsMap["cgroup-driver"] = "systemd"
 	}
 
-	if cfg.ProtectKernelDefaults {
-		argsMap["protect-kernel-defaults"] = "true"
+	if cfg.Systemd {
+		defaultConfig.CgroupDriver = "systemd"
 	}
 
 	if !cfg.DisableServiceLB {
-		argsMap["allowed-unsafe-sysctls"] = "net.ipv4.ip_forward,net.ipv6.conf.all.forwarding"
+		defaultConfig.AllowedUnsafeSysctls = []string{"net.ipv4.ip_forward", "net.ipv6.conf.all.forwarding"}
 	}
-	if cfg.VLevel != 0 {
-		argsMap["v"] = strconv.Itoa(cfg.VLevel)
-	}
-	if cfg.VModule != "" {
-		argsMap["vmodule"] = cfg.VModule
-	}
-	return argsMap
+
+	return argsMap, defaultConfig, nil
 }

pkg/daemons/agent/agent_windows.go

@@ -5,15 +5,15 @@ package agent
 
 import (
 	"net"
-	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/sirupsen/logrus"
-	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	kubeletconfig "k8s.io/kubelet/config/v1beta1"
 	utilsnet "k8s.io/utils/net"
+	utilsptr "k8s.io/utils/ptr"
 )
 
 const (
@@ -38,64 +38,42 @@ func kubeProxyArgs(cfg *config.Agent) map[string]string {
 	return argsMap
 }
 
-func kubeletArgs(cfg *config.Agent) map[string]string {
-	bindAddress := "127.0.0.1"
-	_, IPv6only, _ := util.GetFirstString([]string{cfg.NodeIP})
-	if IPv6only {
-		bindAddress = "::1"
+// kubeletArgsAndConfig generates default kubelet args and configuration.
+// Kubelet config is frustratingly split across deprecated CLI flags that raise warnings if you use them,
+// and a structured configuration file that upstream does not provide a convienent way to initailize with default values.
+// The defaults and our desired config also vary by OS.
+func kubeletArgsAndConfig(cfg *config.Agent) (map[string]string, *kubeletconfig.KubeletConfiguration, error) {
+	defaultConfig, err := defaultKubeletConfig(cfg)
+	if err != nil {
+		return nil, nil, err
 	}
 	argsMap := map[string]string{
-		"healthz-bind-address":         bindAddress,
-		"read-only-port":               "0",
-		"cluster-domain":               cfg.ClusterDomain,
+		"config-dir": cfg.KubeletConfigDir,
 		"kubeconfig": cfg.KubeConfigKubelet,
-		"eviction-hard":                "imagefs.available<5%,nodefs.available<5%",
-		"eviction-minimum-reclaim":     "imagefs.available=10%,nodefs.available=10%",
-		"fail-swap-on":                 "false",
-		"authentication-token-webhook": "true",
-		"anonymous-auth":               "false",
-		"authorization-mode":           modes.ModeWebhook,
-	}
-	if cfg.PodManifests != "" && argsMap["pod-manifest-path"] == "" {
-		argsMap["pod-manifest-path"] = cfg.PodManifests
-	}
-	if err := os.MkdirAll(argsMap["pod-manifest-path"], 0755); err != nil {
-		logrus.Errorf("Failed to mkdir %s: %v", argsMap["pod-manifest-path"], err)
 	}
 	if cfg.RootDir != "" {
 		argsMap["root-dir"] = cfg.RootDir
 		argsMap["cert-dir"] = filepath.Join(cfg.RootDir, "pki")
 	}
-	if len(cfg.ClusterDNS) > 0 {
-		argsMap["cluster-dns"] = util.JoinIPs(cfg.ClusterDNSs)
-	}
-	if cfg.ResolvConf != "" {
-		argsMap["resolv-conf"] = cfg.ResolvConf
-	}
 	if cfg.RuntimeSocket != "" {
-		argsMap["serialize-image-pulls"] = "false"
+		defaultConfig.SerializeImagePulls = utilsptr.To(false)
 		// cadvisor wants the containerd CRI socket without the prefix, but kubelet wants it with the prefix
 		if strings.HasPrefix(cfg.RuntimeSocket, socketPrefix) {
-			argsMap["container-runtime-endpoint"] = cfg.RuntimeSocket
+			defaultConfig.ContainerRuntimeEndpoint = cfg.RuntimeSocket
 		} else {
-			argsMap["container-runtime-endpoint"] = socketPrefix + cfg.RuntimeSocket
+			defaultConfig.ContainerRuntimeEndpoint = socketPrefix + cfg.RuntimeSocket
 		}
 	}
-	if cfg.ListenAddress != "" {
-		argsMap["address"] = cfg.ListenAddress
-	}
-	if cfg.ClientCA != "" {
-		argsMap["anonymous-auth"] = "false"
-		argsMap["client-ca-file"] = cfg.ClientCA
+	if cfg.ImageServiceSocket != "" {
+		if strings.HasPrefix(cfg.ImageServiceSocket, socketPrefix) {
+			defaultConfig.ImageServiceEndpoint = cfg.ImageServiceSocket
+		} else {
+			defaultConfig.ImageServiceEndpoint = socketPrefix + cfg.ImageServiceSocket
 		}
-	if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" {
-		argsMap["tls-cert-file"] = cfg.ServingKubeletCert
-		argsMap["tls-private-key-file"] = cfg.ServingKubeletKey
 	}
 	if cfg.NodeName != "" {
 		argsMap["hostname-override"] = cfg.NodeName
 	}
-
 	// If the embedded CCM is disabled, don't assume that dual-stack node IPs are safe.
 	// When using an external CCM, the user wants dual-stack node IPs, they will need to set the node-ip kubelet arg directly.
 	// This should be fine since most cloud providers have their own way of finding node IPs that doesn't depend on the kubelet
@@ -106,19 +84,13 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 			argsMap["node-ip"] = cfg.NodeIP
 		}
 	} else {
+		argsMap["cloud-provider"] = "external"
 		if nodeIPs := util.JoinIPs(cfg.NodeIPs); nodeIPs != "" {
 			argsMap["node-ip"] = util.JoinIPs(cfg.NodeIPs)
 		}
 	}
 
 	argsMap["node-labels"] = strings.Join(cfg.NodeLabels, ",")
-	if len(cfg.NodeTaints) > 0 {
-		argsMap["register-with-taints"] = strings.Join(cfg.NodeTaints, ",")
-	}
-
-	if !cfg.DisableCCM {
-		argsMap["cloud-provider"] = "external"
-	}
 
 	if ImageCredProvAvailable(cfg) {
 		logrus.Infof("Kubelet image credential provider bin dir and configuration file found.")
@@ -126,9 +98,5 @@ func kubeletArgs(cfg *config.Agent) map[string]string {
 		argsMap["image-credential-provider-config"] = cfg.ImageCredProvConfig
 	}
 
-	if cfg.ProtectKernelDefaults {
-		argsMap["protect-kernel-defaults"] = "true"
-	}
-
-	return argsMap
+	return argsMap, defaultConfig, nil
 }

pkg/daemons/config/types.go

@@ -118,6 +118,7 @@ type Agent struct {
 	ClusterDomain           string
 	ResolvConf              string
 	RootDir                 string
+	KubeletConfigDir        string
 	KubeConfigKubelet       string
 	KubeConfigKubeProxy     string
 	KubeConfigK3sController string

pkg/etcd/etcd.go

@@ -32,7 +32,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/k3s-io/kine/pkg/client"
 	endpoint2 "github.com/k3s-io/kine/pkg/endpoint"
-	cp "github.com/otiai10/copy"
+	"github.com/otiai10/copy"
 	"github.com/pkg/errors"
 	certutil "github.com/rancher/dynamiclistener/cert"
 	controllerv1 "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
@@ -1050,7 +1050,7 @@ func (e *ETCD) StartEmbeddedTemporary(ctx context.Context) error {
 		conn.Close()
 	}()
 
-	if err := cp.Copy(etcdDataDir, tmpDataDir, cp.Options{PreserveOwner: true}); err != nil {
+	if err := copy.Copy(etcdDataDir, tmpDataDir, copy.Options{PreserveOwner: true}); err != nil {
 		return err
 	}