Move request handlers out of server package

pkg/cli/secretsencrypt/secrets_encrypt.go

@@ -15,6 +15,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -54,7 +55,7 @@ func Enable(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{Enable: ptr.To(true)})
+	b, err := json.Marshal(handlers.EncryptionRequest{Enable: ptr.To(true)})
 	if err != nil {
 		return err
 	}
@@ -73,7 +74,7 @@ func Disable(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{Enable: ptr.To(false)})
+	b, err := json.Marshal(handlers.EncryptionRequest{Enable: ptr.To(false)})
 	if err != nil {
 		return err
 	}
@@ -96,7 +97,7 @@ func Status(app *cli.Context) error {
 	if err != nil {
 		return wrapServerError(err)
 	}
-	status := server.EncryptionState{}
+	status := handlers.EncryptionState{}
 	if err := json.Unmarshal(data, &status); err != nil {
 		return err
 	}
@@ -153,7 +154,7 @@ func Prepare(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionPrepare),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
@@ -175,7 +176,7 @@ func Rotate(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionRotate),
 		Force: cmds.ServerConfig.EncryptForce,
 	})
@@ -197,7 +198,7 @@ func Reencrypt(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionReencryptActive),
 		Force: cmds.ServerConfig.EncryptForce,
 		Skip:  cmds.ServerConfig.EncryptSkip,
@@ -220,7 +221,7 @@ func RotateKeys(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.EncryptionRequest{
+	b, err := json.Marshal(handlers.EncryptionRequest{
 		Stage: ptr.To(secretsencrypt.EncryptionRotateKeys),
 	})
 	if err != nil {

pkg/cli/token/token.go

@@ -16,6 +16,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/kubeadm"
 	"github.com/k3s-io/k3s/pkg/proctitle"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
 	"github.com/pkg/errors"
@@ -153,7 +154,7 @@ func Rotate(app *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	b, err := json.Marshal(server.TokenRotateRequest{
+	b, err := json.Marshal(handlers.TokenRotateRequest{
 		NewToken: ptr.To(cmds.TokenConfig.NewToken),
 	})
 	if err != nil {

pkg/daemons/control/deps/deps.go

@@ -391,11 +391,11 @@ func genClientCerts(config *config.Control) error {
 		}
 	}
 
-	if _, err = factory(user.KubeProxy, nil, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil {
+	if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeProxyKey, regen); err != nil {
 		return err
 	}
-	// This user (system:k3s-controller by default) must be bound to a role in rolebindings.yaml or the downstream equivalent
-	if _, err = factory("system:"+version.Program+"-controller", nil, runtime.ClientK3sControllerCert, runtime.ClientK3sControllerKey); err != nil {
+
+	if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientK3sControllerKey, regen); err != nil {
 		return err
 	}
 

pkg/nodepassword/validate.go

+package nodepassword
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/util"
+	"github.com/pkg/errors"
+	coreclient "github.com/rancher/wrangler/v3/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/kubernetes/pkg/auth/nodeidentifier"
+)
+
+var identifier = nodeidentifier.NewDefaultNodeIdentifier()
+
+// NodeAuthValidator returns a node name, or http error code and error
+type NodeAuthValidator func(req *http.Request) (string, int, error)
+
+// nodeInfo contains information on the requesting node, derived from auth creds
+// and request headers.
+type nodeInfo struct {
+	Name     string
+	Password string
+	User     user.Info
+}
+
+// GetNodeAuthValidator returns a function that will be called to validate node password authentication.
+// Node password authentication is used when requesting kubelet certificates, and verifies that the
+// credentials are valid for the requested node name, and that the node password is valid if it exists.
+// These checks prevent a user with access to one agent from requesting kubelet certificates that
+// could be used to impersonate another cluster member.
+func GetNodeAuthValidator(ctx context.Context, control *config.Control) NodeAuthValidator {
+	runtime := control.Runtime
+	deferredNodes := map[string]bool{}
+	var secretClient coreclient.SecretController
+	var nodeClient coreclient.NodeController
+	var mu sync.Mutex
+
+	return func(req *http.Request) (string, int, error) {
+		node, err := getNodeInfo(req)
+		if err != nil {
+			return "", http.StatusBadRequest, err
+		}
+
+		// node identity auth uses an existing kubelet client cert instead of auth token.
+		// If used, validate that the node identity matches the requested node name.
+		nodeName, isNodeAuth := identifier.NodeIdentity(node.User)
+		if isNodeAuth && nodeName != node.Name {
+			return "", http.StatusBadRequest, errors.New("header node name does not match auth node name")
+		}
+
+		// get client address, to see if deferred node password validation should be allowed when the apiserver
+		// is not available. Deferred password validation is only allowed for requests from the local client.
+		client, _, _ := net.SplitHostPort(req.RemoteAddr)
+		isLocal := client == "127.0.0.1" || client == "::1"
+
+		if secretClient == nil || nodeClient == nil {
+			if runtime.Core != nil {
+				// initialize the client if we can
+				secretClient = runtime.Core.Core().V1().Secret()
+				nodeClient = runtime.Core.Core().V1().Node()
+			} else if isLocal && node.Name == os.Getenv("NODE_NAME") {
+				// If we're verifying our own password, verify it locally and ensure a secret later.
+				return verifyLocalPassword(ctx, control, &mu, deferredNodes, node)
+			} else if isLocal && control.DisableAPIServer && !isNodeAuth {
+				// If we're running on an etcd-only node, and the request didn't use Node Identity auth,
+				// defer node password verification until an apiserver joins the cluster.
+				return verifyRemotePassword(ctx, control, &mu, deferredNodes, node)
+			} else {
+				// Otherwise, reject the request until the core is ready.
+				return "", http.StatusServiceUnavailable, util.ErrCoreNotReady
+			}
+		}
+
+		// verify that the node exists, if using Node Identity auth
+		if err := verifyNode(ctx, nodeClient, node); err != nil {
+			return "", http.StatusUnauthorized, err
+		}
+
+		// verify that the node password secret matches, or create it if it does not
+		if err := Ensure(secretClient, node.Name, node.Password); err != nil {
+			// if the verification failed, reject the request
+			if errors.Is(err, ErrVerifyFailed) {
+				return "", http.StatusForbidden, err
+			}
+			// If verification failed due to an error creating the node password secret, allow
+			// the request, but retry verification until the outage is resolved.  This behavior
+			// allows nodes to join the cluster during outages caused by validating webhooks
+			// blocking secret creation - if the outage requires new nodes to join in order to
+			// run the webhook pods, we must fail open here to resolve the outage.
+			return verifyRemotePassword(ctx, control, &mu, deferredNodes, node)
+		}
+
+		return node.Name, http.StatusOK, nil
+	}
+}
+
+// getNodeInfo returns node name, password, and user extracted
+// from request headers and context. An error is returned
+// if any critical fields are missing.
+func getNodeInfo(req *http.Request) (*nodeInfo, error) {
+	user, ok := request.UserFrom(req.Context())
+	if !ok {
+		return nil, errors.New("auth user not set")
+	}
+
+	program := mux.Vars(req)["program"]
+	nodeName := req.Header.Get(program + "-Node-Name")
+	if nodeName == "" {
+		return nil, errors.New("node name not set")
+	}
+
+	nodePassword := req.Header.Get(program + "-Node-Password")
+	if nodePassword == "" {
+		return nil, errors.New("node password not set")
+	}
+
+	return &nodeInfo{
+		Name:     strings.ToLower(nodeName),
+		Password: nodePassword,
+		User:     user,
+	}, nil
+}
+
+// verifyLocalPassword is used to validate the local node's password secret directly against the node password file, when the apiserver is unavailable.
+// This is only used early in startup, when a control-plane node's agent is starting up without a functional apiserver.
+func verifyLocalPassword(ctx context.Context, control *config.Control, mu *sync.Mutex, deferredNodes map[string]bool, node *nodeInfo) (string, int, error) {
+	// do not attempt to verify the node password if the local host is not running an agent and does not have a node resource.
+	// note that the agent certs and kubeconfigs are created even if the agent is disabled; the only thing that is skipped is starting the kubelet and container runtime.
+	if control.DisableAgent {
+		return node.Name, http.StatusOK, nil
+	}
+
+	// use same password file location that the agent creates
+	nodePasswordRoot := "/"
+	if control.Rootless {
+		nodePasswordRoot = filepath.Join(path.Dir(control.DataDir), "agent")
+	}
+	nodeConfigPath := filepath.Join(nodePasswordRoot, "etc", "rancher", "node")
+	nodePasswordFile := filepath.Join(nodeConfigPath, "password")
+
+	passBytes, err := os.ReadFile(nodePasswordFile)
+	if err != nil {
+		return "", http.StatusInternalServerError, errors.Wrap(err, "unable to read node password file")
+	}
+
+	passHash, err := Hasher.CreateHash(strings.TrimSpace(string(passBytes)))
+	if err != nil {
+		return "", http.StatusInternalServerError, errors.Wrap(err, "unable to hash node password file")
+	}
+
+	if err := Hasher.VerifyHash(passHash, node.Password); err != nil {
+		return "", http.StatusForbidden, errors.Wrap(err, "unable to verify local node password")
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	if _, ok := deferredNodes[node.Name]; !ok {
+		deferredNodes[node.Name] = true
+		go ensureSecret(ctx, control, node)
+		logrus.Infof("Password verified locally for node %s", node.Name)
+	}
+
+	return node.Name, http.StatusOK, nil
+}
+
+// verifyRemotePassword is used when the server does not have a local apisever, as in the case of etcd-only nodes.
+// The node password is ensured once an apiserver joins the cluster.
+func verifyRemotePassword(ctx context.Context, control *config.Control, mu *sync.Mutex, deferredNodes map[string]bool, node *nodeInfo) (string, int, error) {
+	mu.Lock()
+	defer mu.Unlock()
+
+	if _, ok := deferredNodes[node.Name]; !ok {
+		deferredNodes[node.Name] = true
+		go ensureSecret(ctx, control, node)
+		logrus.Infof("Password verification deferred for node %s", node.Name)
+	}
+
+	return node.Name, http.StatusOK, nil
+}
+
+// verifyNode confirms that a node with the given name exists, to prevent auth
+// from succeeding with a client certificate for a node that has been deleted from the cluster.
+func verifyNode(ctx context.Context, nodeClient coreclient.NodeController, node *nodeInfo) error {
+	if nodeName, isNodeAuth := identifier.NodeIdentity(node.User); isNodeAuth {
+		if _, err := nodeClient.Cache().Get(nodeName); err != nil {
+			return errors.Wrap(err, "unable to verify node identity")
+		}
+	}
+	return nil
+}
+
+// ensureSecret validates a server's node password secret once the apiserver is up.
+// As the node has already joined the cluster at this point, this is purely informational.
+func ensureSecret(ctx context.Context, control *config.Control, node *nodeInfo) {
+	runtime := control.Runtime
+	_ = wait.PollUntilContextCancel(ctx, time.Second*5, true, func(ctx context.Context) (bool, error) {
+		if runtime.Core != nil {
+			secretClient := runtime.Core.Core().V1().Secret()
+			// This is consistent with events attached to the node generated by the kubelet
+			// https://github.com/kubernetes/kubernetes/blob/612130dd2f4188db839ea5c2dea07a96b0ad8d1c/pkg/kubelet/kubelet.go#L479-L485
+			nodeRef := &corev1.ObjectReference{
+				Kind:      "Node",
+				Name:      node.Name,
+				UID:       types.UID(node.Name),
+				Namespace: "",
+			}
+			if err := Ensure(secretClient, node.Name, node.Password); err != nil {
+				runtime.Event.Eventf(nodeRef, corev1.EventTypeWarning, "NodePasswordValidationFailed", "Deferred node password secret validation failed: %v", err)
+				// Return true to stop polling if the password verification failed; only retry on secret creation errors.
+				return errors.Is(err, ErrVerifyFailed), nil
+			}
+			runtime.Event.Event(nodeRef, corev1.EventTypeNormal, "NodePasswordValidationComplete", "Deferred node password secret validation complete")
+			return true, nil
+		}
+		return false, nil
+	})
+}

pkg/server/cert.go → pkg/server/handlers/cert.go

-package server
+package handlers
 
 import (
 	"bytes"
@@ -28,14 +28,14 @@ import (
 	"k8s.io/client-go/util/keyutil"
 )
 
-func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
+func CACertReplace(control *config.Control) http.HandlerFunc {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.Method != http.MethodPut {
 			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
 			return
 		}
 		force, _ := strconv.ParseBool(req.FormValue("force"))
-		if err := caCertReplace(server, req.Body, force); err != nil {
+		if err := caCertReplace(control, req.Body, force); err != nil {
 			util.SendErrorWithID(err, "certificate", resp, req, http.StatusInternalServerError)
 			return
 		}
@@ -48,54 +48,54 @@ func caCertReplaceHandler(server *config.Control) http.HandlerFunc {
 // validated to confirm that the new certs share a common root with the existing certs, and if so are saved to
 // the datastore.  If the functions succeeds, servers should be restarted immediately to load the new certs
 // from the bootstrap data.
-func caCertReplace(server *config.Control, buf io.ReadCloser, force bool) error {
-	tmpdir, err := os.MkdirTemp(server.DataDir, ".rotate-ca-tmp-")
+func caCertReplace(control *config.Control, buf io.ReadCloser, force bool) error {
+	tmpdir, err := os.MkdirTemp(control.DataDir, ".rotate-ca-tmp-")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(tmpdir)
 
 	runtime := config.NewRuntime(nil)
-	runtime.EtcdConfig = server.Runtime.EtcdConfig
-	runtime.ServerToken = server.Runtime.ServerToken
+	runtime.EtcdConfig = control.Runtime.EtcdConfig
+	runtime.ServerToken = control.Runtime.ServerToken
 
-	tmpServer := &config.Control{
+	tmpControl := &config.Control{
 		Runtime: runtime,
-		Token:   server.Token,
+		Token:   control.Token,
 		DataDir: tmpdir,
 	}
-	deps.CreateRuntimeCertFiles(tmpServer)
+	deps.CreateRuntimeCertFiles(tmpControl)
 
 	bootstrapData := bootstrap.PathsDataformat{}
 	if err := json.NewDecoder(buf).Decode(&bootstrapData); err != nil {
 		return err
 	}
 
-	if err := bootstrap.WriteToDiskFromStorage(bootstrapData, &tmpServer.Runtime.ControlRuntimeBootstrap); err != nil {
+	if err := bootstrap.WriteToDiskFromStorage(bootstrapData, &tmpControl.Runtime.ControlRuntimeBootstrap); err != nil {
 		return err
 	}
 
-	if err := defaultBootstrap(server, tmpServer); err != nil {
+	if err := defaultBootstrap(control, tmpControl); err != nil {
 		return errors.Wrap(err, "failed to set default bootstrap values")
 	}
 
-	if err := validateBootstrap(server, tmpServer); err != nil {
+	if err := validateBootstrap(control, tmpControl); err != nil {
 		if !force {
 			return errors.Wrap(err, "failed to validate new CA certificates and keys")
 		}
 		logrus.Warnf("Save of CA certificates and keys forced, ignoring validation errors: %v", err)
 	}
 
-	return cluster.Save(context.TODO(), tmpServer, true)
+	return cluster.Save(context.TODO(), tmpControl, true)
 }
 
 // defaultBootstrap provides default values from the existing bootstrap fields
 // if the value is not tagged for rotation, or the current value is empty.
-func defaultBootstrap(oldServer, newServer *config.Control) error {
+func defaultBootstrap(oldControl, newControl *config.Control) error {
 	errs := []error{}
 	// Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths.
-	oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem()
-	newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem()
+	oldMeta := reflect.ValueOf(&oldControl.Runtime.ControlRuntimeBootstrap).Elem()
+	newMeta := reflect.ValueOf(&newControl.Runtime.ControlRuntimeBootstrap).Elem()
 
 	// use the existing file if the new file does not exist or is empty
 	for _, field := range reflect.VisibleFields(oldMeta.Type()) {
@@ -122,12 +122,12 @@ func defaultBootstrap(oldServer, newServer *config.Control) error {
 // validateBootstrap checks the new certs and keys to ensure that the cluster would function properly were they to be used.
 // - The new leaf CA certificates must be verifiable using the same root and intermediate certs as the current leaf CA certificates.
 // - The new service account signing key bundle must include the currently active signing key.
-func validateBootstrap(oldServer, newServer *config.Control) error {
+func validateBootstrap(oldControl, newControl *config.Control) error {
 	errs := []error{}
 
 	// Use reflection to iterate over all of the bootstrap fields, checking files at each of the new paths.
-	oldMeta := reflect.ValueOf(&oldServer.Runtime.ControlRuntimeBootstrap).Elem()
-	newMeta := reflect.ValueOf(&newServer.Runtime.ControlRuntimeBootstrap).Elem()
+	oldMeta := reflect.ValueOf(&oldControl.Runtime.ControlRuntimeBootstrap).Elem()
+	newMeta := reflect.ValueOf(&newControl.Runtime.ControlRuntimeBootstrap).Elem()
 
 	for _, field := range reflect.VisibleFields(oldMeta.Type()) {
 		// Only handle bootstrap fields tagged for rotation

pkg/server/handlers/router.go

+package handlers
+
+import (
+	"context"
+	"net/http"
+	"path/filepath"
+
+	"github.com/gorilla/mux"
+	"github.com/k3s-io/k3s/pkg/cli/cmds"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/nodepassword"
+	"github.com/k3s-io/k3s/pkg/server/auth"
+	"github.com/k3s-io/k3s/pkg/version"
+	"k8s.io/apiserver/pkg/authentication/user"
+	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
+)
+
+const (
+	staticURL = "/static/"
+)
+
+func NewHandler(ctx context.Context, control *config.Control, cfg *cmds.Server) http.Handler {
+	nodeAuth := nodepassword.GetNodeAuthValidator(ctx, control)
+
+	prefix := "/v1-{program}"
+	authed := mux.NewRouter().SkipClean(true)
+	authed.NotFoundHandler = APIServer(control, cfg)
+	authed.Use(auth.HasRole(control, version.Program+":agent", user.NodesGroup, bootstrapapi.BootstrapDefaultGroup))
+	authed.Handle(prefix+"/serving-kubelet.crt", ServingKubeletCert(control, nodeAuth))
+	authed.Handle(prefix+"/client-kubelet.crt", ClientKubeletCert(control, nodeAuth))
+	authed.Handle(prefix+"/client-kube-proxy.crt", ClientKubeProxyCert(control))
+	authed.Handle(prefix+"/client-{program}-controller.crt", ClientControllerCert(control))
+	authed.Handle(prefix+"/client-ca.crt", File(control.Runtime.ClientCA))
+	authed.Handle(prefix+"/server-ca.crt", File(control.Runtime.ServerCA))
+	authed.Handle(prefix+"/apiservers", APIServers(control))
+	authed.Handle(prefix+"/config", Config(control, cfg))
+	authed.Handle(prefix+"/readyz", Readyz(control))
+
+	nodeAuthed := mux.NewRouter().SkipClean(true)
+	nodeAuthed.NotFoundHandler = authed
+	nodeAuthed.Use(auth.HasRole(control, user.NodesGroup))
+	nodeAuthed.Handle(prefix+"/connect", control.Runtime.Tunnel)
+
+	serverAuthed := mux.NewRouter().SkipClean(true)
+	serverAuthed.NotFoundHandler = nodeAuthed
+	serverAuthed.Use(auth.HasRole(control, version.Program+":server"))
+	serverAuthed.Handle(prefix+"/encrypt/status", EncryptionStatus(control))
+	serverAuthed.Handle(prefix+"/encrypt/config", EncryptionConfig(ctx, control))
+	serverAuthed.Handle(prefix+"/cert/cacerts", CACertReplace(control))
+	serverAuthed.Handle(prefix+"/server-bootstrap", Bootstrap(control))
+	serverAuthed.Handle(prefix+"/token", TokenRequest(ctx, control))
+
+	systemAuthed := mux.NewRouter().SkipClean(true)
+	systemAuthed.NotFoundHandler = serverAuthed
+	systemAuthed.MethodNotAllowedHandler = serverAuthed
+	systemAuthed.Use(auth.HasRole(control, user.SystemPrivilegedGroup))
+	systemAuthed.Methods(http.MethodConnect).Handler(control.Runtime.Tunnel)
+
+	router := mux.NewRouter().SkipClean(true)
+	router.NotFoundHandler = systemAuthed
+	router.PathPrefix(staticURL).Handler(Static(staticURL, filepath.Join(control.DataDir, "static")))
+	router.Handle("/cacerts", CACerts(control))
+	router.Handle("/ping", Ping())
+
+	return router
+}

pkg/server/token.go → pkg/server/handlers/token.go

-package server
+package handlers
 
 import (
 	"context"
@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"path/filepath"
 
+	"github.com/k3s-io/k3s/pkg/clientaccess"
 	"github.com/k3s-io/k3s/pkg/cluster"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/passwd"
@@ -30,7 +32,7 @@ func getServerTokenRequest(req *http.Request) (TokenRotateRequest, error) {
 	return result, err
 }
 
-func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handler {
+func TokenRequest(ctx context.Context, control *config.Control) http.Handler {
 	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 		if req.Method != http.MethodPut {
 			util.SendError(fmt.Errorf("method not allowed"), resp, req, http.StatusMethodNotAllowed)
@@ -43,7 +45,7 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl
 			util.SendError(err, resp, req, http.StatusBadRequest)
 			return
 		}
-		if err = tokenRotate(ctx, server, *sTokenReq.NewToken); err != nil {
+		if err = tokenRotate(ctx, control, *sTokenReq.NewToken); err != nil {
 			util.SendErrorWithID(err, "token", resp, req, http.StatusInternalServerError)
 			return
 		}
@@ -51,8 +53,20 @@ func tokenRequestHandler(ctx context.Context, server *config.Control) http.Handl
 	})
 }
 
-func tokenRotate(ctx context.Context, server *config.Control, newToken string) error {
-	passwd, err := passwd.Read(server.Runtime.PasswdFile)
+func WriteToken(token, file, certs string) error {
+	if len(token) == 0 {
+		return nil
+	}
+
+	token, err := clientaccess.FormatToken(token, certs)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(file, []byte(token+"\n"), 0600)
+}
+
+func tokenRotate(ctx context.Context, control *config.Control, newToken string) error {
+	passwd, err := passwd.Read(control.Runtime.PasswdFile)
 	if err != nil {
 		return err
 	}
@@ -76,24 +90,24 @@ func tokenRotate(ctx context.Context, server *config.Control, newToken string) e
 	}
 
 	// If the agent token is the same a server, we need to change both
-	if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && server.AgentToken == "" {
+	if agentToken, found := passwd.Pass("node"); found && agentToken == oldToken && control.AgentToken == "" {
 		if err := passwd.EnsureUser("node", version.Program+":agent", newToken); err != nil {
 			return err
 		}
 	}
 
-	if err := passwd.Write(server.Runtime.PasswdFile); err != nil {
+	if err := passwd.Write(control.Runtime.PasswdFile); err != nil {
 		return err
 	}
 
-	serverTokenFile := filepath.Join(server.DataDir, "token")
-	if err := writeToken("server:"+newToken, serverTokenFile, server.Runtime.ServerCA); err != nil {
+	serverTokenFile := filepath.Join(control.DataDir, "token")
+	if err := WriteToken("server:"+newToken, serverTokenFile, control.Runtime.ServerCA); err != nil {
 		return err
 	}
 
-	if err := cluster.RotateBootstrapToken(ctx, server, oldToken); err != nil {
+	if err := cluster.RotateBootstrapToken(ctx, control, oldToken); err != nil {
 		return err
 	}
-	server.Token = newToken
-	return cluster.Save(ctx, server, true)
+	control.Token = newToken
+	return cluster.Save(ctx, control, true)
 }

pkg/server/server.go

@@ -23,6 +23,7 @@ import (
 	"github.com/k3s-io/k3s/pkg/nodepassword"
 	"github.com/k3s-io/k3s/pkg/rootlessports"
 	"github.com/k3s-io/k3s/pkg/secretsencrypt"
+	"github.com/k3s-io/k3s/pkg/server/handlers"
 	"github.com/k3s-io/k3s/pkg/static"
 	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -58,7 +59,7 @@ func StartServer(ctx context.Context, config *Config, cfg *cmds.Server) error {
 	wg := &sync.WaitGroup{}
 	wg.Add(len(config.StartupHooks))
 
-	config.ControlConfig.Runtime.Handler = router(ctx, config, cfg)
+	config.ControlConfig.Runtime.Handler = handlers.NewHandler(ctx, &config.ControlConfig, cfg)
 	config.ControlConfig.Runtime.StartupHooksWg = wg
 
 	shArgs := cmds.StartupHookArgs{
@@ -346,7 +347,7 @@ func printTokens(config *config.Control) error {
 	var serverTokenFile string
 	if config.Runtime.ServerToken != "" {
 		serverTokenFile = filepath.Join(config.DataDir, "token")
-		if err := writeToken(config.Runtime.ServerToken, serverTokenFile, config.Runtime.ServerCA); err != nil {
+		if err := handlers.WriteToken(config.Runtime.ServerToken, serverTokenFile, config.Runtime.ServerCA); err != nil {
 			return err
 		}
 
@@ -374,7 +375,7 @@ func printTokens(config *config.Control) error {
 					return err
 				}
 			}
-			if err := writeToken(config.Runtime.AgentToken, agentTokenFile, config.Runtime.ServerCA); err != nil {
+			if err := handlers.WriteToken(config.Runtime.AgentToken, agentTokenFile, config.Runtime.ServerCA); err != nil {
 				return err
 			}
 		} else if serverTokenFile != "" {
@@ -490,18 +491,6 @@ func printToken(httpsPort int, advertiseIP, prefix, cmd, varName string) {
 	logrus.Infof("%s %s %s -s https://%s:%d -t ${%s}", prefix, version.Program, cmd, advertiseIP, httpsPort, varName)
 }
 
-func writeToken(token, file, certs string) error {
-	if len(token) == 0 {
-		return nil
-	}
-
-	token, err := clientaccess.FormatToken(token, certs)
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(file, []byte(token+"\n"), 0600)
-}
-
 func setNoProxyEnv(config *config.Control) error {
 	splitter := func(c rune) bool {
 		return c == ','