Add wrapper

main.go

+package main
+
+import (
+	"context"
+	"net"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/kubernetes/pkg/wrapper/agent"
+	"k8s.io/kubernetes/pkg/wrapper/server"
+)
+
+func runAgent() {
+	_, ipNet, err := net.ParseCIDR("10.43.0.0/16")
+	if err != nil {
+		panic(err)
+	}
+
+	err = agent.Agent(&agent.AgentConfig{
+		KubeConfig:    "./data/cred/kubeconfig-node.yaml",
+		RuntimeSocket: "unix:///run/containerd/containerd.sock",
+		ClusterCIDR:   *ipNet,
+	})
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	select {}
+}
+
+func runServer() {
+	hostname, err := os.Hostname()
+	if err != nil {
+		panic(err)
+	}
+
+	_, cidr, err := net.ParseCIDR("10.43.0.0/16")
+	if err != nil {
+		panic(err)
+	}
+
+	_, clusterCidr, err := net.ParseCIDR("10.42.0.0/16")
+	if err != nil {
+		panic(err)
+	}
+
+	os.MkdirAll("./data", 0700)
+	os.Chdir("./data")
+
+	cfg := server.ServerConfig{
+		DataDir:        "./data",
+		ListenAddr:     net.ParseIP("0.0.0.0"),
+		ListenPort:     6443,
+		ClusterIPRange: *clusterCidr,
+		ServiceIPRange: *cidr,
+		PublicHostname: hostname,
+	}
+
+	ctx := context.Background()
+	err = server.Server(ctx, &cfg)
+	if err != nil {
+		panic(err)
+	}
+
+	<-ctx.Done()
+
+}
+
+func main() {
+	if len(os.Args) > 1 && os.Args[1] == "agent" {
+		runAgent()
+	}
+	runServer()
+}

pkg/wrapper/agent/agent.go

+package agent
+
+import (
+	"context"
+	"math/rand"
+	"net"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/apiserver/pkg/util/logs"
+	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
+	"k8s.io/kubernetes/cmd/kubelet/app"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+)
+
+type AgentConfig struct {
+	NodeName      string
+	ClusterCIDR   net.IPNet
+	KubeConfig    string
+	NodeIP        string
+	RuntimeSocket string
+	ListenAddress string
+	CACertPath    string
+	CNIBinDir     string
+	CNIConfDir    string
+}
+
+func Agent(config *AgentConfig) error {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	prepare(config)
+
+	kubelet(config)
+	kubeProxy(config)
+
+	return nil
+}
+
+func prepare(config *AgentConfig) {
+	if config.CNIBinDir == "" {
+		config.CNIBinDir = "/opt/cni/bin"
+	}
+	if config.CNIConfDir == "" {
+		config.CNIConfDir = "/etc/cni/net.d"
+	}
+}
+
+func kubeProxy(config *AgentConfig) {
+	command := app2.NewProxyCommand()
+	command.SetArgs([]string{
+		"--proxy-mode", "iptables",
+		"--healthz-bind-address", "127.0.0.1",
+		"--kubeconfig", config.KubeConfig,
+		"--cluster-cidr", config.ClusterCIDR.String(),
+	})
+
+	go func() {
+		err := command.Execute()
+		logrus.Fatalf("kube-proxy exited: %v", err)
+	}()
+}
+
+func kubelet(config *AgentConfig) {
+	command := app.NewKubeletCommand(context.Background().Done())
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	args := []string{
+		"--healthz-bind-address", "127.0.0.1",
+		"--read-only-port", "0",
+		"--allow-privileged=true",
+		"--cluster-domain", "cluster.local",
+		"--cluster-dns", "10.43.0.10",
+		"--kubeconfig", config.KubeConfig,
+		"--eviction-hard", "imagefs.available<5%,nodefs.available<5%",
+		"--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%",
+		"--feature-gates=MountPropagation=true",
+		"--node-ip", config.NodeIP,
+		"--fail-swap-on=false",
+		"--cgroup-root", "/k3s",
+		"--cgroup-driver", "cgroupfs",
+		"--container-runtime-endpoint", config.RuntimeSocket,
+		"--cni-conf-dir", config.CNIConfDir,
+		"--cni-bin-dir", config.CNIBinDir,
+	}
+	if config.ListenAddress != "" {
+		args = append(args, "--address", config.ListenAddress)
+	}
+	if config.CACertPath != "" {
+		args = append(args, "--anonymous-auth=false", "--client-ca-file", config.CACertPath)
+	}
+	if config.NodeName != "" {
+		args = append(args, "--hostname-override", config.NodeName)
+	}
+
+	command.SetArgs(args)
+
+	go func() {
+		logrus.Fatalf("kubelet exited: %v", command.Execute())
+	}()
+}

pkg/wrapper/server/server.go

+package server
+
+import (
+	"context"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/csv"
+	"encoding/hex"
+	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
+	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
+	sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	"k8s.io/kubernetes/pkg/master"
+	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+)
+
+var (
+	kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
+clusters:
+- cluster:
+    server: {{.URL}}
+    certificate-authority-data: {{.CACert}}
+  name: local
+contexts:
+- context:
+    cluster: local
+    namespace: default
+    user: user
+  name: Default
+current-context: Default
+kind: Config
+preferences: {}
+users:
+- name: user
+  user:
+    username: {{.User}}
+    password: {{.Password}}
+`))
+)
+
+type ServerConfig struct {
+	PublicHostname string
+	AdvertiseIP    *net.IP
+	AdvertisePort  int
+	ListenAddr     net.IP
+	ListenPort     int
+	ClusterIPRange net.IPNet
+	ServiceIPRange net.IPNet
+	DataDir        string
+	LeaderElect    bool
+	UseTokenCA     bool
+	ETCDEndpoints  []string
+	ETCDKeyFile    string
+	ETCDCertFile   string
+	ETCDCAFile     string
+	NoScheduler    bool
+
+	tlsCert          string
+	tlsKey           string
+	tlsCA            string
+	tlsCAKey         string
+	tokenCA          string
+	tokenCAKey       string
+	serviceKey       string
+	passwdFile       string
+	kubeConfigSystem string
+	kubeConfigNode   string
+
+	NodeCert      string
+	NodeKey       string
+	ClientToken   string
+	NodeToken     string
+	KubeConfig    string
+	Handler       http.Handler
+	Authenticator authenticator.Request
+}
+
+func Server(ctx context.Context, config *ServerConfig) error {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	auth, handler, err := apiServer(ctx, config)
+	if err != nil {
+		return err
+	}
+
+	config.Handler = handler
+	config.Authenticator = auth
+
+	if !config.NoScheduler {
+		scheduler(ctx, config)
+	}
+	controllerManager(ctx, config)
+
+	return nil
+}
+
+func controllerManager(ctx context.Context, cfg *ServerConfig) {
+	command := cmapp.NewControllerManagerCommand()
+	command.SetArgs([]string{
+		"--kubeconfig", cfg.kubeConfigSystem,
+		fmt.Sprintf("--leader-elect=%v", cfg.LeaderElect),
+		"--service-account-private-key-file", cfg.serviceKey,
+		"--allocate-node-cidrs",
+		"--cluster-cidr", cfg.ClusterIPRange.String(),
+		"--root-ca-file", cfg.tokenCA,
+		"--secure-port", "0",
+	})
+
+	go func() {
+		logrus.Fatalf("controller-manager exited: %v", command.Execute())
+	}()
+}
+
+func scheduler(ctx context.Context, cfg *ServerConfig) {
+	command := sapp.NewSchedulerCommand()
+	command.SetArgs([]string{
+		"--kubeconfig", cfg.kubeConfigSystem,
+	})
+
+	go func() {
+		logrus.Fatalf("scheduler exited: %v", command.Execute())
+	}()
+}
+
+func apiServer(ctx context.Context, config *ServerConfig) (authenticator.Request, http.Handler, error) {
+	if err := prepare(config); err != nil {
+		return nil, nil, err
+	}
+
+	s := options.NewServerRunOptions()
+	s.Etcd.StorageConfig.ServerList = config.ETCDEndpoints
+	s.Etcd.StorageConfig.KeyFile = config.ETCDKeyFile
+	s.Etcd.StorageConfig.CertFile = config.ETCDCertFile
+	s.Etcd.StorageConfig.CAFile = config.ETCDCAFile
+	if len(config.ETCDEndpoints) > 0 {
+		s.Etcd.StorageConfig.Type = storagebackend.StorageTypeETCD3
+	}
+	s.InsecureServing.BindPort = 0
+	s.AllowPrivileged = true
+	s.Authorization.Modes = []string{modes.ModeNode, modes.ModeRBAC}
+	s.ServiceAccountSigningKeyFile = config.serviceKey
+	s.ServiceClusterIPRange = config.ServiceIPRange
+	s.SecureServing.AdvertisePort = config.AdvertisePort
+	if config.AdvertiseIP != nil {
+		s.GenericServerRunOptions.AdvertiseAddress = *config.AdvertiseIP
+	}
+	s.SecureServing.BindPort = config.ListenPort
+	s.SecureServing.BindAddress = config.ListenAddr
+	s.SecureServing.ServerCert = apiserveroptions.GeneratableKeyCert{
+		CertKey: apiserveroptions.CertKey{
+			CertFile: config.tlsCert,
+			KeyFile:  config.tlsKey,
+		},
+	}
+	s.Authentication.ServiceAccounts.KeyFiles = []string{config.serviceKey}
+	s.Authentication.ServiceAccounts.Issuer = "k3s"
+	s.Authentication.ServiceAccounts.APIAudiences = []string{"unknown"}
+	s.Authentication.PasswordFile.BasicAuthFile = config.passwdFile
+	s.KubeletConfig.CertFile = config.NodeCert
+	s.KubeletConfig.KeyFile = config.NodeKey
+
+	sc, err := app.Complete(s)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	masterConfig, server, err := app.CreateServerChain(sc, ctx.Done())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	prepared := server.PrepareRun()
+
+	go func() {
+		err := prepared.Run(ctx.Done())
+		logrus.Fatalf("apiserver exited: %v", err)
+	}()
+
+	return masterConfig.GenericConfig.Authentication.Authenticator, prepared.Handler, nil
+}
+
+func prepare(config *ServerConfig) error {
+	var err error
+
+	if _, err := os.Stat(config.DataDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(config.DataDir, 0700); err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	config.DataDir, err = filepath.Abs(config.DataDir)
+	if err != nil {
+		return err
+	}
+
+	os.MkdirAll(path.Join(config.DataDir, "tls"), 0700)
+	os.MkdirAll(path.Join(config.DataDir, "cred"), 0700)
+
+	name := config.PublicHostname
+	config.tlsCert = path.Join(config.DataDir, "tls", name+".crt")
+	config.tlsKey = path.Join(config.DataDir, "tls", name+".key")
+	config.tlsCA = path.Join(config.DataDir, "tls", "ca.crt")
+	config.tlsCAKey = path.Join(config.DataDir, "tls", "ca.key")
+	config.tokenCA = path.Join(config.DataDir, "tls", "token-ca.crt")
+	config.tokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key")
+	config.serviceKey = path.Join(config.DataDir, "tls", "service.key")
+	config.passwdFile = path.Join(config.DataDir, "cred", "passwd")
+	config.KubeConfig = path.Join(config.DataDir, "cred", "kubeconfig.yaml")
+	config.kubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml")
+	config.kubeConfigNode = path.Join(config.DataDir, "cred", "kubeconfig-node.yaml")
+	config.NodeKey = path.Join(config.DataDir, "tls", "token-node.key")
+	config.NodeCert = path.Join(config.DataDir, "tls", "token-node.crt")
+
+	regen := false
+	if _, err := os.Stat(config.tlsCA); err != nil {
+		regen = true
+		if err := genCA(config); err != nil {
+			return err
+		}
+	}
+
+	if err := genServiceAccount(config); err != nil {
+		return err
+	}
+
+	if err := genTLS(regen, config); err != nil {
+		return err
+	}
+
+	if err := genTokenTLS(config); err != nil {
+		return err
+	}
+
+	if err := genUsers(config); err != nil {
+		return err
+	}
+
+	return readTokens(config)
+}
+
+func readTokens(config *ServerConfig) error {
+	f, err := os.Open(config.passwdFile)
+	if err != nil {
+		return err
+	}
+	reader := csv.NewReader(f)
+	reader.FieldsPerRecord = -1
+
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if len(record) < 2 {
+			continue
+		}
+
+		switch record[1] {
+		case "node":
+			config.NodeToken = "node:" + record[0]
+		case "admin":
+			config.ClientToken = "admin:" + record[0]
+		}
+	}
+
+	return nil
+}
+
+func genUsers(config *ServerConfig) error {
+	if s, err := os.Stat(config.passwdFile); err == nil && s.Size() > 0 {
+		return nil
+	}
+
+	adminToken, err := getToken()
+	if err != nil {
+		return err
+	}
+	systemToken, err := getToken()
+	if err != nil {
+		return err
+	}
+	nodeToken, err := getToken()
+	if err != nil {
+		return err
+	}
+
+	passwd := fmt.Sprintf(`%s,admin,admin,system:masters
+%s,system,system,system:masters
+%s,node,node,system:masters
+`, adminToken, systemToken, nodeToken)
+
+	caCertBytes, err := ioutil.ReadFile(config.tlsCA)
+	if err != nil {
+		return err
+	}
+
+	caCert := base64.StdEncoding.EncodeToString(caCertBytes)
+
+	if err := kubeConfig(config.KubeConfig, fmt.Sprintf("https://%s:%d", config.PublicHostname, config.ListenPort), caCert,
+		"admin", adminToken); err != nil {
+		return err
+	}
+
+	if err := kubeConfig(config.kubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert,
+		"system", systemToken); err != nil {
+		return err
+	}
+
+	if err := kubeConfig(config.kubeConfigNode, fmt.Sprintf("https://%s:%d", config.PublicHostname, config.ListenPort), caCert,
+		"node", nodeToken); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(config.passwdFile, []byte(passwd), 0600)
+}
+
+func getToken() (string, error) {
+	token := make([]byte, 16, 16)
+	_, err := cryptorand.Read(token)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(token), err
+}
+
+func genTokenTLS(config *ServerConfig) error {
+	regen := false
+	if _, err := os.Stat(config.tokenCA); err != nil {
+		regen = true
+		if err := genTokenCA(config); err != nil {
+			return err
+		}
+	}
+
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(config.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: "kubernetes",
+		AltNames: certutil.AltNames{
+			DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
+			IPs:      []net.IP{net.ParseIP("127.0.0.1"), apiServerServiceIP},
+		},
+		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	if _, err := os.Stat(config.NodeCert); err == nil && !regen {
+		return nil
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(config.tokenCAKey)
+	if err != nil {
+		return err
+	}
+
+	caBytes, err := ioutil.ReadFile(config.tokenCA)
+	if err != nil {
+		return err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return err
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(config.NodeKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(config.NodeCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+}
+
+func genTLS(regen bool, config *ServerConfig) error {
+	if !regen {
+		_, certErr := os.Stat(config.tlsCert)
+		_, keyErr := os.Stat(config.tlsKey)
+		if certErr == nil && keyErr == nil {
+			return nil
+		}
+	}
+
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(config.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: config.PublicHostname,
+		AltNames: certutil.AltNames{
+			DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", config.PublicHostname},
+			IPs:      []net.IP{apiServerServiceIP},
+		},
+		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+
+	bindIP := config.ListenAddr.String()
+	if bindIP == "0.0.0.0" {
+		cfg.AltNames.DNSNames = append(cfg.AltNames.DNSNames, "localhost")
+	} else {
+		cfg.AltNames.IPs = append(cfg.AltNames.IPs, config.ListenAddr)
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(config.tlsCAKey)
+	if err != nil {
+		return err
+	}
+
+	caBytes, err := ioutil.ReadFile(config.tlsCA)
+	if err != nil {
+		return err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return err
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(config.tlsKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(config.tlsCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+}
+
+func genServiceAccount(config *ServerConfig) error {
+	_, keyErr := os.Stat(config.serviceKey)
+	if keyErr == nil {
+		return nil
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	return certutil.WriteKey(config.serviceKey, certutil.EncodePrivateKeyPEM(key))
+}
+
+func genTokenCA(config *ServerConfig) error {
+	if !config.UseTokenCA {
+		config.tokenCA = config.tlsCA
+		config.tokenCAKey = config.tlsCAKey
+		return nil
+	}
+
+	caKey, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("%s-ca@%d", "k3s-token", time.Now().Unix()),
+	}
+
+	cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(config.tokenCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(config.tokenCA, certutil.EncodeCertPEM(cert))
+}
+
+func genCA(config *ServerConfig) error {
+	caKey, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("%s-ca@%d", "k3s", time.Now().Unix()),
+	}
+
+	cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(config.tlsCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(config.tlsCA, certutil.EncodeCertPEM(cert))
+}
+
+func kubeConfig(dest, url, cert, user, password string) error {
+	data := struct {
+		URL      string
+		CACert   string
+		User     string
+		Password string
+	}{
+		URL:      url,
+		CACert:   cert,
+		User:     user,
+		Password: password,
+	}
+
+	output, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	return kubeconfigTemplate.Execute(output, &data)
+}