Add support for containerd cri registry config_path

pkg/agent/config/config.go

@@ -560,6 +560,8 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	}
 	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "agent", "containerd")
 	nodeConfig.Containerd.Log = filepath.Join(envInfo.DataDir, "agent", "containerd", "containerd.log")
+	nodeConfig.Containerd.Registry = filepath.Join(envInfo.DataDir, "agent", "etc", "containerd", "certs.d")
+	nodeConfig.Containerd.NoDefault = envInfo.ContainerdNoDefault
 	nodeConfig.Containerd.Debug = envInfo.Debug
 	applyContainerdStateAndAddress(nodeConfig)
 	applyCRIDockerdAddress(nodeConfig)

pkg/agent/containerd/config.go

+package containerd
+
+import (
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/k3s-io/k3s/pkg/agent/templates"
+	util2 "github.com/k3s-io/k3s/pkg/agent/util"
+	"github.com/k3s-io/k3s/pkg/daemons/config"
+	"github.com/k3s-io/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+)
+
+// writeContainerdConfig renders and saves config.toml from the filled template
+func writeContainerdConfig(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	var containerdTemplate string
+	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
+	if err == nil {
+		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
+		containerdTemplate = string(containerdTemplateBytes)
+	} else if os.IsNotExist(err) {
+		containerdTemplate = templates.ContainerdConfigTemplate
+	} else {
+		return err
+	}
+	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
+	if err != nil {
+		return err
+	}
+
+	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+}
+
+// writeContainerdHosts merges registry mirrors/configs, and renders and saves hosts.toml from the filled template
+func writeContainerdHosts(cfg *config.Node, containerdConfig templates.ContainerdConfig) error {
+	registry := containerdConfig.PrivateRegistryConfig
+	hosts := map[string]templates.HostConfig{}
+
+	for host, mirror := range registry.Mirrors {
+		defaultHost, _ := docker.DefaultHost(host)
+		config := templates.HostConfig{
+			Host:    defaultHost,
+			Program: version.Program,
+		}
+		if host == "*" {
+			host = "_default"
+			config.Host = ""
+		} else if containerdConfig.NoDefaultEndpoint {
+			config.Host = ""
+		}
+		// TODO: rewrites are currently copied from the mirror settings into each endpoint.
+		// In the future, we should allow for per-endpoint rewrites, instead of expecting
+		// all mirrors to have the same structure. This will require changes to the registries.yaml
+		// structure, which is defined in rancher/wharfie.
+		for _, endpoint := range mirror.Endpoints {
+			if endpointURL, err := url.Parse(endpoint); err == nil {
+				config.Endpoints = append(config.Endpoints, templates.RegistryEndpoint{
+					OverridePath: endpointURL.Path != "" && endpointURL.Path != "/" && !strings.HasSuffix(endpointURL.Path, "/v2"),
+					Config:       registry.Configs[endpointURL.Host],
+					Rewrites:     mirror.Rewrites,
+					URI:          endpoint,
+				})
+			}
+		}
+		hosts[host] = config
+	}
+
+	for host, registry := range registry.Configs {
+		config, ok := hosts[host]
+		if !ok {
+			config = templates.HostConfig{
+				Program: version.Program,
+			}
+		}
+		if len(config.Endpoints) == 0 {
+			config.Endpoints = []templates.RegistryEndpoint{
+				{
+					Config: registry,
+					URI:    "https://" + host,
+				},
+			}
+		}
+		hosts[host] = config
+	}
+
+	// Clean up previous configuration templates
+	os.RemoveAll(cfg.Containerd.Registry)
+
+	// Write out new templates
+	for host, config := range hosts {
+		hostDir := filepath.Join(cfg.Containerd.Registry, host)
+		hostsFile := filepath.Join(hostDir, "hosts.toml")
+		hostsTemplate, err := templates.ParseHostsTemplateFromConfig(templates.HostsTomlTemplate, config)
+		if err != nil {
+			return err
+		}
+		if err := os.MkdirAll(hostDir, 0700); err != nil {
+			return err
+		}
+		if err := util2.WriteFile(hostsFile, hostsTemplate); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/agent/containerd/config_linux.go

@@ -13,7 +13,6 @@ import (
 	stargz "github.com/containerd/stargz-snapshotter/service"
 	"github.com/docker/docker/pkg/parsers/kernel"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/cgroups"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/version"
@@ -67,7 +66,6 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		return errors.Errorf("default runtime %s was not found", cfg.DefaultRuntime)
 	}
 
-	var containerdTemplate string
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         disableCgroup,
@@ -77,6 +75,7 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		PrivateRegistryConfig: privRegistries.Registry,
 		ExtraRuntimes:         extraRuntimes,
 		Program:               version.Program,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
 	selEnabled, selConfigured, err := selinuxStatus()
@@ -90,21 +89,11 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warnf("SELinux is enabled for "+version.Program+" but process is not running in context '%s', "+version.Program+"-selinux policy may need to be applied", SELinuxContextType)
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

pkg/agent/containerd/config_windows.go

@@ -5,11 +5,9 @@ package containerd
 
 import (
 	"context"
-	"os"
 
 	"github.com/containerd/containerd"
 	"github.com/k3s-io/k3s/pkg/agent/templates"
-	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	util3 "github.com/k3s-io/k3s/pkg/util"
 	"github.com/pkg/errors"
@@ -38,31 +36,20 @@ func setupContainerdConfig(ctx context.Context, cfg *config.Node) error {
 		logrus.Warn("SELinux isn't supported on windows")
 	}
 
-	var containerdTemplate string
-
 	containerdConfig := templates.ContainerdConfig{
 		NodeConfig:            cfg,
 		DisableCgroup:         true,
 		SystemdCgroup:         false,
 		IsRunningInUserNS:     false,
 		PrivateRegistryConfig: privRegistries.Registry,
+		NoDefaultEndpoint:     cfg.Containerd.NoDefault,
 	}
 
-	containerdTemplateBytes, err := os.ReadFile(cfg.Containerd.Template)
-	if err == nil {
-		logrus.Infof("Using containerd template at %s", cfg.Containerd.Template)
-		containerdTemplate = string(containerdTemplateBytes)
-	} else if os.IsNotExist(err) {
-		containerdTemplate = templates.ContainerdConfigTemplate
-	} else {
-		return err
-	}
-	parsedTemplate, err := templates.ParseTemplateFromConfig(containerdTemplate, containerdConfig)
-	if err != nil {
+	if err := writeContainerdConfig(cfg, containerdConfig); err != nil {
 		return err
 	}
 
-	return util2.WriteFile(cfg.Containerd.Config, parsedTemplate)
+	return writeContainerdHosts(cfg, containerdConfig)
 }
 
 func Client(address string) (*containerd.Client, error) {

pkg/agent/templates/templates.go

 package templates
 
 import (
+	"bytes"
+	"text/template"
+
 	"github.com/rancher/wharfie/pkg/registries"
 
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -17,7 +20,71 @@ type ContainerdConfig struct {
 	SystemdCgroup         bool
 	IsRunningInUserNS     bool
 	EnableUnprivileged    bool
+	NoDefaultEndpoint     bool
 	PrivateRegistryConfig *registries.Registry
 	ExtraRuntimes         map[string]ContainerdRuntimeConfig
 	Program               string
 }
+
+type RegistryEndpoint struct {
+	OverridePath bool
+	URI          string
+	Rewrites     map[string]string
+	Config       registries.RegistryConfig
+}
+
+type HostConfig struct {
+	Host      string
+	Program   string
+	Endpoints []RegistryEndpoint
+}
+
+const HostsTomlTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT.
+{{ if .Host }}server = "https://{{ .Host }}"{{ end }}
+
+{{ range $e := .Endpoints -}}
+[host."{{ $e.URI }}"]
+  capabilities = ["pull", "resolve"]
+  {{- if $e.OverridePath }}
+  override_path = true
+  {{- end }}
+{{- if $e.Config.TLS }}
+  {{- if $e.Config.TLS.CAFile }}
+  ca = [{{ printf "%q" $e.Config.TLS.CAFile }}]
+  {{- end }}
+  {{- if or $e.Config.TLS.CertFile $e.Config.TLS.KeyFile }}
+  client = [[{{ printf "%q" $e.Config.TLS.CertFile }}, {{ printf "%q" $e.Config.TLS.KeyFile }}]]
+  {{- end }}
+  {{- if $e.Config.TLS.InsecureSkipVerify }}
+  skip_verify = true
+  {{- end }}
+{{ end }}
+{{- if $e.Rewrites }}
+  [host."{{ $e.URI }}".rewrite]
+  {{- range $pattern, $replace := $e.Rewrites }}
+    "{{ $pattern }}" = "{{ $replace }}"
+  {{- end }}
+{{ end }}
+{{ end -}}
+`
+
+func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}
+
+func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
+	out := new(bytes.Buffer)
+	t := template.Must(template.New("compiled_template").Funcs(templateFuncs).Parse(templateBuffer))
+	if err := t.Execute(out, config); err != nil {
+		return "", err
+	}
+	return out.String(), nil
+}

pkg/agent/templates/templates_linux.go

@@ -3,11 +3,11 @@
 package templates
 
 import (
-	"bytes"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
 # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
 
@@ -95,20 +95,10 @@ enable_keychain = true
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
   SystemdCgroup = {{ .SystemdCgroup }}
 
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{if $v.Rewrites}}
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-{{range $pattern, $replace := $v.Rewrites}}
-    "{{$pattern}}" = "{{$replace}}"
-{{end}}
-{{end}}
-{{end}}
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "{{ .NodeConfig.Containerd.Registry }}"
 
+{{ if .PrivateRegistryConfig }}
 {{range $k, $v := .PrivateRegistryConfig.Configs }}
 {{ if $v.Auth }}
 [plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth]
@@ -117,13 +107,6 @@ enable_keychain = true
   {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
   {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
 {{end}}
-{{ if $v.TLS }}
-[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-  {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-{{end}}
 {{end}}
 {{end}}
 
@@ -136,12 +119,9 @@ enable_keychain = true
 {{end}}
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	t := template.Must(template.New("compiled_template").Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+// Linux config templates do not need fixups
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		return s
+	},
 }

pkg/agent/templates/templates_windows.go

@@ -4,16 +4,17 @@
 package templates
 
 import (
-	"bytes"
 	"net/url"
 	"strings"
 	"text/template"
 )
 
 const ContainerdConfigTemplate = `
+{{- /* */ -}}
+# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead.
 version = 2
-root = "{{ replace .NodeConfig.Containerd.Root }}"
-state = "{{ replace .NodeConfig.Containerd.State }}"
+root = {{ printf "%q" .NodeConfig.Containerd.Root }}
+state = {{ printf "%q" .NodeConfig.Containerd.State }}
 plugin_dir = ""
 disabled_plugins = []
 required_plugins = []
@@ -107,14 +108,15 @@ oom_score = 0
           privileged_without_host_devices = false
           base_runtime_spec = ""
     [plugins."io.containerd.grpc.v1.cri".cni]
-      bin_dir = "{{ replace .NodeConfig.AgentConfig.CNIBinDir }}"
-      conf_dir = "{{ replace .NodeConfig.AgentConfig.CNIConfDir }}"
+      bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }}
+      conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }}
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      config_path = ""
+      config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }}
+
+      {{ if .PrivateRegistryConfig }}
       {{range $k, $v := .PrivateRegistryConfig.Configs }}
-      [plugins."io.containerd.grpc.v1.cri".registry.auths]
       {{ if $v.Auth }}
         [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"]
           {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}}
@@ -122,35 +124,15 @@ oom_score = 0
           {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}}
           {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}}
       {{end}}
-      [plugins."io.containerd.grpc.v1.cri".registry.configs]
-      {{ if $v.TLS }}
-        [plugins."io.containerd.grpc.v1.cri".registry.configs.tls."{{$k}}"]
-          {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-          {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-          {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-          {{ if $v.TLS.InsecureSkipVerify }}insecure_skip_verify = true{{end}}
-      {{end}}
       {{end}}
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-      {{ if .PrivateRegistryConfig.Mirrors }}
-      {{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}"]
-	        endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-	    {{if $v.Rewrites}}
-	      [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{$k}}".rewrite]
-	      {{range $pattern, $replace := $v.Rewrites}}
-	        "{{$pattern}}" = "{{$replace}}"
-	      {{end}}
-	    {{end}}
       {{end}}
-    {{end}}
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
       tls_cert_file = ""
       tls_key_file = ""
   [plugins."io.containerd.internal.v1.opt"]
-    path = "{{ replace .NodeConfig.Containerd.Opt }}"
+    path = {{ printf "%q" .NodeConfig.Containerd.Opt }}
   [plugins."io.containerd.internal.v1.restart"]
     interval = "10s"
   [plugins."io.containerd.metadata.v1.bolt"]
@@ -161,27 +143,16 @@ oom_score = 0
     default = ["windows", "windows-lcow"]
 `
 
-func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, error) {
-	out := new(bytes.Buffer)
-	funcs := template.FuncMap{
-		"replace": func(s string) string {
-			return strings.ReplaceAll(s, "\\", "\\\\")
-		},
-		"deschemify": func(s string) string {
-			if strings.HasPrefix(s, "npipe:") {
-				u, err := url.Parse(s)
-				if err != nil {
-					return ""
-				}
-				return u.Path
+// Windows config templates need named pipe addresses fixed up
+var templateFuncs = template.FuncMap{
+	"deschemify": func(s string) string {
+		if strings.HasPrefix(s, "npipe:") {
+			u, err := url.Parse(s)
+			if err != nil {
+				return ""
 			}
-			return s
-		},
-	}
-	t := template.Must(template.New("compiled_template").Funcs(funcs).Parse(templateBuffer))
-	template.Must(t.New("base").Parse(ContainerdConfigTemplate))
-	if err := t.Execute(out, config); err != nil {
-		return "", err
-	}
-	return out.String(), nil
+			return u.Path
+		}
+		return s
+	},
 }

pkg/cli/cmds/agent.go

@@ -26,6 +26,7 @@ type Agent struct {
 	PauseImage               string
 	Snapshotter              string
 	Docker                   bool
+	ContainerdNoDefault      bool
 	ContainerRuntimeEndpoint string
 	DefaultRuntime           string
 	ImageServiceEndpoint     string
@@ -220,6 +221,11 @@ var (
 		Usage:       "(agent/networking) (experimental) Disable the agent's client-side load-balancer and connect directly to the configured server address",
 		Destination: &AgentConfig.DisableLoadBalancer,
 	}
+	DisableDefaultRegistryEndpointFlag = &cli.BoolFlag{
+		Name:        "disable-default-registry-endpoint",
+		Usage:       "(agent/containerd) Disables containerd's fallback default registry endpoint when a mirror is configured for that registry",
+		Destination: &AgentConfig.ContainerdNoDefault,
+	}
 )
 
 func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
@@ -269,6 +275,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			PauseImageFlag,
 			SnapshotterFlag,
 			PrivateRegistryFlag,
+			DisableDefaultRegistryEndpointFlag,
 			AirgapExtraRegistryFlag,
 			NodeIPFlag,
 			NodeExternalIPFlag,

pkg/cli/cmds/server.go

@@ -499,6 +499,7 @@ var ServerFlags = []cli.Flag{
 	CRIEndpointFlag,
 	DefaultRuntimeFlag,
 	ImageServiceEndpointFlag,
+	DisableDefaultRegistryEndpointFlag,
 	PauseImageFlag,
 	SnapshotterFlag,
 	PrivateRegistryFlag,

pkg/daemons/config/types.go

@@ -68,6 +68,8 @@ type Containerd struct {
 	Template      string
 	BlockIOConfig string
 	RDTConfig     string
+	Registry      string
+	NoDefault     bool
 	SELinux       bool
 	Debug         bool
 }