Merge pull request #8504 from manuelbuil/IPPrecedence127

manifests/coredns.yaml

@@ -205,6 +205,7 @@ spec:
   selector:
     k8s-app: kube-dns
   clusterIP: %{CLUSTER_DNS}%
+  clusterIPs: %{CLUSTER_DNS_LIST}%
   ports:
   - name: dns
     port: 53
@@ -215,3 +216,4 @@ spec:
   - name: metrics
     port: 9153
     protocol: TCP
+  ipFamilyPolicy: %{CLUSTER_DNS_IPFAMILYPOLICY}%

manifests/metrics-server/metrics-server-service.yaml

@@ -15,3 +15,4 @@ spec:
     name: https
     protocol: TCP
     targetPort: https
+  ipFamilyPolicy: PreferDualStack

pkg/agent/config/config.go

@@ -557,22 +557,18 @@ func get(ctx context.Context, envInfo *cmds.Agent, proxy proxy.Proxy) (*config.N
 	nodeConfig.Certificate = servingCert
 
 	nodeConfig.AgentConfig.NodeIPs = nodeIPs
-	nodeIP, listenAddress, _, err := util.GetFirstIP(nodeIPs)
+	listenAddress, _, _, err := util.GetDefaultAddresses(nodeIPs[0])
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-ip")
 	}
-	nodeConfig.AgentConfig.NodeIP = nodeIP.String()
+	nodeConfig.AgentConfig.NodeIP = nodeIPs[0].String()
 	nodeConfig.AgentConfig.ListenAddress = listenAddress
 	nodeConfig.AgentConfig.NodeExternalIPs = nodeExternalIPs
 
 	// if configured, set NodeExternalIP to the first IPv4 address, for legacy clients
 	// unless only IPv6 address given
 	if len(nodeConfig.AgentConfig.NodeExternalIPs) > 0 {
-		nodeExternalIP, _, _, err := util.GetFirstIP(nodeConfig.AgentConfig.NodeExternalIPs)
-		if err != nil {
-			return nil, errors.Wrap(err, "cannot configure IPv4/IPv6 node-external-ip")
-		}
-		nodeConfig.AgentConfig.NodeExternalIP = nodeExternalIP.String()
+		nodeConfig.AgentConfig.NodeExternalIP = nodeConfig.AgentConfig.NodeExternalIPs[0].String()
 	}
 
 	nodeConfig.NoFlannel = nodeConfig.FlannelBackend == config.FlannelBackendNone

pkg/agent/flannel/setup.go

@@ -188,12 +188,13 @@ func createFlannelConf(nodeConfig *config.Node) error {
 		confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "false")
 		confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
 	} else if netMode == (ipv4 + ipv6) {
-		confJSON = strings.ReplaceAll(confJSON, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String())
 		confJSON = strings.ReplaceAll(confJSON, "%IPV6_ENABLED%", "true")
 		for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
 			if utilsnet.IsIPv6(cidr.IP) {
 				// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
 				confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
+			} else {
+				confJSON = strings.ReplaceAll(confJSON, "%CIDR%", cidr.String())
 			}
 		}
 	} else {

pkg/agent/run.go

@@ -69,12 +69,17 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
 	serviceIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ServiceCIDR)
 	clusterIPv6 := utilsnet.IsIPv6CIDR(nodeConfig.AgentConfig.ClusterCIDR)
 	nodeIPv6 := utilsnet.IsIPv6String(nodeConfig.AgentConfig.NodeIP)
+
+	// check that cluster-cidr and service-cidr have the same IP versions
 	if (serviceIPv6 != clusterIPv6) || (dualCluster != dualService) || (serviceIPv4 != clusterIPv4) {
 		return fmt.Errorf("cluster-cidr: %v and service-cidr: %v, must share the same IP version (IPv4, IPv6 or dual-stack)", nodeConfig.AgentConfig.ClusterCIDRs, nodeConfig.AgentConfig.ServiceCIDRs)
 	}
-	if (clusterIPv6 && !nodeIPv6) || (dualCluster && !dualNode) || (clusterIPv4 && !nodeIPv4) {
+
+	// check that node-ip has the IP versions set in cluster-cidr
+	if (clusterIPv6 && !(nodeIPv6 || dualNode)) || (dualCluster && !dualNode) || (clusterIPv4 && !(nodeIPv4 || dualNode)) {
 		return fmt.Errorf("cluster-cidr: %v and node-ip: %v, must share the same IP version (IPv4, IPv6 or dual-stack)", nodeConfig.AgentConfig.ClusterCIDRs, nodeConfig.AgentConfig.NodeIPs)
 	}
+
 	enableIPv6 := dualCluster || clusterIPv6
 	enableIPv4 := dualCluster || clusterIPv4
 

pkg/cli/server/server.go

@@ -298,14 +298,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.SANs = append(serverConfig.ControlConfig.SANs, serverConfig.ControlConfig.AdvertiseIP)
 	}
 
-	// configure ClusterIPRanges
-	_, _, IPv6only, _ := util.GetFirstIP(nodeIPs)
+	// configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it
+	_, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0])
 	if len(cmds.ServerConfig.ClusterCIDR) == 0 {
-		clusterCIDR := "10.42.0.0/16"
-		if IPv6only {
-			clusterCIDR = "fd00:42::/56"
-		}
-		cmds.ServerConfig.ClusterCIDR.Set(clusterCIDR)
+		cmds.ServerConfig.ClusterCIDR.Set(defaultClusterCIDR)
 	}
 	for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ClusterCIDR) {
 		_, parsed, err := net.ParseCIDR(cidr)
@@ -315,21 +311,12 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.ClusterIPRanges = append(serverConfig.ControlConfig.ClusterIPRanges, parsed)
 	}
 
-	// set ClusterIPRange to the first IPv4 block, for legacy clients
-	// unless only IPv6 range given
-	clusterIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ClusterIPRanges)
-	if err != nil {
-		return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-cidr")
-	}
-	serverConfig.ControlConfig.ClusterIPRange = clusterIPRange
+	// set ClusterIPRange to the first address (first defined IPFamily is preferred)
+	serverConfig.ControlConfig.ClusterIPRange = serverConfig.ControlConfig.ClusterIPRanges[0]
 
-	// configure ServiceIPRanges
+	// configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it
 	if len(cmds.ServerConfig.ServiceCIDR) == 0 {
-		serviceCIDR := "10.43.0.0/16"
-		if IPv6only {
-			serviceCIDR = "fd00:43::/112"
-		}
-		cmds.ServerConfig.ServiceCIDR.Set(serviceCIDR)
+		cmds.ServerConfig.ServiceCIDR.Set(defaultServiceCIDR)
 	}
 	for _, cidr := range util.SplitStringSlice(cmds.ServerConfig.ServiceCIDR) {
 		_, parsed, err := net.ParseCIDR(cidr)
@@ -339,13 +326,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.ServiceIPRanges = append(serverConfig.ControlConfig.ServiceIPRanges, parsed)
 	}
 
-	// set ServiceIPRange to the first IPv4 block, for legacy clients
-	// unless only IPv6 range given
-	serviceIPRange, err := util.GetFirstNet(serverConfig.ControlConfig.ServiceIPRanges)
-	if err != nil {
-		return errors.Wrap(err, "cannot configure IPv4/IPv6 service-cidr")
-	}
-	serverConfig.ControlConfig.ServiceIPRange = serviceIPRange
+	// set ServiceIPRange to the first address (first defined IPFamily is preferred)
+	serverConfig.ControlConfig.ServiceIPRange = serverConfig.ControlConfig.ServiceIPRanges[0]
 
 	serverConfig.ControlConfig.ServiceNodePortRange, err = utilnet.ParsePortRange(cfg.ServiceNodePortRange)
 	if err != nil {
@@ -364,12 +346,13 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	// If there are no IPv4 ServiceCIDRs, an IPv6 ServiceCIDRs will be used.
 	// If neither of IPv4 or IPv6 are found an error is raised.
 	if len(cmds.ServerConfig.ClusterDNS) == 0 {
-		clusterDNS, err := utilsnet.GetIndexedIP(serverConfig.ControlConfig.ServiceIPRange, 10)
-		if err != nil {
-			return errors.Wrap(err, "cannot configure default cluster-dns address")
+		for _, svcCIDR := range serverConfig.ControlConfig.ServiceIPRanges {
+			clusterDNS, err := utilsnet.GetIndexedIP(svcCIDR, 10)
+			if err != nil {
+				return errors.Wrap(err, "cannot configure default cluster-dns address")
+			}
+			serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, clusterDNS)
 		}
-		serverConfig.ControlConfig.ClusterDNS = clusterDNS
-		serverConfig.ControlConfig.ClusterDNSs = []net.IP{serverConfig.ControlConfig.ClusterDNS}
 	} else {
 		for _, ip := range util.SplitStringSlice(cmds.ServerConfig.ClusterDNS) {
 			parsed := net.ParseIP(ip)
@@ -378,15 +361,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 			}
 			serverConfig.ControlConfig.ClusterDNSs = append(serverConfig.ControlConfig.ClusterDNSs, parsed)
 		}
-		// Set ClusterDNS to the first IPv4 address, for legacy clients
-		// unless only IPv6 range given
-		clusterDNS, _, _, err := util.GetFirstIP(serverConfig.ControlConfig.ClusterDNSs)
-		if err != nil {
-			return errors.Wrap(err, "cannot configure IPv4/IPv6 cluster-dns address")
-		}
-		serverConfig.ControlConfig.ClusterDNS = clusterDNS
 	}
 
+	serverConfig.ControlConfig.ClusterDNS = serverConfig.ControlConfig.ClusterDNSs[0]
+
 	if err := validateNetworkConfiguration(serverConfig); err != nil {
 		return err
 	}
@@ -577,18 +555,6 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 
 // validateNetworkConfig ensures that the network configuration values make sense.
 func validateNetworkConfiguration(serverConfig server.Config) error {
-	// Dual-stack operation requires fairly extensive manual configuration at the moment - do some
-	// preflight checks to make sure that the user isn't trying to use flannel/npc, or trying to
-	// enable dual-stack DNS (which we don't currently support since it's not easy to template)
-	dualDNS, err := utilsnet.IsDualStackIPs(serverConfig.ControlConfig.ClusterDNSs)
-	if err != nil {
-		return errors.Wrap(err, "failed to validate cluster-dns")
-	}
-
-	if dualDNS == true {
-		return errors.New("dual-stack cluster-dns is not supported")
-	}
-
 	switch serverConfig.ControlConfig.EgressSelectorMode {
 	case config.EgressSelectorModeCluster, config.EgressSelectorModePod:
 	case config.EgressSelectorModeAgent, config.EgressSelectorModeDisabled:

pkg/cluster/cluster.go

@@ -10,10 +10,10 @@ import (
 	"github.com/k3s-io/k3s/pkg/cluster/managed"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
-	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	utilsnet "k8s.io/utils/net"
 )
 
 type Cluster struct {
@@ -54,8 +54,7 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
 			clientURL.Host = clientURL.Hostname() + ":2379"
 			clientURLs = append(clientURLs, clientURL.String())
 		}
-		IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.config.ServiceIPRanges)
-		etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0], IPv6OnlyService)
+		etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0], utilsnet.IsIPv6CIDR(c.config.ServiceIPRanges[0]))
 		if err != nil {
 			return nil, err
 		}

pkg/daemons/config/types.go

@@ -10,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/k3s-io/k3s/pkg/util"
 	"github.com/k3s-io/kine/pkg/endpoint"
 	"github.com/rancher/wrangler/pkg/generated/controllers/core"
 	"github.com/rancher/wrangler/pkg/leader"
@@ -251,7 +250,7 @@ func (c *Control) BindAddressOrLoopback(chooseHostInterface, urlSafe bool) strin
 // service CIDRs indicate an IPv4/Dual-Stack or IPv6 only cluster. If the urlSafe
 // parameter is true, IPv6 addresses are enclosed in square brackets, as per RFC2732.
 func (c *Control) Loopback(urlSafe bool) string {
-	if IPv6OnlyService, _ := util.IsIPv6OnlyCIDRs(c.ServiceIPRanges); IPv6OnlyService {
+	if utilsnet.IsIPv6CIDR(c.ServiceIPRange) {
 		if urlSafe {
 			return "[::1]"
 		}

pkg/server/server.go

@@ -272,8 +272,16 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
 		return err
 	}
 	dataDir = filepath.Join(controlConfig.DataDir, "manifests")
+
+	dnsIPFamilyPolicy := "PreferDualStack"
+	if len(controlConfig.ClusterDNSs) == 1 {
+		dnsIPFamilyPolicy = "SingleStack"
+	}
+
 	templateVars := map[string]string{
 		"%{CLUSTER_DNS}%":                 controlConfig.ClusterDNS.String(),
+		"%{CLUSTER_DNS_LIST}%":            fmt.Sprintf("[%s]", util.JoinIPs(controlConfig.ClusterDNSs)),
+		"%{CLUSTER_DNS_IPFAMILYPOLICY}%":  dnsIPFamilyPolicy,
 		"%{CLUSTER_DOMAIN}%":              controlConfig.ClusterDomain,
 		"%{DEFAULT_LOCAL_STORAGE_PATH}%":  controlConfig.DefaultLocalStoragePath,
 		"%{SYSTEM_DEFAULT_REGISTRY}%":     registryTemplate(controlConfig.SystemDefaultRegistry),

pkg/util/net.go

@@ -31,9 +31,9 @@ func JoinIPNets(elems []*net.IPNet) string {
 	return strings.Join(strs, ",")
 }
 
-// GetFirst4Net returns the first IPv4 network from the list of IP networks.
+// getFirst4Net returns the first IPv4 network from the list of IP networks.
 // If no IPv4 addresses are found, an error is raised.
-func GetFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
+func getFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
 	for _, elem := range elems {
 		if elem == nil || elem.IP.To4() == nil {
 			continue
@@ -43,9 +43,9 @@ func GetFirst4Net(elems []*net.IPNet) (*net.IPNet, error) {
 	return nil, errors.New("no IPv4 CIDRs found")
 }
 
-// GetFirst4 returns the first IPv4 address from the list of IP addresses.
+// getFirst4 returns the first IPv4 address from the list of IP addresses.
 // If no IPv4 addresses are found, an error is raised.
-func GetFirst4(elems []net.IP) (net.IP, error) {
+func getFirst4(elems []net.IP) (net.IP, error) {
 	for _, elem := range elems {
 		if elem == nil || elem.To4() == nil {
 			continue
@@ -64,7 +64,7 @@ func GetFirst4String(elems []string) (string, error) {
 			ips = append(ips, net.ParseIP(v))
 		}
 	}
-	ip, err := GetFirst4(ips)
+	ip, err := getFirst4(ips)
 	if err != nil {
 		return "", err
 	}
@@ -82,9 +82,9 @@ func JoinIP4Nets(elems []*net.IPNet) string {
 	return strings.Join(strs, ",")
 }
 
-// GetFirst6 returns the first IPv6 address from the list of IP addresses.
+// getFirst6 returns the first IPv6 address from the list of IP addresses.
 // If no IPv6 addresses are found, an error is raised.
-func GetFirst6(elems []net.IP) (net.IP, error) {
+func getFirst6(elems []net.IP) (net.IP, error) {
 	for _, elem := range elems {
 		if elem != nil && netutils.IsIPv6(elem) {
 			return elem, nil
@@ -93,9 +93,9 @@ func GetFirst6(elems []net.IP) (net.IP, error) {
 	return nil, errors.New("no IPv6 address found")
 }
 
-// GetFirst6Net returns the first IPv4 network from the list of IP networks.
+// getFirst6Net returns the first IPv4 network from the list of IP networks.
 // If no IPv6 addresses are found, an error is raised.
-func GetFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
+func getFirst6Net(elems []*net.IPNet) (*net.IPNet, error) {
 	for _, elem := range elems {
 		if elem != nil && netutils.IsIPv6(elem.IP) {
 			return elem, nil
@@ -113,7 +113,7 @@ func GetFirst6String(elems []string) (string, error) {
 			ips = append(ips, net.ParseIP(v))
 		}
 	}
-	ip, err := GetFirst6(ips)
+	ip, err := getFirst6(ips)
 	if err != nil {
 		return "", err
 	}
@@ -133,7 +133,7 @@ func JoinIP6Nets(elems []*net.IPNet) string {
 
 // GetHostnameAndIPs takes a node name and list of IPs, usually from CLI args.
 // If set, these are used to return the node's name and addresses. If not set,
-// the system hostname and primary interface address are returned instead.
+// the system hostname and primary interface addresses are returned instead.
 func GetHostnameAndIPs(name string, nodeIPs cli.StringSlice) (string, []net.IP, error) {
 	ips := []net.IP{}
 	if len(nodeIPs) == 0 {
@@ -202,37 +202,26 @@ func GetFirstValidIPString(s cli.StringSlice) string {
 	return ""
 }
 
-// GetFirstIP returns the first IPv4 address from the list of IP addresses.
-// If no IPv4 addresses are found, returns the first IPv6 address
-// if neither of IPv4 or IPv6 are found an error is raised.
-// Additionally matching listen address and IP version is returned.
-func GetFirstIP(nodeIPs []net.IP) (net.IP, string, bool, error) {
-	nodeIP, err := GetFirst4(nodeIPs)
-	ListenAddress := "0.0.0.0"
-	IPv6only := false
-	if err != nil {
-		nodeIP, err = GetFirst6(nodeIPs)
-		if err != nil {
-			return nil, "", false, err
-		}
-		ListenAddress = "::"
-		IPv6only = true
+// GetFirstIP checks what is the IPFamily of the first item. Based on that, returns a set of values
+func GetDefaultAddresses(nodeIP net.IP) (string, string, string, error) {
+
+	if netutils.IsIPv4(nodeIP) {
+		ListenAddress := "0.0.0.0"
+		clusterCIDR := "10.42.0.0/16"
+		serviceCIDR := "10.43.0.0/16"
+
+		return ListenAddress, clusterCIDR, serviceCIDR, nil
 	}
-	return nodeIP, ListenAddress, IPv6only, nil
-}
 
-// GetFirstNet returns the first IPv4 network from the list of IP networks.
-// If no IPv4 addresses are found, returns the first IPv6 address
-// if neither of IPv4 or IPv6 are found an error is raised.
-func GetFirstNet(elems []*net.IPNet) (*net.IPNet, error) {
-	serviceIPRange, err := GetFirst4Net(elems)
-	if err != nil {
-		serviceIPRange, err = GetFirst6Net(elems)
-		if err != nil {
-			return nil, err
-		}
+	if netutils.IsIPv6(nodeIP) {
+		ListenAddress := "::"
+		clusterCIDR := "fd00:42::/56"
+		serviceCIDR := "fd00:43::/112"
+
+		return ListenAddress, clusterCIDR, serviceCIDR, nil
 	}
-	return serviceIPRange, nil
+
+	return "", "", "", fmt.Errorf("ip: %v is not ipv4 or ipv6", nodeIP)
 }
 
 // GetFirstString returns the first IP4 address from a list of IP address strings.
@@ -251,32 +240,6 @@ func GetFirstString(elems []string) (string, bool, error) {
 	return ip, IPv6only, nil
 }
 
-// IsIPv6OnlyCIDRs returns if
-// - all are valid cidrs
-// - at least one cidr from v6 family is found
-// - v4 family cidr is not found
-func IsIPv6OnlyCIDRs(cidrs []*net.IPNet) (bool, error) {
-	v4Found := false
-	v6Found := false
-	for _, cidr := range cidrs {
-		if cidr == nil {
-			return false, fmt.Errorf("cidr %v is invalid", cidr)
-		}
-
-		if v4Found && v6Found {
-			continue
-		}
-
-		if cidr.IP != nil && cidr.IP.To4() == nil {
-			v6Found = true
-			continue
-		}
-		v4Found = true
-	}
-
-	return !v4Found && v6Found, nil
-}
-
 // IPToIPNet converts an IP to an IPNet, using a fully filled mask appropriate for the address family.
 func IPToIPNet(ip net.IP) (*net.IPNet, error) {
 	address := ip.String()