Merge pull request #3906 from manuelbuil/dual-stack

go.mod

@@ -75,7 +75,6 @@ replace (
 
 require (
 	github.com/Microsoft/hcsshim v0.8.20
-	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
 	github.com/containerd/cgroups v1.0.1
 	github.com/containerd/containerd v1.5.5
 	github.com/containerd/fuse-overlayfs-snapshotter v1.0.3
@@ -84,7 +83,7 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
-	github.com/flannel-io/flannel v0.14.0
+	github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/golangplus/testing v1.0.0 // indirect

go.sum

@@ -119,9 +119,8 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
-github.com/bronze1man/goStrongswanVici v0.0.0-20171013065002-4d72634a2f11/go.mod h1:c+n7HXa5FxzR8GDsmu773UtbtrmKvMVerLVQeEbnzAE=
-github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 h1:udTfdeYqe866Z5mxTaEm5irSJK2vupyxwBOHAYEVtJo=
-github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=
+github.com/bronze1man/goStrongswanVici v0.0.0-20201105010758-936f38b697fd h1:qn6a8rGrW+7p4ghypmYHZUKewXURuUDYxKqZxEoFjPc=
+github.com/bronze1man/goStrongswanVici v0.0.0-20201105010758-936f38b697fd/go.mod h1:fWUtBEPt2yjrr3WFhOqvajM8JSEU8bEeBcoeSCsKRpc=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/canonical/go-dqlite v1.5.1 h1:1YjtIrFsC1A3XlgsX38ARAiKhvkZS63PqsEd8z3T4yU=
 github.com/canonical/go-dqlite v1.5.1/go.mod h1:wp00vfMvPYgNCyxcPdHB5XExmDoCGoPUGymloAQT17Y=
@@ -284,8 +283,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/flannel-io/flannel v0.14.0 h1:7RmZN6G0L/mJwdzsLrAjfQKtKokfD1NcncPEfSqr+ac=
-github.com/flannel-io/flannel v0.14.0/go.mod h1:qZhrC3nxQudgshBtTb5rBqFxeYtQGRa4AQGwKi4u4Ds=
+github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc h1:t/L/tIYcAayH7XICVYtAscY/PXaJDKXsKheAMCtiKS0=
+github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc/go.mod h1:fIcQpjXVBEE22oxqfZN0cXN0ZInsMDqTF5YoeGo6DgY=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=

install.sh

@@ -651,6 +651,7 @@ ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore;
 done
 ip link delete cni0
 ip link delete flannel.1
+ip link delete flannel-v6.1
 rm -rf /var/lib/cni/
 iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
 EOF

pkg/agent/flannel/flannel.go

@@ -39,13 +39,13 @@ const (
 	subnetFile = "/run/flannel/subnet.env"
 )
 
-func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string) error {
-	extIface, err := LookupExtInterface(flannelIface)
+func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kubeConfigFile string, netMode int) error {
+	extIface, err := LookupExtInterface(flannelIface, netMode)
 	if err != nil {
 		return err
 	}
 
-	sm, err := kube.NewSubnetManager(ctx, "", kubeConfigFile, "flannel.alpha.coreos.com", flannelConf)
+	sm, err := kube.NewSubnetManager(ctx, "", kubeConfigFile, "flannel.alpha.coreos.com", flannelConf, false)
 	if err != nil {
 		return err
 	}
@@ -71,7 +71,7 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 	go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), 60)
 	go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), 50)
 
-	if err := WriteSubnetFile(subnetFile, config.Network, true, bn); err != nil {
+	if err := WriteSubnetFile(subnetFile, config.Network, config.IPv6Network, true, bn); err != nil {
 		// Continue, even though it failed.
 		log.Warningf("Failed to write subnet file: %s", err)
 	} else {
@@ -84,8 +84,9 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 	return nil
 }
 
-func LookupExtInterface(iface *net.Interface) (*backend.ExternalInterface, error) {
+func LookupExtInterface(iface *net.Interface, netMode int) (*backend.ExternalInterface, error) {
 	var ifaceAddr net.IP
+	var ifacev6Addr net.IP
 	var err error
 
 	if iface == nil {
@@ -102,8 +103,14 @@ func LookupExtInterface(iface *net.Interface) (*backend.ExternalInterface, error
 		return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
 	}
 
-	log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
+	if netMode == (ipv4 + ipv6) {
+		ifacev6Addr, err = ip.GetInterfaceIP6Addr(iface)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find IPv6 address for interface %s", iface.Name)
+		}
 
+		log.Infof("Using ipv6 address %s", ifacev6Addr)
+	}
 	if iface.MTU == 0 {
 		return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
 	}
@@ -111,11 +118,13 @@ func LookupExtInterface(iface *net.Interface) (*backend.ExternalInterface, error
 	return &backend.ExternalInterface{
 		Iface:       iface,
 		IfaceAddr:   ifaceAddr,
+		IfaceV6Addr: ifacev6Addr,
 		ExtAddr:     ifaceAddr,
+		ExtV6Addr:   ifacev6Addr,
 	}, nil
 }
 
-func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
+func WriteSubnetFile(path string, nw ip.IP4Net, nwv6 ip.IP6Net, ipMasq bool, bn backend.Network) error {
 	dir, name := filepath.Split(path)
 	os.MkdirAll(dir, 0755)
 
@@ -132,6 +141,14 @@ func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network)
 
 	fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
 	fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
+
+	if nwv6.String() != emptyIPv6Network {
+		snv6 := bn.Lease().IPv6Subnet
+		snv6.IncrementIP()
+		fmt.Fprintf(f, "FLANNEL_IPV6_NETWORK=%s\n", nwv6)
+		fmt.Fprintf(f, "FLANNEL_IPV6_SUBNET=%s\n", snv6)
+	}
+
 	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
 	_, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
 	f.Close()

pkg/agent/flannel/setup.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"strings"
@@ -15,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	utilsnet "k8s.io/utils/net"
 )
 
 const (
@@ -42,6 +44,8 @@ const (
 
 	flannelConf = `{
 	"Network": "%CIDR%",
+	"EnableIPv6": %DUALSTACK%,
+	"IPv6Network": "%CIDR_IPV6%",
 	"Backend": %backend%
 }
 `
@@ -68,6 +72,11 @@ const (
 	"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET persistent-keepalive 25",
 	"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
 }`
+
+	emptyIPv6Network = "::/0"
+
+	ipv4 = iota
+	ipv6
 )
 
 func Prepare(ctx context.Context, nodeConfig *config.Node) error {
@@ -95,8 +104,13 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) e
 	}
 	logrus.Info("Node CIDR assigned for: " + nodeName)
 
+	netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
+	if err != nil {
+		logrus.Fatalf("Error checking netMode")
+		return err
+	}
 	go func() {
-		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet, netMode)
 		if err != nil && !errors.Is(err, context.Canceled) {
 			logrus.Fatalf("flannel exited: %v", err)
 		}
@@ -142,6 +156,24 @@ func createFlannelConf(nodeConfig *config.Node) error {
 	}
 	confJSON = strings.ReplaceAll(confJSON, "%backend%", backendConf)
 
+	netMode, err := findNetMode(nodeConfig.AgentConfig.ClusterCIDRs)
+	if err != nil {
+		logrus.Fatalf("Error checking netMode")
+		return err
+	}
+
+	if netMode == (ipv4 + ipv6) {
+		confJSON = strings.ReplaceAll(confJSON, "%DUALSTACK%", "true")
+		for _, cidr := range nodeConfig.AgentConfig.ClusterCIDRs {
+			if utilsnet.IsIPv6(cidr.IP) {
+				// Only one ipv6 range available. This might change in future: https://github.com/kubernetes/enhancements/issues/2593
+				confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", cidr.String())
+			}
+		}
+	} else {
+		confJSON = strings.ReplaceAll(confJSON, "%DUALSTACK%", "false")
+		confJSON = strings.ReplaceAll(confJSON, "%CIDR_IPV6%", emptyIPv6Network)
+	}
 	return util.WriteFile(nodeConfig.FlannelConf, confJSON)
 }
 
@@ -172,3 +204,24 @@ func setupStrongSwan(nodeConfig *config.Node) error {
 	// make new strongswan link
 	return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir)
 }
+
+// fundNetMode returns the mode (ipv4, ipv6 or dual-stack) in which flannel is operating
+func findNetMode(cidrs []*net.IPNet) (int, error) {
+	dualStack, err := utilsnet.IsDualStackCIDRs(cidrs)
+	if err != nil {
+		return 0, err
+	}
+	if dualStack {
+		return ipv4 + ipv6, nil
+	}
+
+	for _, cidr := range cidrs {
+		if utilsnet.IsIPv4CIDR(cidr) {
+			return ipv4, nil
+		}
+		if utilsnet.IsIPv6CIDR(cidr) {
+			return ipv6, nil
+		}
+	}
+	return 0, errors.New("Failed checking netMode")
+}

pkg/agent/flannel/setup_test.go

+package flannel
+
+import (
+	"io/ioutil"
+	"net"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
+)
+
+func stringToCIDR(s string) []*net.IPNet {
+	var netCidrs []*net.IPNet
+	for _, v := range strings.Split(s, ",") {
+		_, parsed, _ := net.ParseCIDR(v)
+		netCidrs = append(netCidrs, parsed)
+	}
+	return netCidrs
+}
+
+func Test_findNetMode(t *testing.T) {
+	tests := []struct {
+		name    string
+		args    string
+		want    int
+		wantErr bool
+	}{
+		{"dual-stack", "10.42.0.0/16,2001:cafe:22::/56", ipv4 + ipv6, false},
+		{"ipv4 only", "10.42.0.0/16", ipv4, false},
+		{"ipv6 only", "2001:cafe:42:0::/56", ipv6, false},
+		{"wrong input", "wrong", 0, true},
+	}
+	for _, tt := range tests {
+
+		t.Run(tt.name, func(t *testing.T) {
+			netCidrs := stringToCIDR(tt.args)
+			got, err := findNetMode(netCidrs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("findNetMode() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("findNetMode() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_createFlannelConf(t *testing.T) {
+	tests := []struct {
+		name       string
+		args       string
+		wantConfig []string
+		wantErr    bool
+	}{
+		{"dual-stack", "10.42.0.0/16,2001:cafe:22::/56", []string{"\"Network\": \"10.42.0.0/16\"", "\"IPv6Network\": \"2001:cafe:22::/56\"", "\"EnableIPv6\": true"}, false},
+		{"ipv4 only", "10.42.0.0/16", []string{"\"Network\": \"10.42.0.0/16\"", "\"IPv6Network\": \"::/0\"", "\"EnableIPv6\": false"}, false},
+	}
+	var containerd = config.Containerd{}
+	for _, tt := range tests {
+		var agent = config.Agent{}
+		agent.ClusterCIDR = stringToCIDR(tt.args)[0]
+		agent.ClusterCIDRs = stringToCIDR(tt.args)
+		var nodeConfig = &config.Node{Docker: false, ContainerRuntimeEndpoint: "", NoFlannel: false, SELinux: false, FlannelBackend: "vxlan", FlannelConf: "test_file", FlannelConfOverride: false, FlannelIface: nil, Containerd: containerd, Images: "", AgentConfig: agent, Token: "", Certificate: nil, ServerHTTPSPort: 0}
+
+		t.Run(tt.name, func(t *testing.T) {
+			if err := createFlannelConf(nodeConfig); (err != nil) != tt.wantErr {
+				t.Errorf("createFlannelConf() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			data, err := ioutil.ReadFile("test_file")
+			if err != nil {
+				t.Errorf("Something went wrong when reading the flannel config file")
+			}
+			for _, config := range tt.wantConfig {
+				isExist, _ := regexp.Match(config, data)
+				if !isExist {
+					t.Errorf("Config is wrong, %s is not present", config)
+				}
+			}
+		})
+	}
+}

pkg/cli/server/server.go

@@ -467,8 +467,8 @@ func validateNetworkConfiguration(serverConfig server.Config) error {
 		return errors.Wrap(err, "failed to validate cluster-dns")
 	}
 
-	if (serverConfig.ControlConfig.FlannelBackend != "none" || serverConfig.ControlConfig.DisableNPC == false) && (dualCluster || dualService) {
-		return errors.New("flannel CNI and network policy enforcement are not compatible with dual-stack operation; server must be restarted with --flannel-backend=none --disable-network-policy and an alternative CNI plugin deployed")
+	if (serverConfig.ControlConfig.DisableNPC == false) && (dualCluster || dualService) {
+		return errors.New("network policy enforcement is not compatible with dual-stack operation; server must be restarted with --disable-network-policy")
 	}
 	if dualDNS == true {
 		return errors.New("dual-stack cluster-dns is not supported")

tests/integration/dual_stack_int_test.go

+package integration
+
+import (
+	"strings"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+	testutil "github.com/rancher/k3s/tests/util"
+)
+
+var dualStackServerArgs = []string{"--cluster-init", "--cluster-cidr 10.42.0.0/16,2001:cafe:42:0::/56", "--service-cidr 10.43.0.0/16,2001:cafe:42:1::/112"}
+var _ = BeforeSuite(func() {
+	if !testutil.IsExistingServer() {
+		var err error
+		server, err = testutil.K3sStartServer(dualStackServerArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
+})
+
+var _ = Describe("dual stack", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(dualStackServerArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(dualStackServerArgs, " "))
+		}
+	})
+	When("a ipv4 and ipv6 cidr is present", func() {
+		It("starts up with no problems", func() {
+			Eventually(func() (string, error) {
+				return testutil.K3sCmd("kubectl", "get", "pods", "-A")
+			}, "90s", "1s").Should(MatchRegexp("kube-system.+traefik.+1\\/1.+Running"))
+		})
+		It("creates pods with two IPs", func() {
+			podname, err := testutil.K3sCmd("kubectl", "get", "pods", "-nkube-system", "-ojsonpath={.items[?(@.metadata.labels.app\\.kubernetes\\.io/name==\"traefik\")].metadata.name}")
+			Expect(err).NotTo(HaveOccurred())
+			result, err := testutil.K3sCmd("kubectl", "exec", podname, "-nkube-system", "--", "ip", "a")
+			Expect(result).To(ContainSubstring("2001:cafe:42:"))
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(server)).To(Succeed())
+	}
+})
+
+func Test_IntegrationDualStack(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "Dual-Stack Suite", []Reporter{
+		reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
+	})
+}

vendor/github.com/bronze1man/goStrongswanVici/clientConn.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"sync"
 	"time"
 )
 
@@ -22,11 +23,16 @@ type ClientConn struct {
 	// ReadTimeout specifies a time limit for requests made
 	// by this client.
 	ReadTimeout time.Duration
+
+	lock sync.RWMutex
 }
 
 func (c *ClientConn) Close() error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
 	close(c.responseChan)
 	c.lastError = io.ErrClosedPipe
+
 	return c.conn.Close()
 }
 
@@ -38,6 +44,7 @@ func NewClientConn(conn net.Conn) (client *ClientConn) {
 		ReadTimeout:   DefaultReadTimeout,
 	}
 	go client.readThread()
+
 	return client
 }
 
@@ -47,6 +54,7 @@ func NewClientConnFromDefaultSocket() (client *ClientConn, err error) {
 	if err != nil {
 		return
 	}
+
 	return NewClientConn(conn), nil
 }
 
@@ -58,16 +66,23 @@ func (c *ClientConn) Request(apiname string, request map[string]interface{}) (re
 	})
 	if err != nil {
 		fmt.Printf("error writing segment \n")
+
 		return
 	}
 
 	outMsg := c.readResponse()
-	if c.lastError != nil {
-		return nil, c.lastError
+	c.lock.RLock()
+	err = c.lastError
+	if err != nil {
+		c.lock.RUnlock()
+		return nil, err
 	}
+	c.lock.RUnlock()
+
 	if outMsg.typ != stCMD_RESPONSE {
 		return nil, fmt.Errorf("[%s] response error %d", apiname, outMsg.typ)
 	}
+
 	return outMsg.msg, nil
 }
 
@@ -77,16 +92,22 @@ func (c *ClientConn) readResponse() segment {
 		return outMsg
 	case <-time.After(c.ReadTimeout):
 		if c.lastError == nil {
+			c.lock.Lock()
 			c.lastError = fmt.Errorf("Timeout waiting for message response")
+			c.lock.Unlock()
 		}
+
 		return segment{}
 	}
 }
 
 func (c *ClientConn) RegisterEvent(name string, handler func(response map[string]interface{})) (err error) {
+	c.lock.Lock()
 	if c.eventHandlers[name] != nil {
+		c.lock.Unlock()
 		return fmt.Errorf("[event %s] register a event twice.", name)
 	}
+
 	c.eventHandlers[name] = handler
 	err = writeSegment(c.conn, segment{
 		typ:  stEVENT_REGISTER,
@@ -94,19 +115,31 @@ func (c *ClientConn) RegisterEvent(name string, handler func(response map[string
 	})
 	if err != nil {
 		delete(c.eventHandlers, name)
+		c.lock.Unlock()
+
 		return
 	}
+	c.lock.Unlock()
 	outMsg := c.readResponse()
-	//fmt.Printf("registerEvent %#v\n", outMsg)
-	if c.lastError != nil {
+	// fmt.Printf("registerEvent %#v\n", outMsg)
+	c.lock.Lock()
+	lastError := c.lastError
+
+	if lastError != nil {
 		delete(c.eventHandlers, name)
-		return c.lastError
+		c.lock.Unlock()
+
+		return err
 	}
 
 	if outMsg.typ != stEVENT_CONFIRM {
 		delete(c.eventHandlers, name)
+		c.lock.Unlock()
+
 		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
 	}
+	c.lock.Unlock()
+
 	return nil
 }
 
@@ -118,16 +151,25 @@ func (c *ClientConn) UnregisterEvent(name string) (err error) {
 	if err != nil {
 		return
 	}
+
 	outMsg := c.readResponse()
-	//fmt.Printf("UnregisterEvent %#v\n", outMsg)
+	// fmt.Printf("UnregisterEvent %#v\n", outMsg)
+	c.lock.Lock()
 	if c.lastError != nil {
+		c.lock.Unlock()
+
 		return c.lastError
 	}
+	c.lock.Unlock()
 
 	if outMsg.typ != stEVENT_CONFIRM {
 		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
 	}
+
+	c.lock.Lock()
 	delete(c.eventHandlers, name)
+	c.lock.Unlock()
+
 	return nil
 }
 
@@ -135,19 +177,29 @@ func (c *ClientConn) readThread() {
 	for {
 		outMsg, err := readSegment(c.conn)
 		if err != nil {
+			c.lock.Lock()
 			c.lastError = err
+			c.lock.Unlock()
+
 			return
 		}
+
 		switch outMsg.typ {
 		case stCMD_RESPONSE, stEVENT_CONFIRM:
 			c.responseChan <- outMsg
 		case stEVENT:
+			c.lock.Lock()
 			handler := c.eventHandlers[outMsg.name]
+			c.lock.Unlock()
+
 			if handler != nil {
 				handler(outMsg.msg)
 			}
 		default:
+			c.lock.Lock()
 			c.lastError = fmt.Errorf("[Client.readThread] unknow msg type %d", outMsg.typ)
+			c.lock.Unlock()
+
 			return
 		}
 	}

vendor/github.com/bronze1man/goStrongswanVici/loadCert.go

@@ -30,7 +30,7 @@ func (c *ClientConn) LoadCertificate(s string, typ string, flag string) (err err
 	}
 
 	if msg["success"] != "yes" {
-		return fmt.Errorf("unsuccessful loadCert: %v", msg["success"])
+		return fmt.Errorf("unsuccessful loadCert: %v", msg["errmsg"])
 	}
 
 	return nil

vendor/github.com/bronze1man/goStrongswanVici/loadConn.go

 package goStrongswanVici
 
 import (
+	"crypto"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 )
 
@@ -11,7 +14,10 @@ type Connection struct {
 type IKEConf struct {
 	LocalAddrs  []string               `json:"local_addrs"`
 	RemoteAddrs []string               `json:"remote_addrs,omitempty"`
+	LocalPort   string                 `json:"local_port,omitempty"`
+	RemotePort  string                 `json:"remote_port,omitempty"`
 	Proposals   []string               `json:"proposals,omitempty"`
+	Vips        []string               `json:"vips,omitempty"`
 	Version     string                 `json:"version"` //1 for ikev1, 0 for ikev1 & ikev2
 	Encap       string                 `json:"encap"`   //yes,no
 	KeyingTries string                 `json:"keyingtries"`
@@ -21,6 +27,7 @@ type IKEConf struct {
 	RemoteAuth  AuthConf               `json:"remote"`
 	Pools       []string               `json:"pools,omitempty"`
 	Children    map[string]ChildSAConf `json:"children"`
+	Mobike      string                 `json:"mobike,omitempty"`
 }
 
 type AuthConf struct {
@@ -28,6 +35,7 @@ type AuthConf struct {
 	Round      string   `json:"round,omitempty"`
 	AuthMethod string   `json:"auth"` // (psk|pubkey)
 	EAP_ID     string   `json:"eap_id,omitempty"`
+	PubKeys    []string `json:"pubkeys,omitempty"` // PEM encoded public keys
 }
 
 type ChildSAConf struct {
@@ -49,6 +57,28 @@ type ChildSAConf struct {
 	LifeTime      string   `json:"life_time,omitempty"`
 }
 
+// SetPublicKeys is a helper method that converts Public Keys to x509 PKIX PEM format
+// Supported formats are those implemented by x509.MarshalPKIXPublicKey
+func (a *AuthConf) SetPublicKeys(keys []crypto.PublicKey) error {
+	var newKeys []string
+
+	for _, key := range keys {
+		asn1Bytes, err := x509.MarshalPKIXPublicKey(key)
+		if err != nil {
+			return fmt.Errorf("Error marshaling key: %v", err)
+		}
+		pemKey := pem.Block{
+			Type:  "PUBLIC KEY",
+			Bytes: asn1Bytes,
+		}
+		pemBytes := pem.EncodeToMemory(&pemKey)
+		newKeys = append(newKeys, string(pemBytes))
+	}
+
+	a.PubKeys = newKeys
+	return nil
+}
+
 func (c *ClientConn) LoadConn(conn *map[string]IKEConf) error {
 	requestMap := &map[string]interface{}{}
 

vendor/github.com/bronze1man/goStrongswanVici/loadPrivateKey.go

@@ -59,7 +59,7 @@ func (c *ClientConn) loadPrivateKey(typ, data string) (err error) {
 
 	msg, err := c.Request("load-key", *requestMap)
 	if msg["success"] != "yes" {
-		return fmt.Errorf("unsuccessful loadPrivateKey: %v", msg["success"])
+		return fmt.Errorf("unsuccessful loadPrivateKey: %v", msg["errmsg"])
 	}
 
 	return nil

vendor/github.com/bronze1man/goStrongswanVici/monitorSA.go

@@ -31,11 +31,13 @@ type EventIkeSAUpDown struct {
 	Remote_id     string                         `json:"remote-id"`
 	Remote_host   string                         `json:"remote-host"`
 	Remote_port   string                         `json:"remote-port"`
+	Remote_vips   []string                       `json:"remote-vips"`
 	Responder_spi string                         `json:"responder-spi"`
 	State         string                         `json:"state"`
 	Task_Active   []string                       `json:"tasks-active"`
 	Uniqueid      string                         `json:"uniqueid"`
 	Version       string                         `json:"version"`
+	Remote_eap_id string                         `json:"remote-eap-id"` // client user name
 }
 
 type EventChildSAUpDown struct {
@@ -60,6 +62,7 @@ type EventChildSAUpDown struct {
 	Spi_out       string   `json:"spi-out"`
 	State         string   `json:"state"`
 	UniqueId      string   `json:"uniqueid"`
+	Remote_eap_id string   `json:"remote-eap-id"` // client user name
 }
 
 type EventIkeRekeyPair struct {
@@ -85,12 +88,14 @@ type EventIkeRekeySA struct {
 	Remote_id     string                          `json:"remote-id"`
 	Remote_host   string                          `json:"remote-host"`
 	Remote_port   string                          `json:"remote-port"`
+	Remote_vips   []string                        `json:"remote-vips"`
 	Responder_spi string                          `json:"responder-spi"`
 	State         string                          `json:"state"`
 	Task_Active   []string                        `json:"tasks-active"`
 	Task_Passive  []string                        `json:"tasks-passive"`
 	Uniqueid      string                          `json:"uniqueid"`
 	Version       string                          `json:"version"`
+	Remote_eap_id string                          `json:"remote-eap-id"` // client user name
 }
 
 type EventChildRekeyPair struct {
@@ -160,7 +165,6 @@ func prettyprint(b []byte) string {
 
 type monitorCallBack func(event string, info interface{})
 
-
 func handleIkeUpDown(eventName string, callback monitorCallBack, response map[string]interface{}) {
 	event := &EventIkeUpDown{}
 	event.Ike = map[string]*EventIkeSAUpDown{}
@@ -221,7 +225,7 @@ func handleChildRekey(eventName string, callback monitorCallBack, response map[s
 	callback(eventName, event)
 }
 
-func (c *ClientConn) MonitorSA(callback monitorCallBack,watchdog time.Duration) (err error) {
+func (c *ClientConn) MonitorSA(callback monitorCallBack, watchdog time.Duration) (err error) {
 	//register event
 	c.RegisterEvent(EVENT_CHILD_UPDOWN, func(response map[string]interface{}) {
 		//dumpResponse(response)

vendor/github.com/bronze1man/goStrongswanVici/pools.go

@@ -29,7 +29,7 @@ func (c *ClientConn) LoadPool(ph Pool) error {
 	msg, err := c.Request("load-pool", requestMap)
 
 	if msg["success"] != "yes" {
-		return fmt.Errorf("unsuccessful LoadPool: %v", msg["success"])
+		return fmt.Errorf("unsuccessful LoadPool: %v", msg["errmsg"])
 	}
 
 	return nil

vendor/github.com/bronze1man/goStrongswanVici/terminate.go

@@ -9,6 +9,7 @@ type TerminateRequest struct {
 	Ike      string `json:"ike,omitempty"`
 	Child_id string `json:"child-id,omitempty"`
 	Ike_id   string `json:"ike-id,omitempty"`
+	Force    string `json:"force,omitempty"`
 	Timeout  string `json:"timeout,omitempty"`
 	Loglevel string `json:"loglevel,omitempty"`
 }

vendor/github.com/flannel-io/flannel/backend/common.go

@@ -18,15 +18,16 @@ import (
 	"net"
 	"sync"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 )
 
 type ExternalInterface struct {
 	Iface       *net.Interface
 	IfaceAddr   net.IP
+	IfaceV6Addr net.IP
 	ExtAddr     net.IP
+	ExtV6Addr   net.IP
 }
 
 // Besides the entry points in the Backend interface, the backend's New()

vendor/github.com/flannel-io/flannel/backend/hostgw/hostgw.go

@@ -19,7 +19,6 @@ package hostgw
 
 import (
 	"fmt"
-
 	"sync"
 
 	"github.com/flannel-io/flannel/backend"
@@ -60,6 +59,13 @@ func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup
 		Mtu:         be.extIface.Iface.MTU,
 		LinkIndex:   be.extIface.Iface.Index,
 	}
+
+	attrs := subnet.LeaseAttrs{
+		BackendType: "host-gw",
+	}
+
+	if config.EnableIPv4 {
+		attrs.PublicIP = ip.FromIP(be.extIface.ExtAddr)
 		n.GetRoute = func(lease *subnet.Lease) *netlink.Route {
 			return &netlink.Route{
 				Dst:       lease.Subnet.ToIPNet(),
@@ -67,10 +73,17 @@ func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup
 				LinkIndex: n.LinkIndex,
 			}
 		}
+	}
 
-	attrs := subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
-		BackendType: "host-gw",
+	if config.EnableIPv6 {
+		attrs.PublicIPv6 = ip.FromIP6(be.extIface.ExtV6Addr)
+		n.GetV6Route = func(lease *subnet.Lease) *netlink.Route {
+			return &netlink.Route{
+				Dst:       lease.IPv6Subnet.ToIPNet(),
+				Gw:        lease.Attrs.PublicIPv6.ToIP(),
+				LinkIndex: n.LinkIndex,
+			}
+		}
 	}
 
 	l, err := be.sm.AcquireLease(ctx, &attrs)

vendor/github.com/flannel-io/flannel/backend/ipsec/handle_charon.go

@@ -161,9 +161,10 @@ func (charon *CharonIKEDaemon) LoadConnection(localLease, remoteLease *subnet.Le
 		ESPProposals:  []string{charon.espProposal},
 		StartAction:   "start",
 		CloseAction:   "trap",
+		DpdAction:     "restart",
 		Mode:          "tunnel",
 		ReqID:         reqID,
-		//		RekeyTime:     rekeyTime,
+		RekeyTime:     "1h",
 		InstallPolicy: "no",
 	}
 

vendor/github.com/flannel-io/flannel/backend/ipsec/ipsec_windows.go

@@ -13,9 +13,3 @@
 // limitations under the License.
 
 package ipsec
-
-import log "k8s.io/klog"
-
-func init() {
-	log.Infof("ipsec is not supported on this platform")
-}

vendor/github.com/flannel-io/flannel/backend/manager.go

@@ -19,9 +19,8 @@ import (
 	"strings"
 	"sync"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 )
 
 var constructors = make(map[string]BackendCtor)

vendor/github.com/flannel-io/flannel/backend/route_network.go

@@ -22,10 +22,9 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/subnet"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 
@@ -37,8 +36,10 @@ type RouteNetwork struct {
 	SimpleNetwork
 	BackendType string
 	routes      []netlink.Route
+	v6Routes    []netlink.Route
 	SM          subnet.Manager
 	GetRoute    func(lease *subnet.Lease) *netlink.Route
+	GetV6Route  func(lease *subnet.Lease) *netlink.Route
 	Mtu         int
 	LinkIndex   int
 }
@@ -83,54 +84,53 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
 	for _, evt := range batch {
 		switch evt.Type {
 		case subnet.EventAdded:
-			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
-
 			if evt.Lease.Attrs.BackendType != n.BackendType {
 				log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
 				continue
 			}
-			route := n.GetRoute(&evt.Lease)
 
-			n.addToRouteList(*route)
-			// Check if route exists before attempting to add it
-			routeList, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
-			if err != nil {
-				log.Warningf("Unable to list routes: %v", err)
-			}
+			if evt.Lease.EnableIPv4 {
+				log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
 
-			if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
-				// Same Dst different Gw or different link index. Remove it, correct route will be added below.
-				log.Warningf("Replacing existing route to %v via %v dev index %d with %v via %v dev index %d.", evt.Lease.Subnet, routeList[0].Gw, routeList[0].LinkIndex, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex)
-				if err := netlink.RouteDel(&routeList[0]); err != nil {
-					log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
-					continue
-				}
-				n.removeFromRouteList(routeList[0])
+				route := n.GetRoute(&evt.Lease)
+				routeAdd(route, netlink.FAMILY_V4, n.addToRouteList, n.removeFromV4RouteList)
 			}
 
-			if len(routeList) > 0 && routeEqual(routeList[0], *route) {
-				// Same Dst and same Gw, keep it and do not attempt to add it.
-				log.Infof("Route to %v via %v dev index %d already exists, skipping.", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, routeList[0].LinkIndex)
-			} else if err := netlink.RouteAdd(route); err != nil {
-				log.Errorf("Error adding route to %v via %v dev index %d: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex, err)
-				continue
+			if evt.Lease.EnableIPv6 {
+				log.Infof("Subnet added: %v via %v", evt.Lease.IPv6Subnet, evt.Lease.Attrs.PublicIPv6)
+
+				route := n.GetV6Route(&evt.Lease)
+				routeAdd(route, netlink.FAMILY_V6, n.addToV6RouteList, n.removeFromV6RouteList)
 			}
 
 		case subnet.EventRemoved:
-			log.Info("Subnet removed: ", evt.Lease.Subnet)
-
 			if evt.Lease.Attrs.BackendType != n.BackendType {
 				log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
 				continue
 			}
 
+			if evt.Lease.EnableIPv4 {
+				log.Info("Subnet removed: ", evt.Lease.Subnet)
+
 				route := n.GetRoute(&evt.Lease)
 				// Always remove the route from the route list.
-			n.removeFromRouteList(*route)
+				n.removeFromV4RouteList(*route)
 
 				if err := netlink.RouteDel(route); err != nil {
 					log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
-				continue
+				}
+			}
+
+			if evt.Lease.EnableIPv6 {
+				log.Info("Subnet removed: ", evt.Lease.IPv6Subnet)
+
+				route := n.GetV6Route(&evt.Lease)
+				// Always remove the route from the route list.
+				n.removeFromV6RouteList(*route)
+
+				if err := netlink.RouteDel(route); err != nil {
+					log.Errorf("Error deleting route to %v: %v", evt.Lease.IPv6Subnet, err)
+				}
 			}
 
 		default:
@@ -139,22 +139,74 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
 	}
 }
 
-func (n *RouteNetwork) addToRouteList(route netlink.Route) {
-	for _, r := range n.routes {
-		if routeEqual(r, route) {
+func routeAdd(route *netlink.Route, ipFamily int, addToRouteList, removeFromRouteList func(netlink.Route)) {
+	addToRouteList(*route)
+	// Check if route exists before attempting to add it
+	routeList, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
+	}
+
+	if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
+		// Same Dst different Gw or different link index. Remove it, correct route will be added below.
+		log.Warningf("Replacing existing route to %v with %v", routeList[0], route)
+		if err := netlink.RouteDel(&routeList[0]); err != nil {
+			log.Errorf("Effor deleteing route to %v: %v", routeList[0].Dst, err)
 			return
 		}
+		removeFromRouteList(routeList[0])
+	}
+	routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
 	}
-	n.routes = append(n.routes, route)
-}
 
-func (n *RouteNetwork) removeFromRouteList(route netlink.Route) {
-	for index, r := range n.routes {
-		if routeEqual(r, route) {
-			n.routes = append(n.routes[:index], n.routes[index+1:]...)
+	if len(routeList) > 0 && routeEqual(routeList[0], *route) {
+		// Same Dst and same Gw, keep it and do not attempt to add it.
+		log.Infof("Route to %v already exists, skipping.", route)
+	} else if err := netlink.RouteAdd(route); err != nil {
+		log.Errorf("Error adding route to %v", route)
 		return
 	}
+	routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
+	if err != nil {
+		log.Warningf("Unable to list routes: %v", err)
+	}
+}
+
+func (n *RouteNetwork) addToRouteList(route netlink.Route) {
+	n.routes = addToRouteList(&route, n.routes)
+}
+
+func (n *RouteNetwork) addToV6RouteList(route netlink.Route) {
+	n.v6Routes = addToRouteList(&route, n.v6Routes)
+}
+
+func addToRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
+	for _, r := range routes {
+		if routeEqual(r, *route) {
+			return routes
+		}
 	}
+	return append(routes, *route)
+}
+
+func (n *RouteNetwork) removeFromV4RouteList(route netlink.Route) {
+	n.routes = n.removeFromRouteList(&route, n.routes)
+}
+
+func (n *RouteNetwork) removeFromV6RouteList(route netlink.Route) {
+	n.v6Routes = n.removeFromRouteList(&route, n.v6Routes)
+}
+
+func (n *RouteNetwork) removeFromRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
+	for index, r := range routes {
+		if routeEqual(r, *route) {
+			routes = append(routes[:index], routes[index+1:]...)
+			return routes
+		}
+	}
+	return routes
 }
 
 func (n *RouteNetwork) routeCheck(ctx context.Context) {
@@ -163,15 +215,24 @@ func (n *RouteNetwork) routeCheck(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-time.After(routeCheckRetries * time.Second):
-			n.checkSubnetExistInRoutes()
+			n.checkSubnetExistInV4Routes()
+			n.checkSubnetExistInV6Routes()
 		}
 	}
 }
 
-func (n *RouteNetwork) checkSubnetExistInRoutes() {
-	routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+func (n *RouteNetwork) checkSubnetExistInV4Routes() {
+	n.checkSubnetExistInRoutes(n.routes, netlink.FAMILY_V4)
+}
+
+func (n *RouteNetwork) checkSubnetExistInV6Routes() {
+	n.checkSubnetExistInRoutes(n.v6Routes, netlink.FAMILY_V6)
+}
+
+func (n *RouteNetwork) checkSubnetExistInRoutes(routes []netlink.Route, ipFamily int) {
+	routeList, err := netlink.RouteList(nil, ipFamily)
 	if err == nil {
-		for _, route := range n.routes {
+		for _, route := range routes {
 			exist := false
 			for _, r := range routeList {
 				if r.Dst == nil {

vendor/github.com/flannel-io/flannel/backend/route_network_windows.go

@@ -19,10 +19,9 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/pkg/routing"
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 

vendor/github.com/flannel-io/flannel/backend/simple_network.go

@@ -15,9 +15,8 @@
 package backend
 
 import (
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 )
 
 type SimpleNetwork struct {

vendor/github.com/flannel-io/flannel/backend/vxlan/device.go

@@ -124,6 +124,18 @@ func (dev *vxlanDevice) Configure(ipa ip.IP4Net, flannelnet ip.IP4Net) error {
 	return nil
 }
 
+func (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net, flannelnet ip.IP6Net) error {
+	if err := ip.EnsureV6AddressOnLink(ipn, flannelnet, dev.link); err != nil {
+		return fmt.Errorf("failed to ensure v6 address of interface %s: %w", dev.link.Attrs().Name, err)
+	}
+
+	if err := netlink.LinkSetUp(dev.link); err != nil {
+		return fmt.Errorf("failed to set v6 interface %s to UP state: %w", dev.link.Attrs().Name, err)
+	}
+
+	return nil
+}
+
 func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
 	return dev.link.HardwareAddr
 }
@@ -131,6 +143,7 @@ func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
 type neighbor struct {
 	MAC net.HardwareAddr
 	IP  ip.IP4
+	IP6 *ip.IP6
 }
 
 func (dev *vxlanDevice) AddFDB(n neighbor) error {
@@ -145,6 +158,18 @@ func (dev *vxlanDevice) AddFDB(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) AddV6FDB(n neighbor) error {
+	log.V(4).Infof("calling AddV6FDB: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighSet(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Family:       syscall.AF_BRIDGE,
+		Flags:        netlink.NTF_SELF,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) DelFDB(n neighbor) error {
 	log.V(4).Infof("calling DelFDB: %v, %v", n.IP, n.MAC)
 	return netlink.NeighDel(&netlink.Neigh{
@@ -156,6 +181,17 @@ func (dev *vxlanDevice) DelFDB(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) DelV6FDB(n neighbor) error {
+	log.V(4).Infof("calling DelV6FDB: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighDel(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		Family:       syscall.AF_BRIDGE,
+		Flags:        netlink.NTF_SELF,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) AddARP(n neighbor) error {
 	log.V(4).Infof("calling AddARP: %v, %v", n.IP, n.MAC)
 	return netlink.NeighSet(&netlink.Neigh{
@@ -167,6 +203,17 @@ func (dev *vxlanDevice) AddARP(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) AddV6ARP(n neighbor) error {
+	log.V(4).Infof("calling AddV6ARP: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighSet(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Type:         syscall.RTN_UNICAST,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func (dev *vxlanDevice) DelARP(n neighbor) error {
 	log.V(4).Infof("calling DelARP: %v, %v", n.IP, n.MAC)
 	return netlink.NeighDel(&netlink.Neigh{
@@ -178,6 +225,17 @@ func (dev *vxlanDevice) DelARP(n neighbor) error {
 	})
 }
 
+func (dev *vxlanDevice) DelV6ARP(n neighbor) error {
+	log.V(4).Infof("calling DelV6ARP: %v, %v", n.IP6, n.MAC)
+	return netlink.NeighDel(&netlink.Neigh{
+		LinkIndex:    dev.link.Index,
+		State:        netlink.NUD_PERMANENT,
+		Type:         syscall.RTN_UNICAST,
+		IP:           n.IP6.ToIP(),
+		HardwareAddr: n.MAC,
+	})
+}
+
 func vxlanLinksIncompat(l1, l2 netlink.Link) string {
 	if l1.Type() != l2.Type() {
 		return fmt.Sprintf("link type: %v vs %v", l1.Type(), l2.Type())

vendor/github.com/flannel-io/flannel/backend/vxlan/vxlan.go

@@ -58,11 +58,10 @@ import (
 	"net"
 	"sync"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/backend"
 	"github.com/flannel-io/flannel/pkg/ip"
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 
@@ -88,19 +87,34 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
 	return backend, nil
 }
 
-func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
+func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, vnid uint16, dev, v6Dev *vxlanDevice) (*subnet.LeaseAttrs, error) {
+	leaseAttrs := &subnet.LeaseAttrs{
+		BackendType: "vxlan",
+	}
+	if publicIP != nil && dev != nil {
 		data, err := json.Marshal(&vxlanLeaseAttrs{
 			VNI:     vnid,
-		VtepMAC: hardwareAddr(mac)})
+			VtepMAC: hardwareAddr(dev.MACAddr()),
+		})
 		if err != nil {
 			return nil, err
 		}
+		leaseAttrs.PublicIP = ip.FromIP(publicIP)
+		leaseAttrs.BackendData = json.RawMessage(data)
+	}
 
-	return &subnet.LeaseAttrs{
-		PublicIP:    ip.FromIP(publicIP),
-		BackendType: "vxlan",
-		BackendData: json.RawMessage(data),
-	}, nil
+	if publicIPv6 != nil && v6Dev != nil {
+		data, err := json.Marshal(&vxlanLeaseAttrs{
+			VNI:     vnid,
+			VtepMAC: hardwareAddr(v6Dev.MACAddr()),
+		})
+		if err != nil {
+			return nil, err
+		}
+		leaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)
+		leaseAttrs.BackendV6Data = json.RawMessage(data)
+	}
+	return leaseAttrs, nil
 }
 
 func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
@@ -122,6 +136,9 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 	}
 	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)
 
+	var dev, v6Dev *vxlanDevice
+	var err error
+	if config.EnableIPv4 {
 		devAttrs := vxlanDeviceAttrs{
 			vni:       uint32(cfg.VNI),
 			name:      fmt.Sprintf("flannel.%v", cfg.VNI),
@@ -132,13 +149,30 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 			learning:  cfg.Learning,
 		}
 
-	dev, err := newVXLANDevice(&devAttrs)
+		dev, err = newVXLANDevice(&devAttrs)
 		if err != nil {
 			return nil, err
 		}
 		dev.directRouting = cfg.DirectRouting
+	}
+	if config.EnableIPv6 {
+		v6DevAttrs := vxlanDeviceAttrs{
+			vni:       uint32(cfg.VNI),
+			name:      fmt.Sprintf("flannel-v6.%v", cfg.VNI),
+			vtepIndex: be.extIface.Iface.Index,
+			vtepAddr:  be.extIface.IfaceV6Addr,
+			vtepPort:  cfg.Port,
+			gbp:       cfg.GBP,
+			learning:  cfg.Learning,
+		}
+		v6Dev, err = newVXLANDevice(&v6DevAttrs)
+		if err != nil {
+			return nil, err
+		}
+		v6Dev.directRouting = cfg.DirectRouting
+	}
 
-	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), dev.MACAddr())
+	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint16(cfg.VNI), dev, v6Dev)
 	if err != nil {
 		return nil, err
 	}
@@ -155,11 +189,17 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 	// Ensure that the device has a /32 address so that no broadcast routes are created.
 	// This IP is just used as a source address for host to workload traffic (so
 	// the return path for the traffic has an address on the flannel network to use as the destination)
+	if config.EnableIPv4 {
 		if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
-		return nil, fmt.Errorf("failed to configure interface %s: %s", dev.link.Attrs().Name, err)
+			return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
 		}
-
-	return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
+	}
+	if config.EnableIPv6 {
+		if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}, config.IPv6Network); err != nil {
+			return nil, fmt.Errorf("failed to configure interface %s: %w", v6Dev.link.Attrs().Name, err)
+		}
+	}
+	return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease)
 }
 
 // So we can make it JSON (un)marshalable

vendor/github.com/flannel-io/flannel/backend/vxlan/vxlan_network.go

@@ -21,18 +21,18 @@ import (
 	"sync"
 	"syscall"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/backend"
 	"github.com/flannel-io/flannel/pkg/ip"
 	"github.com/flannel-io/flannel/subnet"
 	"github.com/vishvananda/netlink"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 
 type network struct {
 	backend.SimpleNetwork
 	dev       *vxlanDevice
+	v6Dev     *vxlanDevice
 	subnetMgr subnet.Manager
 }
 
@@ -40,7 +40,7 @@ const (
 	encapOverhead = 50
 )
 
-func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, _ ip.IP4Net, lease *subnet.Lease) (*network, error) {
+func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, dev *vxlanDevice, v6Dev *vxlanDevice, _ ip.IP4Net, lease *subnet.Lease) (*network, error) {
 	nw := &network{
 		SimpleNetwork: backend.SimpleNetwork{
 			SubnetLease: lease,
@@ -48,6 +48,7 @@ func newNetwork(subnetMgr subnet.Manager, extIface *backend.ExternalInterface, d
 		},
 		subnetMgr: subnetMgr,
 		dev:       dev,
+		v6Dev:     v6Dev,
 	}
 
 	return nw, nil
@@ -91,20 +92,28 @@ type vxlanLeaseAttrs struct {
 func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 	for _, event := range batch {
 		sn := event.Lease.Subnet
+		v6Sn := event.Lease.IPv6Subnet
 		attrs := event.Lease.Attrs
 		if attrs.BackendType != "vxlan" {
-			log.Warningf("ignoring non-vxlan subnet(%s): type=%v", sn, attrs.BackendType)
+			log.Warningf("ignoring non-vxlan v4Subnet(%s) v6Subnet(%s): type=%v", sn, v6Sn, attrs.BackendType)
 			continue
 		}
 
-		var vxlanAttrs vxlanLeaseAttrs
+		var (
+			vxlanAttrs, v6VxlanAttrs           vxlanLeaseAttrs
+			directRoutingOK, v6DirectRoutingOK bool
+			directRoute, v6DirectRoute         netlink.Route
+			vxlanRoute, v6VxlanRoute           netlink.Route
+		)
+
+		if event.Lease.EnableIPv4 && nw.dev != nil {
 			if err := json.Unmarshal(attrs.BackendData, &vxlanAttrs); err != nil {
 				log.Error("error decoding subnet lease JSON: ", err)
 				continue
 			}
 
 			// This route is used when traffic should be vxlan encapsulated
-		vxlanRoute := netlink.Route{
+			vxlanRoute = netlink.Route{
 				LinkIndex: nw.dev.link.Attrs().Index,
 				Scope:     netlink.SCOPE_UNIVERSE,
 				Dst:       sn.ToIPNet(),
@@ -113,11 +122,10 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 			vxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
 
 			// directRouting is where the remote host is on the same subnet so vxlan isn't required.
-		directRoute := netlink.Route{
+			directRoute = netlink.Route{
 				Dst: sn.ToIPNet(),
 				Gw:  attrs.PublicIP.ToIP(),
 			}
-		var directRoutingOK = false
 			if nw.dev.directRouting {
 				if dr, err := ip.DirectRouting(attrs.PublicIP.ToIP()); err != nil {
 					log.Error(err)
@@ -125,9 +133,41 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 					directRoutingOK = dr
 				}
 			}
+		}
+
+		if event.Lease.EnableIPv6 && nw.v6Dev != nil {
+			if err := json.Unmarshal(attrs.BackendV6Data, &v6VxlanAttrs); err != nil {
+				log.Error("error decoding v6 subnet lease JSON: ", err)
+				continue
+			}
+			if v6Sn.IP != nil && nw.v6Dev != nil {
+				v6VxlanRoute = netlink.Route{
+					LinkIndex: nw.v6Dev.link.Attrs().Index,
+					Scope:     netlink.SCOPE_UNIVERSE,
+					Dst:       v6Sn.ToIPNet(),
+					Gw:        v6Sn.IP.ToIP(),
+				}
+				v6VxlanRoute.SetFlag(syscall.RTNH_F_ONLINK)
+
+				// directRouting is where the remote host is on the same subnet so vxlan isn't required.
+				v6DirectRoute = netlink.Route{
+					Dst: v6Sn.ToIPNet(),
+					Gw:  attrs.PublicIPv6.ToIP(),
+				}
+
+				if nw.v6Dev.directRouting {
+					if v6Dr, err := ip.DirectRouting(attrs.PublicIPv6.ToIP()); err != nil {
+						log.Error(err)
+					} else {
+						v6DirectRoutingOK = v6Dr
+					}
+				}
+			}
+		}
 
 		switch event.Type {
 		case subnet.EventAdded:
+			if event.Lease.EnableIPv4 {
 				if directRoutingOK {
 					log.V(2).Infof("Adding direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
 
@@ -170,7 +210,53 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 						continue
 					}
 				}
+			}
+			if event.Lease.EnableIPv6 {
+				if v6DirectRoutingOK {
+					log.V(2).Infof("Adding v6 direct route to v6 subnet: %s PublicIPv6: %s", v6Sn, attrs.PublicIPv6)
+
+					if err := netlink.RouteReplace(&v6DirectRoute); err != nil {
+						log.Errorf("Error adding v6 route to %v via %v: %v", v6Sn, attrs.PublicIPv6, err)
+						continue
+					}
+				} else {
+					log.V(2).Infof("adding v6 subnet: %s PublicIPv6: %s VtepMAC: %s", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))
+					if err := nw.v6Dev.AddV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddV6ARP failed: ", err)
+						continue
+					}
+
+					if err := nw.v6Dev.AddV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("AddV6FDB failed: ", err)
+
+						// Try to clean up the ARP entry then continue
+						if err := nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6ARP failed: ", err)
+						}
+
+						continue
+					}
+
+					// Set the route - the kernel would ARP for the Gw IP address if it hadn't already been set above so make sure
+					// this is done last.
+					if err := netlink.RouteReplace(&v6VxlanRoute); err != nil {
+						log.Errorf("failed to add v6 vxlanRoute (%s -> %s): %v", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)
+
+						// Try to clean up both the ARP and FDB entries then continue
+						if err := nw.v6Dev.DelV6ARP(neighbor{IP6: event.Lease.IPv6Subnet.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6ARP failed: ", err)
+						}
+
+						if err := nw.v6Dev.DelV6FDB(neighbor{IP6: event.Lease.Attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+							log.Error("DelV6FDB failed: ", err)
+						}
+
+						continue
+					}
+				}
+			}
 		case subnet.EventRemoved:
+			if event.Lease.EnableIPv4 {
 				if directRoutingOK {
 					log.V(2).Infof("Removing direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIP)
 					if err := netlink.RouteDel(&directRoute); err != nil {
@@ -192,6 +278,30 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 						log.Errorf("failed to delete vxlanRoute (%s -> %s): %v", vxlanRoute.Dst, vxlanRoute.Gw, err)
 					}
 				}
+			}
+			if event.Lease.EnableIPv6 {
+				if v6DirectRoutingOK {
+					log.V(2).Infof("Removing v6 direct route to subnet: %s PublicIP: %s", sn, attrs.PublicIPv6)
+					if err := netlink.RouteDel(&directRoute); err != nil {
+						log.Errorf("Error deleting v6 route to %v via %v: %v", v6Sn, attrs.PublicIPv6, err)
+					}
+				} else {
+					log.V(2).Infof("removing v6subnet: %s PublicIPv6: %s VtepMAC: %s", v6Sn, attrs.PublicIPv6, net.HardwareAddr(v6VxlanAttrs.VtepMAC))
+
+					// Try to remove all entries - don't bail out if one of them fails.
+					if err := nw.v6Dev.DelV6ARP(neighbor{IP6: v6Sn.IP, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("DelV6ARP failed: ", err)
+					}
+
+					if err := nw.v6Dev.DelV6FDB(neighbor{IP6: attrs.PublicIPv6, MAC: net.HardwareAddr(v6VxlanAttrs.VtepMAC)}); err != nil {
+						log.Error("DelV6FDB failed: ", err)
+					}
+
+					if err := netlink.RouteDel(&v6VxlanRoute); err != nil {
+						log.Errorf("failed to delete v6 vxlanRoute (%s -> %s): %v", v6VxlanRoute.Dst, v6VxlanRoute.Gw, err)
+					}
+				}
+			}
 		default:
 			log.Error("internal error: unknown event type: ", int(event.Type))
 		}

vendor/github.com/flannel-io/flannel/backend/vxlan/vxlan_network_windows.go

@@ -20,12 +20,11 @@ import (
 	"strings"
 	"sync"
 
-	"golang.org/x/net/context"
-
 	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/flannel-io/flannel/backend"
 	"github.com/flannel-io/flannel/pkg/ip"
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 

vendor/github.com/flannel-io/flannel/backend/vxlan/vxlan_windows.go

@@ -30,12 +30,11 @@ import (
 	"net"
 	"sync"
 
-	"golang.org/x/net/context"
-
 	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/flannel-io/flannel/backend"
 	"github.com/flannel-io/flannel/pkg/ip"
 	"github.com/flannel-io/flannel/subnet"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 

vendor/github.com/flannel-io/flannel/network/iptables.go

@@ -77,6 +77,40 @@ func MasqRules(ipn ip.IP4Net, lease *subnet.Lease) []IPTablesRule {
 	}
 }
 
+func MasqIP6Rules(ipn ip.IP6Net, lease *subnet.Lease) []IPTablesRule {
+	n := ipn.String()
+	sn := lease.IPv6Subnet.String()
+	supports_random_fully := false
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err == nil {
+		supports_random_fully = ipt.HasRandomFully()
+	}
+
+	if supports_random_fully {
+		return []IPTablesRule{
+			// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
+			{"nat", "POSTROUTING", []string{"-s", n, "-d", n, "-j", "RETURN"}},
+			// NAT if it's not multicast traffic
+			{"nat", "POSTROUTING", []string{"-s", n, "!", "-d", "ff00::/8", "-j", "MASQUERADE", "--random-fully"}},
+			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", sn, "-j", "RETURN"}},
+			// Masquerade anything headed towards flannel from the host
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", n, "-j", "MASQUERADE", "--random-fully"}},
+		}
+	} else {
+		return []IPTablesRule{
+			// This rule makes sure we don't NAT traffic within overlay network (e.g. coming out of docker0)
+			{"nat", "POSTROUTING", []string{"-s", n, "-d", n, "-j", "RETURN"}},
+			// NAT if it's not multicast traffic
+			{"nat", "POSTROUTING", []string{"-s", n, "!", "-d", "ff00::/8", "-j", "MASQUERADE"}},
+			// Prevent performing Masquerade on external traffic which arrives from a Node that owns the container/pod IP address
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", sn, "-j", "RETURN"}},
+			// Masquerade anything headed towards flannel from the host
+			{"nat", "POSTROUTING", []string{"!", "-s", n, "-d", n, "-j", "MASQUERADE"}},
+		}
+	}
+}
+
 func ForwardRules(flannelNetwork string) []IPTablesRule {
 	return []IPTablesRule{
 		// These rules allow traffic to be forwarded if it is to or from the flannel network range.
@@ -122,6 +156,28 @@ func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int) {
 	}
 }
 
+func SetupAndEnsureIP6Tables(rules []IPTablesRule, resyncPeriod int) {
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err != nil {
+		// if we can't find iptables, give up and return
+		log.Errorf("Failed to setup IP6Tables. iptables binary was not found: %v", err)
+		return
+	}
+
+	defer func() {
+		teardownIPTables(ipt, rules)
+	}()
+
+	for {
+		// Ensure that all the iptables rules exist every 5 seconds
+		if err := ensureIPTables(ipt, rules); err != nil {
+			log.Errorf("Failed to ensure iptables rules: %v", err)
+		}
+
+		time.Sleep(time.Duration(resyncPeriod) * time.Second)
+	}
+}
+
 // DeleteIPTables delete specified iptables rules
 func DeleteIPTables(rules []IPTablesRule) error {
 	ipt, err := iptables.New()
@@ -134,6 +190,18 @@ func DeleteIPTables(rules []IPTablesRule) error {
 	return nil
 }
 
+// DeleteIP6Tables delete specified iptables rules
+func DeleteIP6Tables(rules []IPTablesRule) error {
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv6)
+	if err != nil {
+		// if we can't find iptables, give up and return
+		log.Errorf("Failed to setup IP6Tables. iptables binary was not found: %v", err)
+		return err
+	}
+	teardownIPTables(ipt, rules)
+	return nil
+}
+
 func ensureIPTables(ipt IPTables, rules []IPTablesRule) error {
 	exists, err := ipTablesRulesExist(ipt, rules)
 	if err != nil {

vendor/github.com/flannel-io/flannel/network/iptables_windows.go

@@ -31,21 +31,11 @@ type IPTablesRule struct {
 	rulespec []string
 }
 
-func MasqRules(ipn ip.IP4Net, lease *subnet.Lease) []IPTablesRule {
-	return nil
-}
-
-func ForwardRules(flannelNetwork string) []IPTablesRule {
-	return nil
-}
-
-func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int) {
-
-}
-
-func DeleteIPTables(rules []IPTablesRule) error {
-	return nil
-}
-
-func teardownIPTables(ipt IPTables, rules []IPTablesRule) {
-}
+func MasqRules(ipn ip.IP4Net, lease *subnet.Lease) []IPTablesRule    { return nil }
+func ForwardRules(flannelNetwork string) []IPTablesRule              { return nil }
+func SetupAndEnsureIPTables(rules []IPTablesRule, resyncPeriod int)  {}
+func DeleteIPTables(rules []IPTablesRule) error                      { return nil }
+func teardownIPTables(ipt IPTables, rules []IPTablesRule)            {}
+func SetupAndEnsureIP6Tables(rules []IPTablesRule, resyncPeriod int) {}
+func MasqIP6Rules(ipn ip.IP6Net, lease *subnet.Lease) []IPTablesRule { return nil }
+func DeleteIP6Tables(rules []IPTablesRule) error                     { return nil }

vendor/github.com/flannel-io/flannel/pkg/ip/iface.go

@@ -36,6 +36,16 @@ func getIfaceAddrs(iface *net.Interface) ([]netlink.Addr, error) {
 	return netlink.AddrList(link, syscall.AF_INET)
 }
 
+func getIfaceV6Addrs(iface *net.Interface) ([]netlink.Addr, error) {
+	link := &netlink.Device{
+		netlink.LinkAttrs{
+			Index: iface.Index,
+		},
+	}
+
+	return netlink.AddrList(link, syscall.AF_INET6)
+}
+
 func GetInterfaceIP4Addr(iface *net.Interface) (net.IP, error) {
 	addrs, err := getIfaceAddrs(iface)
 	if err != nil {
@@ -67,6 +77,37 @@ func GetInterfaceIP4Addr(iface *net.Interface) (net.IP, error) {
 	return nil, errors.New("No IPv4 address found for given interface")
 }
 
+func GetInterfaceIP6Addr(iface *net.Interface) (net.IP, error) {
+	addrs, err := getIfaceV6Addrs(iface)
+	if err != nil {
+		return nil, err
+	}
+
+	// prefer non link-local addr
+	var ll net.IP
+
+	for _, addr := range addrs {
+		if addr.IP.To16() == nil {
+			continue
+		}
+
+		if addr.IP.IsGlobalUnicast() {
+			return addr.IP, nil
+		}
+
+		if addr.IP.IsLinkLocalUnicast() {
+			ll = addr.IP
+		}
+	}
+
+	if ll != nil {
+		// didn't find global but found link-local. it'll do.
+		return ll, nil
+	}
+
+	return nil, errors.New("No IPv6 address found for given interface")
+}
+
 func GetInterfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
 	addrs, err := getIfaceAddrs(iface)
 	if err != nil {
@@ -86,6 +127,25 @@ func GetInterfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
 	return errors.New("No IPv4 address found for given interface")
 }
 
+func GetInterfaceIP6AddrMatch(iface *net.Interface, matchAddr net.IP) error {
+	addrs, err := getIfaceV6Addrs(iface)
+	if err != nil {
+		return err
+	}
+
+	for _, addr := range addrs {
+		// Attempt to parse the address in CIDR notation
+		// and assert it is IPv6
+		if addr.IP.To16() != nil {
+			if addr.IP.To16().Equal(matchAddr) {
+				return nil
+			}
+		}
+	}
+
+	return errors.New("No IPv6 address found for given interface")
+}
+
 func GetDefaultGatewayInterface() (*net.Interface, error) {
 	routes, err := netlink.RouteList(nil, syscall.AF_INET)
 	if err != nil {
@@ -104,6 +164,24 @@ func GetDefaultGatewayInterface() (*net.Interface, error) {
 	return nil, errors.New("Unable to find default route")
 }
 
+func GetDefaultV6GatewayInterface() (*net.Interface, error) {
+	routes, err := netlink.RouteList(nil, syscall.AF_INET6)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, route := range routes {
+		if route.Dst == nil || route.Dst.String() == "::/0" {
+			if route.LinkIndex <= 0 {
+				return nil, errors.New("Found default v6 route but could not determine interface")
+			}
+			return net.InterfaceByIndex(route.LinkIndex)
+		}
+	}
+
+	return nil, errors.New("Unable to find default v6 route")
+}
+
 func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
 	ifaces, err := net.Interfaces()
 	if err != nil {
@@ -120,6 +198,22 @@ func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
 	return nil, errors.New("No interface with given IP found")
 }
 
+func GetInterfaceByIP6(ip net.IP) (*net.Interface, error) {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, iface := range ifaces {
+		err := GetInterfaceIP6AddrMatch(&iface, ip)
+		if err == nil {
+			return &iface, nil
+		}
+	}
+
+	return nil, errors.New("No interface with given IPv6 found")
+}
+
 func DirectRouting(ip net.IP) (bool, error) {
 	routes, err := netlink.RouteGet(ip)
 	if err != nil {
@@ -164,3 +258,41 @@ func EnsureV4AddressOnLink(ipa IP4Net, ipn IP4Net, link netlink.Link) error {
 
 	return nil
 }
+
+// EnsureV6AddressOnLink ensures that there is only one v6 Addr on `link` and it equals `ipn`.
+// If there exist multiple addresses on link, it returns an error message to tell callers to remove additional address.
+func EnsureV6AddressOnLink(ipa IP6Net, ipn IP6Net, link netlink.Link) error {
+	addr := netlink.Addr{IPNet: ipa.ToIPNet()}
+	existingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
+	if err != nil {
+		return err
+	}
+
+	onlyLinkLocal := true
+	for _, existingAddr := range existingAddrs {
+		if !existingAddr.IP.IsLinkLocalUnicast() {
+			if !existingAddr.Equal(addr) {
+				if err := netlink.AddrDel(link, &existingAddr); err != nil {
+					return fmt.Errorf("failed to remove v6 IP address %s from %s: %w", ipn.String(), link.Attrs().Name, err)
+				}
+				existingAddrs = []netlink.Addr{}
+				onlyLinkLocal = false
+			} else {
+				return nil
+			}
+		}
+	}
+
+	if onlyLinkLocal {
+		existingAddrs = []netlink.Addr{}
+	}
+
+	// Actually add the desired address to the interface if needed.
+	if len(existingAddrs) == 0 {
+		if err := netlink.AddrAdd(link, &addr); err != nil {
+			return fmt.Errorf("failed to add v6 IP address %s to %s: %w", ipn.String(), link.Attrs().Name, err)
+		}
+	}
+
+	return nil
+}

vendor/github.com/flannel-io/flannel/pkg/ip/iface_windows.go

@@ -19,8 +19,9 @@ package ip
 import (
 	"errors"
 	"fmt"
-	"github.com/flannel-io/flannel/pkg/powershell"
 	"net"
+
+	"github.com/flannel-io/flannel/pkg/powershell"
 )
 
 // GetInterfaceIP4Addr returns the IPv4 address for the given network interface
@@ -143,3 +144,7 @@ func IsForwardingEnabledForInterface(iface *net.Interface) (bool, error) {
 
 	return powerShellJsonData.Forwarding == 1, nil
 }
+
+func GetInterfaceByIP6(ip net.IP) (*net.Interface, error)      { return nil, nil }
+func GetInterfaceIP6Addr(iface *net.Interface) (net.IP, error) { return nil, nil }
+func GetDefaultV6GatewayInterface() (*net.Interface, error)    { return nil, nil }

vendor/github.com/flannel-io/flannel/pkg/ip/ip6net.go

+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ip
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+)
+
+type IP6 big.Int
+
+func FromIP16Bytes(ip []byte) *IP6 {
+	return (*IP6)(big.NewInt(0).SetBytes(ip))
+}
+
+func FromIP6(ip net.IP) *IP6 {
+	ipv6 := ip.To16()
+
+	if ipv6 == nil {
+		panic("Address is not an IPv6 address")
+	}
+
+	return FromIP16Bytes(ipv6)
+}
+
+func ParseIP6(s string) (*IP6, error) {
+	ip := net.ParseIP(s)
+	if ip == nil {
+		return (*IP6)(big.NewInt(0)), errors.New("Invalid IP address format")
+	}
+	return FromIP6(ip), nil
+}
+
+func Mask(prefixLen int) *big.Int {
+	mask := net.CIDRMask(prefixLen, 128)
+	return big.NewInt(0).SetBytes(mask)
+}
+
+func IsEmpty(subnet *IP6) bool {
+	if subnet == nil || (*big.Int)(subnet).Cmp(big.NewInt(0)) == 0 {
+		return true
+	}
+	return false
+}
+
+func GetIPv6SubnetMin(networkIP *IP6, subnetSize *big.Int) *IP6 {
+	return (*IP6)(big.NewInt(0).Add((*big.Int)(networkIP), subnetSize))
+}
+
+func GetIPv6SubnetMax(networkIP *IP6, subnetSize *big.Int) *IP6 {
+	return (*IP6)(big.NewInt(0).Sub((*big.Int)(networkIP), subnetSize))
+}
+
+func CheckIPv6Subnet(subnetIP *IP6, mask *big.Int) bool {
+	if (*big.Int)(subnetIP).Cmp(big.NewInt(0).And((*big.Int)(subnetIP), mask)) != 0 {
+		return false
+	}
+	return true
+}
+
+func MustParseIP6(s string) *IP6 {
+	ip, err := ParseIP6(s)
+	if err != nil {
+		panic(err)
+	}
+	return ip
+}
+
+func (ip6 *IP6) ToIP() net.IP {
+	ip := net.IP((*big.Int)(ip6).Bytes())
+	if ip.To4() != nil {
+		return ip
+	}
+	a := (*big.Int)(ip6).FillBytes(make([]byte, 16))
+	return a
+}
+
+func (ip6 IP6) String() string {
+	return ip6.ToIP().String()
+}
+
+// MarshalJSON: json.Marshaler impl
+func (ip6 IP6) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, ip6)), nil
+}
+
+// UnmarshalJSON: json.Unmarshaler impl
+func (ip6 *IP6) UnmarshalJSON(j []byte) error {
+	j = bytes.Trim(j, "\"")
+	if val, err := ParseIP6(string(j)); err != nil {
+		return err
+	} else {
+		*ip6 = *val
+		return nil
+	}
+}
+
+// similar to net.IPNet but has uint based representation
+type IP6Net struct {
+	IP        *IP6
+	PrefixLen uint
+}
+
+func (n IP6Net) String() string {
+	if n.IP == nil {
+		n.IP = (*IP6)(big.NewInt(0))
+	}
+	return fmt.Sprintf("%s/%d", n.IP.String(), n.PrefixLen)
+}
+
+func (n IP6Net) StringSep(hexSep, prefixSep string) string {
+	return fmt.Sprintf("%s%s%d", n.IP.String(), prefixSep, n.PrefixLen)
+}
+
+func (n IP6Net) Network() IP6Net {
+	mask := net.CIDRMask(int(n.PrefixLen), 128)
+	return IP6Net{
+		FromIP6(n.IP.ToIP().Mask(mask)),
+		n.PrefixLen,
+	}
+}
+
+func (n IP6Net) Next() IP6Net {
+	return IP6Net{
+		(*IP6)(big.NewInt(0).Add((*big.Int)(n.IP), big.NewInt(0).Lsh(big.NewInt(1), 128-n.PrefixLen))),
+		n.PrefixLen,
+	}
+}
+
+// IncrementIP() increments the IP of IP6Net CIDR by 1
+func (n *IP6Net) IncrementIP() {
+	n.IP = (*IP6)(big.NewInt(0).Add((*big.Int)(n.IP), big.NewInt(1)))
+}
+
+func FromIP6Net(n *net.IPNet) IP6Net {
+	prefixLen, _ := n.Mask.Size()
+	return IP6Net{
+		FromIP6(n.IP),
+		uint(prefixLen),
+	}
+}
+
+func (n IP6Net) ToIPNet() *net.IPNet {
+	return &net.IPNet{
+		IP:   n.IP.ToIP(),
+		Mask: net.CIDRMask(int(n.PrefixLen), 128),
+	}
+}
+
+func (n IP6Net) Overlaps(other IP6Net) bool {
+	var mask *big.Int
+	if n.PrefixLen < other.PrefixLen {
+		mask = n.Mask()
+	} else {
+		mask = other.Mask()
+	}
+	return (IP6)(*big.NewInt(0).And((*big.Int)(n.IP), mask)).String() ==
+		(IP6)(*big.NewInt(0).And((*big.Int)(other.IP), mask)).String()
+}
+
+func (n IP6Net) Equal(other IP6Net) bool {
+	return ((*big.Int)(n.IP).Cmp((*big.Int)(other.IP)) == 0) &&
+		n.PrefixLen == other.PrefixLen
+}
+
+func (n IP6Net) Mask() *big.Int {
+	mask := net.CIDRMask(int(n.PrefixLen), 128)
+	return big.NewInt(0).SetBytes(mask)
+}
+
+func (n IP6Net) Contains(ip *IP6) bool {
+	network := big.NewInt(0).And((*big.Int)(n.IP), n.Mask())
+	subnet := big.NewInt(0).And((*big.Int)(ip), n.Mask())
+	return (IP6)(*network).String() == (IP6)(*subnet).String()
+}
+
+func (n IP6Net) Empty() bool {
+	return n.IP == (*IP6)(big.NewInt(0)) && n.PrefixLen == uint(0)
+}
+
+// MarshalJSON: json.Marshaler impl
+func (n IP6Net) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s"`, n)), nil
+}
+
+// UnmarshalJSON: json.Unmarshaler impl
+func (n *IP6Net) UnmarshalJSON(j []byte) error {
+	j = bytes.Trim(j, "\"")
+	if _, val, err := net.ParseCIDR(string(j)); err != nil {
+		return err
+	} else {
+		*n = FromIP6Net(val)
+		return nil
+	}
+}

vendor/github.com/flannel-io/flannel/pkg/ip/ipnet.go

@@ -127,6 +127,11 @@ func (n IP4Net) Next() IP4Net {
 	}
 }
 
+// IncrementIP() increments the IP of IP4Net CIDR by 1
+func (n *IP4Net) IncrementIP() {
+	n.IP++
+}
+
 func FromIPNet(n *net.IPNet) IP4Net {
 	prefixLen, _ := n.Mask.Size()
 	return IP4Net{

vendor/github.com/flannel-io/flannel/subnet/config.go

@@ -18,15 +18,22 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/big"
 
 	"github.com/flannel-io/flannel/pkg/ip"
 )
 
 type Config struct {
+	EnableIPv4    bool
+	EnableIPv6    bool
 	Network       ip.IP4Net
+	IPv6Network   ip.IP6Net
 	SubnetMin     ip.IP4
 	SubnetMax     ip.IP4
+	IPv6SubnetMin *ip.IP6
+	IPv6SubnetMax *ip.IP6
 	SubnetLen     uint
+	IPv6SubnetLen uint
 	BackendType   string          `json:"-"`
 	Backend       json.RawMessage `json:",omitempty"`
 }
@@ -47,11 +54,14 @@ func parseBackendType(be json.RawMessage) (string, error) {
 
 func ParseConfig(s string) (*Config, error) {
 	cfg := new(Config)
+	// Enable ipv4 by default
+	cfg.EnableIPv4 = true
 	err := json.Unmarshal([]byte(s), cfg)
 	if err != nil {
 		return nil, err
 	}
 
+	if cfg.EnableIPv4 {
 		if cfg.SubnetLen > 0 {
 			// SubnetLen needs to allow for a tunnel and bridge device on each host.
 			if cfg.SubnetLen > 30 {
@@ -107,6 +117,64 @@ func ParseConfig(s string) (*Config, error) {
 		if cfg.SubnetMax != cfg.SubnetMax&mask {
 			return nil, fmt.Errorf("SubnetMax is not on a SubnetLen boundary: %v", cfg.SubnetMax)
 		}
+	}
+	if cfg.EnableIPv6 {
+		if cfg.IPv6SubnetLen > 0 {
+			// SubnetLen needs to allow for a tunnel and bridge device on each host.
+			if cfg.IPv6SubnetLen > 126 {
+				return nil, errors.New("SubnetLen must be less than /127")
+			}
+
+			// SubnetLen needs to fit _more_ than twice into the Network.
+			// the first subnet isn't used, so splitting into two one only provide one usable host.
+			if cfg.IPv6SubnetLen < cfg.IPv6Network.PrefixLen+2 {
+				return nil, errors.New("Network must be able to accommodate at least four subnets")
+			}
+		} else {
+			// If the network is smaller than a /124 then the network isn't big enough for flannel so return an error.
+			// Default to giving each host at least a /64 (as long as the network is big enough to support at least four hosts)
+			// Otherwise, if the network is too small to give each host a /64 just split the network into four.
+			if cfg.IPv6Network.PrefixLen > 124 {
+				// Each subnet needs at least four addresses (/126) and the network needs to accommodate at least four
+				// since the first subnet isn't used, so splitting into two would only provide one usable host.
+				// So the min useful PrefixLen is /124
+				return nil, errors.New("IPv6Network is too small. Minimum useful network prefix is /124")
+			} else if cfg.IPv6Network.PrefixLen <= 62 {
+				// Network is big enough to give each host a /64
+				cfg.IPv6SubnetLen = 64
+			} else {
+				// Use +2 to provide four hosts per subnet.
+				cfg.IPv6SubnetLen = cfg.IPv6Network.PrefixLen + 2
+			}
+		}
+
+		ipv6SubnetSize := big.NewInt(0).Lsh(big.NewInt(1), 128-cfg.IPv6SubnetLen)
+
+		if ip.IsEmpty(cfg.IPv6SubnetMin) {
+			// skip over the first subnet otherwise it causes problems. e.g.
+			// if Network is fc00::/48, having an interface with fc00::
+			// conflicts with the broadcast address.
+			cfg.IPv6SubnetMin = ip.GetIPv6SubnetMin(cfg.IPv6Network.IP, ipv6SubnetSize)
+		} else if !cfg.IPv6Network.Contains(cfg.IPv6SubnetMin) {
+			return nil, errors.New("IPv6SubnetMin is not in the range of the IPv6Network")
+		}
+
+		if ip.IsEmpty(cfg.IPv6SubnetMax) {
+			cfg.IPv6SubnetMax = ip.GetIPv6SubnetMax(cfg.IPv6Network.Next().IP, ipv6SubnetSize)
+		} else if !cfg.IPv6Network.Contains(cfg.IPv6SubnetMax) {
+			return nil, errors.New("IPv6SubnetMax is not in the range of the IPv6Network")
+		}
+
+		// The SubnetMin and SubnetMax need to be aligned to a SubnetLen boundary
+		mask := ip.Mask(int(cfg.IPv6SubnetLen))
+		if !ip.CheckIPv6Subnet(cfg.IPv6SubnetMin, mask) {
+			return nil, fmt.Errorf("IPv6SubnetMin is not on a SubnetLen boundary: %v", cfg.IPv6SubnetMin)
+		}
+
+		if !ip.CheckIPv6Subnet(cfg.IPv6SubnetMax, mask) {
+			return nil, fmt.Errorf("IPv6SubnetMax is not on a SubnetLen boundary: %v", cfg.IPv6SubnetMax)
+		}
+	}
 
 	bt, err := parseBackendType(cfg.Backend)
 	if err != nil {

vendor/github.com/flannel-io/flannel/subnet/kube/annotations.go

@@ -23,9 +23,12 @@ import (
 type annotations struct {
 	SubnetKubeManaged          string
 	BackendData                string
+	BackendV6Data              string
 	BackendType                string
 	BackendPublicIP            string
+	BackendPublicIPv6          string
 	BackendPublicIPOverwrite   string
+	BackendPublicIPv6Overwrite string
 }
 
 func newAnnotations(prefix string) (annotations, error) {
@@ -57,9 +60,12 @@ func newAnnotations(prefix string) (annotations, error) {
 	a := annotations{
 		SubnetKubeManaged:          prefix + "kube-subnet-manager",
 		BackendData:                prefix + "backend-data",
+		BackendV6Data:              prefix + "backend-v6-data",
 		BackendType:                prefix + "backend-type",
 		BackendPublicIP:            prefix + "public-ip",
 		BackendPublicIPOverwrite:   prefix + "public-ip-overwrite",
+		BackendPublicIPv6:          prefix + "public-ipv6",
+		BackendPublicIPv6Overwrite: prefix + "public-ipv6-overwrite",
 	}
 
 	return a, nil

vendor/github.com/flannel-io/flannel/subnet/kube/kube.go

@@ -51,6 +51,8 @@ const (
 )
 
 type kubeSubnetManager struct {
+	enableIPv4                bool
+	enableIPv6                bool
 	annotations               annotations
 	client                    clientset.Interface
 	nodeName                  string
@@ -58,9 +60,10 @@ type kubeSubnetManager struct {
 	nodeController            cache.Controller
 	subnetConf                *subnet.Config
 	events                    chan subnet.Event
+	setNodeNetworkUnavailable bool
 }
 
-func NewSubnetManager(ctx context.Context, apiUrl, kubeconfig, prefix, netConfPath string) (subnet.Manager, error) {
+func NewSubnetManager(ctx context.Context, apiUrl, kubeconfig, prefix, netConfPath string, setNodeNetworkUnavailable bool) (subnet.Manager, error) {
 	var cfg *rest.Config
 	var err error
 	// Try to build kubernetes config from a master url or a kubeconfig filepath. If neither masterUrl
@@ -111,6 +114,7 @@ func NewSubnetManager(ctx context.Context, apiUrl, kubeconfig, prefix, netConfPa
 	if err != nil {
 		return nil, fmt.Errorf("error creating network manager: %s", err)
 	}
+	sm.setNodeNetworkUnavailable = setNodeNetworkUnavailable
 	go sm.Run(context.Background())
 
 	log.Infof("Waiting %s for node controller to sync", nodeControllerSyncTimeout)
@@ -132,6 +136,8 @@ func newKubeSubnetManager(ctx context.Context, c clientset.Interface, sc *subnet
 	if err != nil {
 		return nil, err
 	}
+	ksm.enableIPv4 = sc.EnableIPv4
+	ksm.enableIPv6 = sc.EnableIPv6
 	ksm.client = c
 	ksm.nodeName = nodeName
 	ksm.subnetConf = sc
@@ -198,9 +204,20 @@ func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{})
 	if s, ok := n.Annotations[ksm.annotations.SubnetKubeManaged]; !ok || s != "true" {
 		return
 	}
-	if o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
+	var changed = true
+	if ksm.enableIPv4 && o.Annotations[ksm.annotations.BackendData] == n.Annotations[ksm.annotations.BackendData] &&
 		o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
 		o.Annotations[ksm.annotations.BackendPublicIP] == n.Annotations[ksm.annotations.BackendPublicIP] {
+		changed = false
+	}
+
+	if ksm.enableIPv6 && o.Annotations[ksm.annotations.BackendV6Data] == n.Annotations[ksm.annotations.BackendV6Data] &&
+		o.Annotations[ksm.annotations.BackendType] == n.Annotations[ksm.annotations.BackendType] &&
+		o.Annotations[ksm.annotations.BackendPublicIPv6] == n.Annotations[ksm.annotations.BackendPublicIPv6] {
+		changed = false
+	}
+
+	if !changed {
 		return // No change to lease
 	}
 
@@ -226,20 +243,50 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 	if n.Spec.PodCIDR == "" {
 		return nil, fmt.Errorf("node %q pod cidr not assigned", ksm.nodeName)
 	}
-	bd, err := attrs.BackendData.MarshalJSON()
+
+	var bd, v6Bd []byte
+	bd, err = attrs.BackendData.MarshalJSON()
 	if err != nil {
 		return nil, err
 	}
-	_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
+
+	v6Bd, err = attrs.BackendV6Data.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	var cidr, ipv6Cidr *net.IPNet
+	_, cidr, err = net.ParseCIDR(n.Spec.PodCIDR)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, podCidr := range n.Spec.PodCIDRs {
+		_, parseCidr, err := net.ParseCIDR(podCidr)
 		if err != nil {
 			return nil, err
 		}
-	if n.Annotations[ksm.annotations.BackendData] != string(bd) ||
+		if len(parseCidr.IP) == net.IPv6len {
+			ipv6Cidr = parseCidr
+			break
+		}
+	}
+
+	if (n.Annotations[ksm.annotations.BackendData] != string(bd) ||
 		n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
 		n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
 		n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
-		(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String()) {
+		(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) ||
+		(attrs.PublicIPv6 != nil &&
+			(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
+				n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
+				n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
+				n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
+				(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String()))) {
 		n.Annotations[ksm.annotations.BackendType] = attrs.BackendType
+
+		//TODO -i only vxlan and host-gw backends support dual stack now.
+		if (attrs.BackendType == "vxlan" && string(bd) != "null") || attrs.BackendType != "vxlan" {
 			n.Annotations[ksm.annotations.BackendData] = string(bd)
 			if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
 				if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
@@ -251,6 +298,21 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 			} else {
 				n.Annotations[ksm.annotations.BackendPublicIP] = attrs.PublicIP.String()
 			}
+		}
+
+		if (attrs.BackendType == "vxlan" && string(v6Bd) != "null") || (attrs.BackendType == "host-gw" && attrs.PublicIPv6 != nil) {
+			n.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd)
+			if n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" {
+				if n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] {
+					log.Infof("Overriding public ipv6 with '%s' from node annotation '%s'",
+						n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite],
+						ksm.annotations.BackendPublicIPv6Overwrite)
+					n.Annotations[ksm.annotations.BackendPublicIPv6] = n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite]
+				}
+			} else {
+				n.Annotations[ksm.annotations.BackendPublicIPv6] = attrs.PublicIPv6.String()
+			}
+		}
 		n.Annotations[ksm.annotations.SubnetKubeManaged] = "true"
 
 		oldData, err := json.Marshal(cachedNode)
@@ -273,15 +335,32 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 			return nil, err
 		}
 	}
+	if ksm.setNodeNetworkUnavailable {
+		log.Infoln("Setting NodeNetworkUnavailable")
 		err = ksm.setNodeNetworkUnavailableFalse(ctx)
 		if err != nil {
-		log.Errorf("Unable to set NetworkUnavailable to False for %q: %v", ksm.nodeName, err)
+			log.Errorf("Unable to set NodeNetworkUnavailable to False for %q: %v", ksm.nodeName, err)
+		}
+	} else {
+		log.Infoln("Skip setting NodeNetworkUnavailable")
 	}
-	return &subnet.Lease{
-		Subnet:     ip.FromIPNet(cidr),
+
+	lease := &subnet.Lease{
 		Attrs:      *attrs,
 		Expiration: time.Now().Add(24 * time.Hour),
-	}, nil
+	}
+	if cidr != nil {
+		lease.Subnet = ip.FromIPNet(cidr)
+	}
+	if ipv6Cidr != nil {
+		lease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
+	}
+	//TODO - only vxlan and host-gw backends support dual stack now.
+	if attrs.BackendType != "vxlan" && attrs.BackendType != "host-gw" {
+		lease.EnableIPv4 = true
+		lease.EnableIPv6 = false
+	}
+	return lease, nil
 }
 
 func (ksm *kubeSubnetManager) WatchLeases(ctx context.Context, cursor interface{}) (subnet.LeaseWatchResult, error) {
@@ -301,20 +380,43 @@ func (ksm *kubeSubnetManager) Run(ctx context.Context) {
 }
 
 func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error) {
+	if ksm.enableIPv4 {
 		l.Attrs.PublicIP, err = ip.ParseIP4(n.Annotations[ksm.annotations.BackendPublicIP])
 		if err != nil {
 			return l, err
 		}
-
-	l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
 		l.Attrs.BackendData = json.RawMessage(n.Annotations[ksm.annotations.BackendData])
 
 		_, cidr, err := net.ParseCIDR(n.Spec.PodCIDR)
 		if err != nil {
 			return l, err
 		}
-
 		l.Subnet = ip.FromIPNet(cidr)
+		l.EnableIPv4 = ksm.enableIPv4
+	}
+
+	if ksm.enableIPv6 {
+		l.Attrs.PublicIPv6, err = ip.ParseIP6(n.Annotations[ksm.annotations.BackendPublicIPv6])
+		if err != nil {
+			return l, err
+		}
+		l.Attrs.BackendV6Data = json.RawMessage(n.Annotations[ksm.annotations.BackendV6Data])
+
+		ipv6Cidr := new(net.IPNet)
+		for _, podCidr := range n.Spec.PodCIDRs {
+			_, parseCidr, err := net.ParseCIDR(podCidr)
+			if err != nil {
+				return l, err
+			}
+			if len(parseCidr.IP) == net.IPv6len {
+				ipv6Cidr = parseCidr
+				break
+			}
+		}
+		l.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
+		l.EnableIPv6 = ksm.enableIPv6
+	}
+	l.Attrs.BackendType = n.Annotations[ksm.annotations.BackendType]
 	return l, nil
 }
 

vendor/github.com/flannel-io/flannel/subnet/subnet.go

@@ -35,12 +35,17 @@ var (
 
 type LeaseAttrs struct {
 	PublicIP      ip.IP4
+	PublicIPv6    *ip.IP6
 	BackendType   string          `json:",omitempty"`
 	BackendData   json.RawMessage `json:",omitempty"`
+	BackendV6Data json.RawMessage `json:",omitempty"`
 }
 
 type Lease struct {
+	EnableIPv4 bool
+	EnableIPv6 bool
 	Subnet     ip.IP4Net
+	IPv6Subnet ip.IP6Net
 	Attrs      LeaseAttrs
 	Expiration time.Time
 

vendor/github.com/flannel-io/flannel/subnet/watch.go

@@ -17,9 +17,8 @@ package subnet
 import (
 	"time"
 
-	"golang.org/x/net/context"
-
 	"github.com/flannel-io/flannel/pkg/ip"
+	"golang.org/x/net/context"
 	log "k8s.io/klog"
 )
 
@@ -76,13 +75,39 @@ func (lw *leaseWatcher) reset(leases []Lease) []Event {
 	batch := []Event{}
 
 	for _, nl := range leases {
-		if lw.ownLease != nil && nl.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && nl.EnableIPv4 && !nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !nl.EnableIPv4 && nl.EnableIPv6 &&
+			nl.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && nl.EnableIPv4 && nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) &&
+			nl.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !nl.EnableIPv4 && !nl.EnableIPv6 &&
+			nl.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 
 		found := false
 		for i, ol := range lw.leases {
-			if ol.Subnet.Equal(nl.Subnet) {
+			if ol.EnableIPv4 && !ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if ol.EnableIPv4 && !ol.EnableIPv6 && ol.IPv6Subnet.Equal(nl.IPv6Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if ol.EnableIPv4 && ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) &&
+				ol.IPv6Subnet.Equal(nl.IPv6Subnet) {
+				lw.leases = deleteLease(lw.leases, i)
+				found = true
+				break
+			} else if !ol.EnableIPv4 && !ol.EnableIPv6 && ol.Subnet.Equal(nl.Subnet) {
+				//TODO - dual-stack temporarily only compatible with kube subnet manager
 				lw.leases = deleteLease(lw.leases, i)
 				found = true
 				break
@@ -97,7 +122,19 @@ func (lw *leaseWatcher) reset(leases []Lease) []Event {
 
 	// everything left in sm.leases has been deleted
 	for _, l := range lw.leases {
-		if lw.ownLease != nil && l.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && l.EnableIPv4 && !l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !l.EnableIPv4 && l.EnableIPv6 &&
+			l.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && l.EnableIPv4 && l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) &&
+			l.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !l.EnableIPv4 && !l.EnableIPv6 &&
+			l.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 		batch = append(batch, Event{EventRemoved, l})
@@ -114,7 +151,19 @@ func (lw *leaseWatcher) update(events []Event) []Event {
 	batch := []Event{}
 
 	for _, e := range events {
-		if lw.ownLease != nil && e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+		if lw.ownLease != nil && e.Lease.EnableIPv4 && !e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+			continue
+		} else if lw.ownLease != nil && !e.Lease.EnableIPv4 && e.Lease.EnableIPv6 &&
+			e.Lease.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && e.Lease.EnableIPv4 && e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) &&
+			e.Lease.IPv6Subnet.Equal(lw.ownLease.IPv6Subnet) {
+			continue
+		} else if lw.ownLease != nil && !e.Lease.EnableIPv4 && !e.Lease.EnableIPv6 &&
+			e.Lease.Subnet.Equal(lw.ownLease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			continue
 		}
 
@@ -132,12 +181,22 @@ func (lw *leaseWatcher) update(events []Event) []Event {
 
 func (lw *leaseWatcher) add(lease *Lease) Event {
 	for i, l := range lw.leases {
-		if l.Subnet.Equal(lease.Subnet) {
+		if l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if !l.EnableIPv4 && l.EnableIPv6 && l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if l.EnableIPv4 && l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) &&
+			l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases[i] = *lease
+			return Event{EventAdded, lw.leases[i]}
+		} else if !l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			lw.leases[i] = *lease
 			return Event{EventAdded, lw.leases[i]}
 		}
 	}
-
 	lw.leases = append(lw.leases, *lease)
 
 	return Event{EventAdded, lw.leases[len(lw.leases)-1]}
@@ -145,13 +204,24 @@ func (lw *leaseWatcher) add(lease *Lease) Event {
 
 func (lw *leaseWatcher) remove(lease *Lease) Event {
 	for i, l := range lw.leases {
-		if l.Subnet.Equal(lease.Subnet) {
+		if l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if !l.EnableIPv4 && l.EnableIPv6 && l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if l.EnableIPv4 && l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) &&
+			l.IPv6Subnet.Equal(lease.IPv6Subnet) {
+			lw.leases = deleteLease(lw.leases, i)
+			return Event{EventRemoved, l}
+		} else if !l.EnableIPv4 && !l.EnableIPv6 && l.Subnet.Equal(lease.Subnet) {
+			//TODO - dual-stack temporarily only compatible with kube subnet manager
 			lw.leases = deleteLease(lw.leases, i)
 			return Event{EventRemoved, l}
 		}
 	}
 
-	log.Errorf("Removed subnet (%s) was not found", lease.Subnet)
+	log.Errorf("Removed subnet (%s) and ipv6 subnet (%s) were not found", lease.Subnet, lease.IPv6Subnet)
 	return Event{EventRemoved, *lease}
 }
 

vendor/modules.txt

@@ -146,8 +146,7 @@ github.com/beorn7/perks/quantile
 github.com/bits-and-blooms/bitset
 # github.com/blang/semver v3.5.1+incompatible
 github.com/blang/semver
-# github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40
-## explicit
+# github.com/bronze1man/goStrongswanVici v0.0.0-20201105010758-936f38b697fd
 github.com/bronze1man/goStrongswanVici
 # github.com/canonical/go-dqlite v1.5.1
 github.com/canonical/go-dqlite
@@ -525,7 +524,7 @@ github.com/exponent-io/jsonpath
 github.com/fatih/camelcase
 # github.com/felixge/httpsnoop v1.0.1
 github.com/felixge/httpsnoop
-# github.com/flannel-io/flannel v0.14.0
+# github.com/flannel-io/flannel v0.14.1-0.20210827074410-fca1560c91cc
 ## explicit
 github.com/flannel-io/flannel/backend
 github.com/flannel-io/flannel/backend/extension