Merge pull request #719 from erikwilson/encrypted-network-support

.dockerignore

 ./bin
+./etc
 ./build/data
 ./build/data.tar.gz
 ./pkg/data/zz_generated_bindata.go

.gitignore

@@ -11,6 +11,7 @@
 /.lesshst
 /*.log
 /bin
+/etc
 /build
 /data-dir
 /dist

Vagrantfile

-BOX = "generic/alpine39"
+BOX = "generic/alpine310"
 HOME = File.dirname(__FILE__)
 PROJECT = File.basename(HOME)
 MOUNT_TYPE = ENV['MOUNT_TYPE'] || "nfs"

cmd/k3s/main.go

@@ -87,6 +87,9 @@ func stageAndRun(dataDir string, cmd string, args []string) error {
 	if err := os.Setenv("PATH", filepath.Join(dir, "bin")+":"+os.Getenv("PATH")+":"+filepath.Join(dir, "bin/aux")); err != nil {
 		return err
 	}
+	if err := os.Setenv("K3S_DATA_DIR", dir); err != nil {
+		return err
+	}
 
 	cmd, err = exec.LookPath(cmd)
 	if err != nil {

go.mod

@@ -12,7 +12,7 @@ replace (
 	github.com/containerd/go-runc => github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
 	github.com/containerd/typeurl => github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
 	github.com/containernetworking/plugins => github.com/rancher/plugins v0.8.2-k3s.2
-	github.com/coreos/flannel => github.com/ibuildthecloud/flannel v0.10.1-0.20190131215433-823afe66b226
+	github.com/coreos/flannel => github.com/rancher/flannel v0.11.0-k3s.1
 	github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7
 	github.com/docker/distribution => github.com/docker/distribution v0.0.0-20190205005809-0d3efadf0154
 	github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34
@@ -57,6 +57,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e // indirect
+	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
 	github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect
 	github.com/containerd/cgroups v0.0.0-20190923161937-abd0b19954a6 // indirect
 	github.com/containerd/containerd v1.2.8

pkg/agent/config/config.go

@@ -330,8 +330,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 
 	nodeConfig := &config.Node{
 		Docker:                   envInfo.Docker,
-		NoFlannel:                envInfo.NoFlannel,
 		ContainerRuntimeEndpoint: envInfo.ContainerRuntimeEndpoint,
+		FlannelBackend:           controlConfig.FlannelBackend,
 	}
 	nodeConfig.FlannelIface = flannelIface
 	nodeConfig.Images = filepath.Join(envInfo.DataDir, "images")
@@ -349,6 +349,8 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy
 	nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage
+	nodeConfig.AgentConfig.IPSECPSK = controlConfig.IPSECPSK
+	nodeConfig.AgentConfig.StrongSwanDir = filepath.Join(envInfo.DataDir, "strongswan")
 	nodeConfig.CACerts = info.CACerts
 	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
 	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
@@ -361,8 +363,20 @@ func get(envInfo *cmds.Agent) (*config.Node, error) {
 	nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl")
 	nodeConfig.ServerAddress = serverURLParsed.Host
 	nodeConfig.Certificate = servingCert
+
+	if nodeConfig.FlannelBackend == config.FlannelBackendNone {
+		nodeConfig.NoFlannel = true
+	} else {
+		nodeConfig.NoFlannel = envInfo.NoFlannel
+	}
+
 	if !nodeConfig.NoFlannel {
-		nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
+		if envInfo.FlannelConf == "" {
+			nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
+		} else {
+			nodeConfig.FlannelConf = envInfo.FlannelConf
+			nodeConfig.FlannelConfOverride = true
+		}
 		nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
 		nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "etc/cni/net.d")
 	}

pkg/agent/flannel/flannel.go

@@ -29,6 +29,8 @@ import (
 	log "k8s.io/klog"
 
 	// Backends need to be imported for their init() to get executed and them to register
+	_ "github.com/coreos/flannel/backend/extension"
+	_ "github.com/coreos/flannel/backend/ipsec"
 	_ "github.com/coreos/flannel/backend/vxlan"
 )
 
@@ -42,7 +44,7 @@ func flannel(ctx context.Context, flannelIface *net.Interface, flannelConf, kube
 		return err
 	}
 
-	sm, err := kube.NewSubnetManager("", flannelConf, kubeConfigFile, "flannel.alpha.coreos.com")
+	sm, err := kube.NewSubnetManager("", kubeConfigFile, "flannel.alpha.coreos.com", flannelConf)
 	if err != nil {
 		return err
 	}

pkg/agent/flannel/setup.go

@@ -2,6 +2,9 @@ package flannel
 
 import (
 	"context"
+	"fmt"
+	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -22,6 +25,7 @@ const (
     {
       "type":"flannel",
       "delegate":{
+        "hairpinMode":true,
         "forceAddress":true,
         "isDefaultGateway":true
       }
@@ -35,27 +39,45 @@ const (
   ]
 }
 `
-	netJSON = `{
-    "Network": "%CIDR%",
-    "Backend": {
-    "Type": "vxlan"
-    }
+
+	flannelConf = `{
+	"Network": "%CIDR%",
+	"Backend": %backend%
 }
 `
+
+	vxlanBackend = `{
+	"Type": "vxlan"
+}`
+
+	ipsecBackend = `{
+	"Type": "ipsec",
+	"UDPEncap": true,
+	"PSK": "%psk%"
+}`
+
+	wireguardBackend = `{
+	"Type": "extension",
+	"PreStartupCommand": "wg genkey | tee privatekey | wg pubkey",
+	"PostStartupCommand": "export SUBNET_IP=$(echo $SUBNET | cut -d'/' -f 1); ip link del flannel.1 2>/dev/null; echo $PATH >&2; wg-add.sh flannel.1 && wg set flannel.1 listen-port 51820 private-key privatekey && ip addr add $SUBNET_IP/32 dev flannel.1 && ip link set flannel.1 up && ip route add $NETWORK dev flannel.1",
+	"ShutdownCommand": "ip link del flannel.1",
+	"SubnetAddCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY endpoint $PUBLIC_IP:51820 allowed-ips $SUBNET",
+	"SubnetRemoveCommand": "read PUBLICKEY; wg set flannel.1 peer $PUBLICKEY remove"
+}`
 )
 
-func Prepare(ctx context.Context, config *config.Node) error {
-	if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
+func Prepare(ctx context.Context, nodeConfig *config.Node) error {
+	if err := createCNIConf(nodeConfig.AgentConfig.CNIConfDir); err != nil {
 		return err
 	}
 
-	return createFlannelConf(config)
+	return createFlannelConf(nodeConfig)
 }
 
-func Run(ctx context.Context, config *config.Node) error {
-	nodeName := config.AgentConfig.NodeName
+func Run(ctx context.Context, nodeConfig *config.Node) error {
+	nodeName := nodeConfig.AgentConfig.NodeName
 
-	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
+	restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigNode)
 	if err != nil {
 		return err
 	}
@@ -79,7 +101,7 @@ func Run(ctx context.Context, config *config.Node) error {
 	}
 
 	go func() {
-		err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode)
+		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigNode)
 		logrus.Fatalf("flannel exited: %v", err)
 	}()
 
@@ -94,10 +116,53 @@ func createCNIConf(dir string) error {
 	return util.WriteFile(p, cniConf)
 }
 
-func createFlannelConf(config *config.Node) error {
-	if config.FlannelConf == "" {
+func createFlannelConf(nodeConfig *config.Node) error {
+	if nodeConfig.FlannelConf == "" {
 		return nil
 	}
-	return util.WriteFile(config.FlannelConf,
-		strings.Replace(netJSON, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
+	if nodeConfig.FlannelConfOverride {
+		logrus.Infof("Using custom flannel conf defined at %s", nodeConfig.FlannelConf)
+		return nil
+	}
+	confJSON := strings.Replace(flannelConf, "%CIDR%", nodeConfig.AgentConfig.ClusterCIDR.String(), -1)
+
+	var backendConf string
+
+	switch nodeConfig.FlannelBackend {
+	case config.FlannelBackendVXLAN:
+		backendConf = vxlanBackend
+	case config.FlannelBackendIPSEC:
+		backendConf = strings.Replace(ipsecBackend, "%psk%", nodeConfig.AgentConfig.IPSECPSK, -1)
+		if err := setupStrongSwan(nodeConfig); err != nil {
+			return err
+		}
+	case config.FlannelBackendWireguard:
+		backendConf = wireguardBackend
+	default:
+		return fmt.Errorf("Cannot configure unknown flannel backend '%s'", nodeConfig.FlannelBackend)
+	}
+	confJSON = strings.Replace(confJSON, "%backend%", backendConf, -1)
+
+	return util.WriteFile(nodeConfig.FlannelConf, confJSON)
+}
+
+func setupStrongSwan(nodeConfig *config.Node) error {
+	// if we don't know the location of extracted strongswan data then return
+	dataDir := os.Getenv("K3S_DATA_DIR")
+	if dataDir == "" {
+		return nil
+	}
+	dataDir = path.Join(dataDir, "etc", "strongswan")
+
+	info, err := os.Lstat(nodeConfig.AgentConfig.StrongSwanDir)
+	// something exists but is not a symlink, return
+	if err == nil && info.Mode()&os.ModeSymlink == 0 {
+		return nil
+	}
+
+	// clean up strongswan old link
+	os.Remove(nodeConfig.AgentConfig.StrongSwanDir)
+
+	// make new strongswan link
+	return os.Symlink(dataDir, nodeConfig.AgentConfig.StrongSwanDir)
 }

pkg/cli/cmds/agent.go

@@ -22,6 +22,7 @@ type Agent struct {
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
 	FlannelIface             string
+	FlannelConf              string
 	Debug                    bool
 	Rootless                 bool
 	AgentShared
@@ -64,6 +65,11 @@ var (
 		Usage:       "(agent) Override default flannel interface",
 		Destination: &AgentConfig.FlannelIface,
 	}
+	FlannelConfFlag = cli.StringFlag{
+		Name:        "flannel-conf",
+		Usage:       "(agent) (experimental) Override default flannel config file",
+		Destination: &AgentConfig.FlannelConf,
+	}
 	CRIEndpointFlag = cli.StringFlag{
 		Name:        "container-runtime-endpoint",
 		Usage:       "(agent) Disable embedded containerd and use alternative CRI implementation",
@@ -151,6 +157,7 @@ func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
 			DockerFlag,
 			FlannelFlag,
 			FlannelIfaceFlag,
+			FlannelConfFlag,
 			NodeNameFlag,
 			NodeIPFlag,
 			CRIEndpointFlag,

pkg/cli/cmds/server.go

 package cmds
 
 import (
+	"fmt"
+
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/urfave/cli"
 )
 
@@ -30,6 +33,7 @@ type Server struct {
 	AdvertiseIP         string
 	AdvertisePort       int
 	DisableScheduler    bool
+	FlannelBackend      string
 }
 
 var ServerConfig Server
@@ -189,11 +193,18 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "Disable Kubernetes default scheduler",
 				Destination: &ServerConfig.DisableScheduler,
 			},
+			cli.StringFlag{
+				Name:        "flannel-backend",
+				Usage:       fmt.Sprintf("(experimental) One of '%s', '%s', '%s', or '%s'", config.FlannelBackendNone, config.FlannelBackendVXLAN, config.FlannelBackendIPSEC, config.FlannelBackendWireguard),
+				Destination: &ServerConfig.FlannelBackend,
+				Value:       config.FlannelBackendVXLAN,
+			},
 			NodeIPFlag,
 			NodeNameFlag,
 			DockerFlag,
 			FlannelFlag,
 			FlannelIfaceFlag,
+			FlannelConfFlag,
 			CRIEndpointFlag,
 			PauseImageFlag,
 			ResolvConfFlag,

pkg/cli/server/server.go

@@ -84,6 +84,7 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP
 	serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort
 	serverConfig.ControlConfig.BootstrapReadOnly = !cfg.StoreBootstrap
+	serverConfig.ControlConfig.FlannelBackend = cfg.FlannelBackend
 
 	if cmds.AgentConfig.FlannelIface != "" && cmds.AgentConfig.NodeIP == "" {
 		cmds.AgentConfig.NodeIP = netutil.GetIPFromInterface(cmds.AgentConfig.FlannelIface)

pkg/daemons/config/types.go

@@ -13,11 +13,20 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
 
+const (
+	FlannelBackendNone      = "none"
+	FlannelBackendVXLAN     = "vxlan"
+	FlannelBackendIPSEC     = "ipsec"
+	FlannelBackendWireguard = "wireguard"
+)
+
 type Node struct {
 	Docker                   bool
 	ContainerRuntimeEndpoint string
 	NoFlannel                bool
+	FlannelBackend           string
 	FlannelConf              string
+	FlannelConfOverride      bool
 	FlannelIface             *net.Interface
 	Containerd               Containerd
 	Images                   string
@@ -65,6 +74,8 @@ type Agent struct {
 	CNIPlugin           bool
 	NodeTaints          []string
 	NodeLabels          []string
+	IPSECPSK            string
+	StrongSwanDir       string
 }
 
 type Control struct {
@@ -89,6 +100,8 @@ type Control struct {
 	ExtraControllerArgs   []string
 	ExtraSchedulerAPIArgs []string
 	NoLeaderElect         bool
+	FlannelBackend        string
+	IPSECPSK              string
 
 	Runtime *ControlRuntime `json:"-"`
 }
@@ -105,6 +118,7 @@ type ControlRuntimeBootstrap struct {
 	ClientKubeletKey   string
 	ClientKubeProxyKey string
 	ServingKubeletKey  string
+	IPSECKey           string
 }
 
 type ControlRuntime struct {

pkg/daemons/control/server.go

@@ -62,6 +62,11 @@ users:
 `))
 )
 
+const (
+	userTokenSize  = 16
+	ipsecTokenSize = 48
+)
+
 func Server(ctx context.Context, cfg *config.Control) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
@@ -252,6 +257,7 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key")
 	runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt")
 	runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key")
+	runtime.IPSECKey = path.Join(config.DataDir, "cred", "ipsec.psk")
 
 	runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
 	runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
@@ -304,6 +310,10 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
+	if err := genEncryptedNetworkInfo(config, runtime); err != nil {
+		return err
+	}
+
 	if err := storeBootstrapData(ctx, config, etcdClient); err != nil {
 		return err
 	}
@@ -422,20 +432,43 @@ func WritePasswords(passwdFile string, records [][]string) error {
 	return os.Rename(passwdFile+".tmp", passwdFile)
 }
 
+func genEncryptedNetworkInfo(controlConfig *config.Control, runtime *config.ControlRuntime) error {
+	if s, err := os.Stat(runtime.IPSECKey); err == nil && s.Size() > 0 {
+		psk, err := ioutil.ReadFile(runtime.IPSECKey)
+		if err != nil {
+			return err
+		}
+		controlConfig.IPSECPSK = strings.TrimSpace(string(psk))
+		return nil
+	}
+
+	psk, err := getToken(ipsecTokenSize)
+	if err != nil {
+		return err
+	}
+
+	controlConfig.IPSECPSK = psk
+	if err := ioutil.WriteFile(runtime.IPSECKey, []byte(psk+"\n"), 0600); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
 	if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
 		return ensureNodeToken(config, runtime)
 	}
 
-	adminToken, err := getToken()
+	adminToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
-	systemToken, err := getToken()
+	systemToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
-	nodeToken, err := getToken()
+	nodeToken, err := getToken(userTokenSize)
 	if err != nil {
 		return err
 	}
@@ -451,8 +484,8 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
 	})
 }
 
-func getToken() (string, error) {
-	token := make([]byte, 16, 16)
+func getToken(size int) (string, error) {
+	token := make([]byte, size, size)
 	_, err := cryptorand.Read(token)
 	if err != nil {
 		return "", err

scripts/download

@@ -16,3 +16,5 @@ mkdir -p bin/aux && rm bin/mount && ln -sf ../busybox bin/aux/mount
 
 TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz
 curl -sfL https://kubernetes-charts.storage.googleapis.com/${TRAEFIK_FILE} -o ${CHARTS_DIR}/${TRAEFIK_FILE}
+
+cp scripts/wg-add.sh bin/aux/

scripts/package-cli

@@ -22,7 +22,7 @@ rm -rf build/data
 mkdir -p build/data build/out
 mkdir -p dist/artifacts
 
-tar cvzf ./build/out/data.tar.gz --exclude ./bin/hyperkube ./bin
+tar cvzf ./build/out/data.tar.gz --exclude ./bin/hyperkube ./bin ./etc
 HASH=$(sha256sum ./build/out/data.tar.gz | awk '{print $1}')
 
 cp ./build/out/data.tar.gz ./build/data/${HASH}.tgz

scripts/wg-add.sh

+#!/usr/bin/env bash
+
+auto-mtu() {
+	local mtu=0 endpoint output
+	while read -r _ endpoint; do
+		[[ $endpoint =~ ^\[?([a-z0-9:.]+)\]?:[0-9]+$ ]] || continue
+		output="$(ip route get "${BASH_REMATCH[1]}" || true)"
+		[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
+	done < <(wg show "$1" endpoints)
+	if [[ $mtu -eq 0 ]]; then
+		read -r output < <(ip route show default || true) || true
+		[[ ( $output =~ mtu\ ([0-9]+) || ( $output =~ dev\ ([^ ]+) && $(ip link show dev "${BASH_REMATCH[1]}") =~ mtu\ ([0-9]+) ) ) && ${BASH_REMATCH[1]} -gt $mtu ]] && mtu="${BASH_REMATCH[1]}"
+	fi
+	[[ $mtu -gt 0 ]] || mtu=1500
+	ip link set mtu $(( mtu - 80 )) up dev "$1"
+}
+
+# probe for any modules that may be needed
+modprobe wireguard 
+modprobe tun
+
+# try wireguard kernel module first
+ip link add "$1" type wireguard && exit
+
+# try boringtun and let it drop privileges
+boringtun "$1" && auto-mtu "$1" && exit
+
+# try boringtun w/o dropping privileges
+WG_SUDO=1 boringtun "$1" && auto-mtu "$1" && exit
+
+# try wireguard-go - p.s. should not use wireguard-go, it leaks memory
+WG_I_PREFER_BUGGY_USERSPACE_TO_POLISHED_KMOD=1 wireguard-go "$1" && auto-mtu "$1" && exit
+
+exit 1

vendor/github.com/bronze1man/goStrongswanVici/.gitignore

+*.swp

vendor/github.com/bronze1man/goStrongswanVici/.travis.yml

+language: go
+
+services:
+  - docker
+
+go:
+  - "1.11.x"
+
+before_script:
+  - docker-compose up -d strongswan
+
+script:
+  - docker-compose run app go test -v ./...
+
+after_script:
+  - docker-compose down -v

vendor/github.com/bronze1man/goStrongswanVici/Dockerfile

+# Docker image for building the Go app
+FROM golang:latest
+
+WORKDIR /app
+ADD . /app
+
+RUN GO111MODULE=on go mod download

vendor/github.com/bronze1man/goStrongswanVici/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2015 bronze1man
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
\ No newline at end of file

vendor/github.com/bronze1man/goStrongswanVici/README.md

+strongswan vici golang client
+=============================
+[![Build Status](https://travis-ci.org/bronze1man/goStrongswanVici.svg)](https://travis-ci.org/bronze1man/goStrongswanVici)
+[![GoDoc](https://godoc.org/github.com/bronze1man/goStrongswanVici?status.svg)](https://godoc.org/github.com/bronze1man/goStrongswanVici)
+[![docs examples](https://sourcegraph.com/api/repos/github.com/bronze1man/goStrongswanVici/badges/docs-examples.png)](https://sourcegraph.com/github.com/bronze1man/goStrongswanVici)
+[![Total views](https://sourcegraph.com/api/repos/github.com/bronze1man/goStrongswanVici/counters/views.png)](https://sourcegraph.com/github.com/bronze1man/goStrongswanVici)
+[![GitHub issues](https://img.shields.io/github/issues/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/issues)
+[![GitHub stars](https://img.shields.io/github/stars/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/stargazers)
+[![GitHub forks](https://img.shields.io/github/forks/bronze1man/goStrongswanVici.svg)](https://github.com/bronze1man/goStrongswanVici/network)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](https://github.com/bronze1man/goStrongswanVici/blob/master/LICENSE)
+
+a golang implement of strongswan vici plugin client.
+
+### document
+* http://godoc.org/github.com/bronze1man/goStrongswanVici
+* https://github.com/strongswan/strongswan/tree/master/src/libcharon/plugins/vici
+
+### Implemented command list
+* version()
+* list-sas()
+* get-shared()
+* terminate()
+* load-conn()
+* load-cert()
+* load-key()
+* load-pool()
+* load-shared()
+* list-conns()
+* unload-conn()
+* unload-shared()
+
+If you need some commands, but it is not here .you can implement yourself, and send a pull request to this project.
+
+### Testing
+
+To test the library's functionality, `docker-compose` is used to spin up strongswan in a separate Docker container.
+
+```bash
+$ docker-compose up -V
+Creating network "gostrongswanvici_default" with the default drive
+Creating volume "gostrongswanvici_charondata" with default driver
+Creating gostrongswanvici_strongswan_1 ... done
+Creating gostrongswanvici_go-test_1    ... done
+Attaching to gostrongswanvici_strongswan_1, gostrongswanvici_go-test_1
+...
+go-test_1     | ok      github.com/RenaultAI/goStrongswanVici   0.017s
+gostrongswanvici_go-test_1 exited with code 0
+```
+
+### example
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/bronze1man/goStrongswanVici"
+)
+
+func main(){
+    // create a client.
+	client, err := goStrongswanVici.NewClientConnFromDefaultSocket()
+	if err != nil {
+		panic(err)
+	}
+	defer client.Close()
+
+	// get strongswan version
+	v, err := client.Version()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%#v\n", v)
+
+	childConfMap := make(map[string]goStrongswanVici.ChildSAConf)
+        childSAConf := goStrongswanVici.ChildSAConf{
+                Local_ts:      []string{"10.10.59.0/24"},
+                Remote_ts:     []string{"10.10.40.0/24"},
+                ESPProposals:  []string{"aes256-sha256-modp2048"},
+                StartAction:   "trap",
+		CloseAction:   "restart",
+                Mode:          "tunnel",
+                ReqID:         "10",
+                RekeyTime:     "10m",
+                InstallPolicy: "no",
+        }
+        childConfMap["test-child-conn"] = childSAConf
+
+        localAuthConf := goStrongswanVici.AuthConf{
+                AuthMethod: "psk",
+        }
+        remoteAuthConf := goStrongswanVici.AuthConf{
+                AuthMethod: "psk",
+        }
+
+	ikeConfMap := make(map[string] goStrongswanVici.IKEConf)
+
+        ikeConf := goStrongswanVici.IKEConf{
+                LocalAddrs:  []string{"192.168.198.10"},
+                RemoteAddrs: []string{"192.168.198.11"},
+                Proposals:   []string{"aes256-sha256-modp2048"},
+                Version:     "1",
+                LocalAuth:   localAuthConf,
+                RemoteAuth:  remoteAuthConf,
+                Children:    childConfMap,
+                Encap:       "no",
+        }
+
+	ikeConfMap["test-connection"] = ikeConf
+
+	//load connenction information into strongswan
+        err = client.LoadConn(&ikeConfMap)
+        if err != nil {
+                fmt.Printf("error loading connection: %v")
+                panic(err)
+        }
+
+	sharedKey := &goStrongswanVici.Key{
+                Typ:    "IKE",
+                Data:   "this is the key",
+                Owners: []string{"192.168.198.10"}, //IP of the remote host
+        }
+
+	//load shared key into strongswan
+        err = client.LoadShared(sharedKey)
+        if err != nil {
+                fmt.Printf("error returned from loadsharedkey \n")
+                panic(err)
+        }
+
+	//list-conns 
+	connList, err := client.ListConns("")
+	if err != nil {
+		fmt.Printf("error list-conns: %v \n", err)
+	}
+
+	for _, connection := range connList {
+		fmt.Printf("connection map: %v", connection)
+	}	
+
+	// get all conns info from strongswan
+	connInfo, err := client.ListAllVpnConnInfo()
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("found %d connections. \n", len(connInfo))
+
+	//unload connection from strongswan
+	unloadConnReq := &goStrongswanVici.UnloadConnRequest{
+			Name: "test-connection",
+			}
+	err = client.UnloadConn(unloadConnReq)
+	if err != nil {
+		panic(err)
+	}
+
+	// kill all conns in strongswan
+	for _, info := range connInfo {
+		fmt.Printf("kill connection id %s\n", info.Uniqueid)
+		err = client.Terminate(&goStrongswanVici.TerminateRequest{
+			Ike_id: info.Uniqueid,
+		})
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+```

vendor/github.com/bronze1man/goStrongswanVici/client.go

+package goStrongswanVici
+
+import (
+	"net"
+)
+
+type ClientOptions struct {
+	Network string
+	Addr    string
+	// Dialer creates new network connection and has priority over
+	// Network and Addr options.
+	Dialer func() (net.Conn, error)
+}
+
+type Client struct {
+	o ClientOptions
+}
+
+func NewClient(options ClientOptions) (client *Client) {
+	if options.Dialer == nil {
+		options.Dialer = func() (net.Conn, error) {
+			return net.Dial(options.Network, options.Addr)
+		}
+	}
+	return &Client{
+		o: options,
+	}
+}
+
+func NewClientFromDefaultSocket() (client *Client) {
+	return NewClient(ClientOptions{
+		Network: "unix",
+		Addr:    "/var/run/charon.vici",
+	})
+}
+
+func (c *Client) NewConn() (conn *ClientConn, err error) {
+	conn1, err := c.o.Dialer()
+	if err != nil {
+		return nil, err
+	}
+	return NewClientConn(conn1), nil
+}
+
+func (c *Client) ListSas(ike string, ike_id string) (sas []map[string]IkeSa, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.ListSas(ike, ike_id)
+}
+
+func (c *Client) ListAllVpnConnInfo() (list []VpnConnInfo, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.ListAllVpnConnInfo()
+}
+
+func (c *Client) Version() (out *Version, err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.Version()
+}
+
+func (c *Client) Terminate(r *TerminateRequest) (err error) {
+	conn, err := c.NewConn()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+	return conn.Terminate(r)
+}

vendor/github.com/bronze1man/goStrongswanVici/clientConn.go

+package goStrongswanVici
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"time"
+)
+
+const (
+	DefaultReadTimeout = 15 * time.Second
+)
+
+// This object is not thread safe.
+// if you want concurrent, you need create more clients.
+type ClientConn struct {
+	conn          net.Conn
+	responseChan  chan segment
+	eventHandlers map[string]func(response map[string]interface{})
+	lastError     error
+
+	// ReadTimeout specifies a time limit for requests made
+	// by this client.
+	ReadTimeout time.Duration
+}
+
+func (c *ClientConn) Close() error {
+	close(c.responseChan)
+	c.lastError = io.ErrClosedPipe
+	return c.conn.Close()
+}
+
+func NewClientConn(conn net.Conn) (client *ClientConn) {
+	client = &ClientConn{
+		conn:          conn,
+		responseChan:  make(chan segment, 2),
+		eventHandlers: map[string]func(response map[string]interface{}){},
+		ReadTimeout:   DefaultReadTimeout,
+	}
+	go client.readThread()
+	return client
+}
+
+// it dial from unix:///var/run/charon.vici
+func NewClientConnFromDefaultSocket() (client *ClientConn, err error) {
+	conn, err := net.Dial("unix", "/var/run/charon.vici")
+	if err != nil {
+		return
+	}
+	return NewClientConn(conn), nil
+}
+
+func (c *ClientConn) Request(apiname string, request map[string]interface{}) (response map[string]interface{}, err error) {
+	err = writeSegment(c.conn, segment{
+		typ:  stCMD_REQUEST,
+		name: apiname,
+		msg:  request,
+	})
+	if err != nil {
+		fmt.Printf("error writing segment \n")
+		return
+	}
+
+	outMsg := c.readResponse()
+	if c.lastError != nil {
+		return nil, c.lastError
+	}
+	if outMsg.typ != stCMD_RESPONSE {
+		return nil, fmt.Errorf("[%s] response error %d", apiname, outMsg.typ)
+	}
+	return outMsg.msg, nil
+}
+
+func (c *ClientConn) readResponse() segment {
+	select {
+	case outMsg := <-c.responseChan:
+		return outMsg
+	case <-time.After(c.ReadTimeout):
+		if c.lastError == nil {
+			c.lastError = fmt.Errorf("Timeout waiting for message response")
+		}
+		return segment{}
+	}
+}
+
+func (c *ClientConn) RegisterEvent(name string, handler func(response map[string]interface{})) (err error) {
+	if c.eventHandlers[name] != nil {
+		return fmt.Errorf("[event %s] register a event twice.", name)
+	}
+	c.eventHandlers[name] = handler
+	err = writeSegment(c.conn, segment{
+		typ:  stEVENT_REGISTER,
+		name: name,
+	})
+	if err != nil {
+		delete(c.eventHandlers, name)
+		return
+	}
+	outMsg := c.readResponse()
+	//fmt.Printf("registerEvent %#v\n", outMsg)
+	if c.lastError != nil {
+		delete(c.eventHandlers, name)
+		return c.lastError
+	}
+
+	if outMsg.typ != stEVENT_CONFIRM {
+		delete(c.eventHandlers, name)
+		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
+	}
+	return nil
+}
+
+func (c *ClientConn) UnregisterEvent(name string) (err error) {
+	err = writeSegment(c.conn, segment{
+		typ:  stEVENT_UNREGISTER,
+		name: name,
+	})
+	if err != nil {
+		return
+	}
+	outMsg := c.readResponse()
+	//fmt.Printf("UnregisterEvent %#v\n", outMsg)
+	if c.lastError != nil {
+		return c.lastError
+	}
+
+	if outMsg.typ != stEVENT_CONFIRM {
+		return fmt.Errorf("[event %s] response error %d", name, outMsg.typ)
+	}
+	delete(c.eventHandlers, name)
+	return nil
+}
+
+func (c *ClientConn) readThread() {
+	for {
+		outMsg, err := readSegment(c.conn)
+		if err != nil {
+			c.lastError = err
+			return
+		}
+		switch outMsg.typ {
+		case stCMD_RESPONSE, stEVENT_CONFIRM:
+			c.responseChan <- outMsg
+		case stEVENT:
+			handler := c.eventHandlers[outMsg.name]
+			if handler != nil {
+				handler(outMsg.msg)
+			}
+		default:
+			c.lastError = fmt.Errorf("[Client.readThread] unknow msg type %d", outMsg.typ)
+			return
+		}
+	}
+}

vendor/github.com/bronze1man/goStrongswanVici/doc.go

+/*
+a golang implement of strongswan vici plugin client.
+https://github.com/strongswan/strongswan/tree/master/src/libcharon/plugins/vici
+*/
+package goStrongswanVici

vendor/github.com/bronze1man/goStrongswanVici/docker-compose.yaml

+version: '3.3'
+services:
+  strongswan:
+    image: philplckthun/strongswan
+    ports:
+      - "500:500/udp"
+      - "4500:4500/udp"
+      - "1701:1701/udp"
+    privileged: true
+    volumes:
+      - "charondata:/var/run"
+
+  app:
+    build: .
+    volumes:
+      - ".:/app"
+      - "charondata:/var/run"
+    depends_on:
+      - strongswan
+
+volumes:
+  charondata:

vendor/github.com/bronze1man/goStrongswanVici/err.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+func handlePanic(f func() error) (err error) {
+	defer func() {
+		r := recover()
+		//no panic
+		if r == nil {
+			return
+		}
+		//panic a error
+		if e, ok := r.(error); ok {
+			err = e
+			return
+		}
+		//panic another stuff
+		err = fmt.Errorf("%s", r)
+	}()
+	err = f()
+	return
+}

vendor/github.com/bronze1man/goStrongswanVici/go.mod

+module github.com/bronze1man/goStrongswanVici
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.2.2
+)

vendor/github.com/bronze1man/goStrongswanVici/go.sum

+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=

vendor/github.com/bronze1man/goStrongswanVici/initiate.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+// Initiate is used to initiate an SA. This is the
+// equivalent of `swanctl --initiate -c childname`
+func (c *ClientConn) Initiate(child string, ike string) (err error) {
+	inMap := map[string]interface{}{}
+	if child != "" {
+		inMap["child"] = child
+	}
+	if ike != "" {
+		inMap["ike"] = ike
+	}
+	msg, err := c.Request("initiate", inMap)
+	if err != nil {
+		return err
+	}
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful Initiate: %v", msg["errmsg"])
+	}
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/listConns.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+func (c *ClientConn) ListConns(ike string) ([]map[string]IKEConf, error) {
+	conns := []map[string]IKEConf{}
+	var eventErr error
+	var err error
+
+	err = c.RegisterEvent("list-conn", func(response map[string]interface{}) {
+		conn := &map[string]IKEConf{}
+		err = ConvertFromGeneral(response, conn)
+		if err != nil {
+			eventErr = fmt.Errorf("list-conn event error: %v", err)
+			return
+		}
+		conns = append(conns, *conn)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("error registering list-conn event: %v", err)
+	}
+
+	if eventErr != nil {
+		return nil, eventErr
+	}
+
+	reqMap := map[string]interface{}{}
+
+	if ike != "" {
+		reqMap["ike"] = ike
+	}
+
+	_, err = c.Request("list-conns", reqMap)
+	if err != nil {
+		return nil, fmt.Errorf("error requesting list-conns: %v", err)
+	}
+
+	err = c.UnregisterEvent("list-conn")
+	if err != nil {
+		return nil, fmt.Errorf("error unregistering list-conns event: %v", err)
+	}
+
+	return conns, nil
+}

vendor/github.com/bronze1man/goStrongswanVici/listSas.go

+package goStrongswanVici
+
+import (
+	"fmt"
+	"strconv"
+)
+
+//from list-sa event
+type IkeSa struct {
+	Uniqueid        string               `json:"uniqueid"` //called ike_id in terminate() argument.
+	Version         string               `json:"version"`
+	State           string               `json:"state"` //had saw: ESTABLISHED
+	Local_host      string               `json:"local-host"`
+	Local_port      string               `json:"local-port"`
+	Local_id        string               `json:"local-id"`
+	Remote_host     string               `json:"remote-host"`
+	Remote_port     string               `json:"remote-port"`
+	Remote_id       string               `json:"remote-id"`
+	Remote_xauth_id string               `json:"remote-xauth-id"` //client username
+	Initiator       string               `json:"initiator"`
+	Initiator_spi   string               `json:"initiator-spi"`
+	Responder_spi   string               `json:"responder-spi"`
+	Encr_alg        string               `json:"encr-alg"`
+	Encr_keysize    string               `json:"encr-keysize"`
+	Integ_alg       string               `json:"integ-alg"`
+	Integ_keysize   string               `json:"integ-keysize"`
+	Prf_alg         string               `json:"prf-alg"`
+	Dh_group        string               `json:"dh-group"`
+	Established     string               `json:"established"`
+	Rekey_time      string               `json:"rekey-time"`
+	Reauth_time     string               `json:"reauth-time"`
+	Remote_vips     []string             `json:"remote-vips"`
+	Child_sas       map[string]Child_sas `json:"child-sas"` //key means child-sa-name(conn name in ipsec.conf)
+}
+
+type Child_sas struct {
+	Reqid         string   `json:"reqid"`
+	State         string   `json:"state"` //had saw: INSTALLED
+	Mode          string   `json:"mode"`  //had saw: TUNNEL
+	Protocol      string   `json:"protocol"`
+	Encap         string   `json:"encap"`
+	Spi_in        string   `json:"spi-in"`
+	Spi_out       string   `json:"spi-out"`
+	Cpi_in        string   `json:"cpi-in"`
+	Cpi_out       string   `json:"cpi-out"`
+	Encr_alg      string   `json:"encr-alg"`
+	Encr_keysize  string   `json:"encr-keysize"`
+	Integ_alg     string   `json:"integ-alg"`
+	Integ_keysize string   `json:"integ-keysize"`
+	Prf_alg       string   `json:"prf-alg"`
+	Dh_group      string   `json:"dh-group"`
+	Esn           string   `json:"esn"`
+	Bytes_in      string   `json:"bytes-in"` //bytes into this machine
+	Packets_in    string   `json:"packets-in"`
+	Use_in        string   `json:"use-in"`
+	Bytes_out     string   `json:"bytes-out"` // bytes out of this machine
+	Packets_out   string   `json:"packets-out"`
+	Use_out       string   `json:"use-out"`
+	Rekey_time    string   `json:"rekey-time"`
+	Life_time     string   `json:"life-time"`
+	Install_time  string   `json:"install-time"`
+	Local_ts      []string `json:"local-ts"`
+	Remote_ts     []string `json:"remote-ts"`
+}
+
+func (s *Child_sas) GetBytesIn() uint64 {
+	num, err := strconv.ParseUint(s.Bytes_in, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return num
+}
+
+func (s *Child_sas) GetBytesOut() uint64 {
+	num, err := strconv.ParseUint(s.Bytes_out, 10, 64)
+	if err != nil {
+		return 0
+	}
+	return num
+}
+
+// To be simple, list all clients that are connecting to this server .
+// A client is a sa.
+// Lists currently active IKE_SAs
+func (c *ClientConn) ListSas(ike string, ike_id string) (sas []map[string]IkeSa, err error) {
+	sas = []map[string]IkeSa{}
+	var eventErr error
+	//register event
+	err = c.RegisterEvent("list-sa", func(response map[string]interface{}) {
+		sa := &map[string]IkeSa{}
+		err = ConvertFromGeneral(response, sa)
+		if err != nil {
+			fmt.Printf("list-sa event error: %s\n", err)
+			eventErr = err
+			return
+		}
+		sas = append(sas, *sa)
+		//fmt.Printf("event %#v\n", response)
+	})
+	if err != nil {
+		return
+	}
+	if eventErr != nil {
+		return
+	}
+
+	inMap := map[string]interface{}{}
+	if ike != "" {
+		inMap["ike"] = ike
+	}
+	if ike_id != "" {
+		inMap["ike_id"] = ike_id
+	}
+	_, err = c.Request("list-sas", inMap)
+	if err != nil {
+		return
+	}
+	//fmt.Printf("request finish %#v\n", sas)
+	err = c.UnregisterEvent("list-sa")
+	if err != nil {
+		return
+	}
+	return
+}
+
+//a vpn conn in the strongswan server
+type VpnConnInfo struct {
+	IkeSa
+	Child_sas
+	IkeSaName   string //looks like conn name in ipsec.conf, content is same as ChildSaName
+	ChildSaName string //looks like conn name in ipsec.conf
+}
+
+func (c *VpnConnInfo) GuessUserName() string {
+	if c.Remote_xauth_id != "" {
+		return c.Remote_xauth_id
+	}
+	if c.Remote_id != "" {
+		return c.Remote_id
+	}
+	return ""
+}
+
+// a helper method to avoid complex data struct in ListSas
+// if it only have one child_sas ,it will put it into info.Child_sas
+func (c *ClientConn) ListAllVpnConnInfo() (list []VpnConnInfo, err error) {
+	sasList, err := c.ListSas("", "")
+	if err != nil {
+		return
+	}
+	list = make([]VpnConnInfo, len(sasList))
+	for i, sa := range sasList {
+		info := VpnConnInfo{}
+		if len(sa) != 1 {
+			fmt.Printf("[vici.ListAllVpnConnInfo] warning: len(sa)[%d]!=1\n", len(sa))
+		}
+		for ikeSaName, ikeSa := range sa {
+			info.IkeSaName = ikeSaName
+			info.IkeSa = ikeSa
+			//if len(ikeSa.Child_sas) != 1 {
+			//	fmt.Println("[vici.ListAllVpnConnInfo] warning: len(ikeSa.Child_sas)[%d]!=1", len(ikeSa.Child_sas))
+			//}
+			for childSaName, childSa := range ikeSa.Child_sas {
+				info.ChildSaName = childSaName
+				info.Child_sas = childSa
+				break
+			}
+			break
+		}
+		if len(info.IkeSa.Child_sas) == 1 {
+			info.IkeSa.Child_sas = nil
+		}
+		list[i] = info
+	}
+	return
+}

vendor/github.com/bronze1man/goStrongswanVici/loadCert.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type certPayload struct {
+	Typ  string `json:"type"` // (X509|X509_AC|X509_CRL)
+	Flag string `json:"flag"` // (CA|AA|OCSP|NONE)
+	Data string `json:"data"`
+}
+
+func (c *ClientConn) LoadCertificate(s string, typ string, flag string) (err error) {
+	requestMap := &map[string]interface{}{}
+
+	var k = certPayload{
+		Typ:  typ,
+		Flag: flag,
+		Data: s,
+	}
+
+	if err = ConvertToGeneral(k, requestMap); err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-cert", *requestMap)
+
+	if err != nil {
+		return fmt.Errorf("unsuccessful loadCert: %v", err.Error())
+	}
+
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful loadCert: %v", msg["success"])
+	}
+
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/loadConn.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type Connection struct {
+	ConnConf map[string]IKEConf `json:"connections"`
+}
+
+type IKEConf struct {
+	LocalAddrs  []string               `json:"local_addrs"`
+	RemoteAddrs []string               `json:"remote_addrs,omitempty"`
+	Proposals   []string               `json:"proposals,omitempty"`
+	Version     string                 `json:"version"` //1 for ikev1, 0 for ikev1 & ikev2
+	Encap       string                 `json:"encap"`   //yes,no
+	KeyingTries string                 `json:"keyingtries"`
+	RekeyTime   string                 `json:"rekey_time"`
+	DPDDelay    string                 `json:"dpd_delay,omitempty"`
+	LocalAuth   AuthConf               `json:"local"`
+	RemoteAuth  AuthConf               `json:"remote"`
+	Pools       []string               `json:"pools,omitempty"`
+	Children    map[string]ChildSAConf `json:"children"`
+}
+
+type AuthConf struct {
+	ID         string `json:"id"`
+	Round      string `json:"round,omitempty"`
+	AuthMethod string `json:"auth"` // (psk|pubkey)
+	EAP_ID     string `json:"eap_id,omitempty"`
+}
+
+type ChildSAConf struct {
+	Local_ts      []string `json:"local_ts"`
+	Remote_ts     []string `json:"remote_ts"`
+	ESPProposals  []string `json:"esp_proposals,omitempty"` //aes128-sha1_modp1024
+	StartAction   string   `json:"start_action"`            //none,trap,start
+	CloseAction   string   `json:"close_action"`
+	ReqID         string   `json:"reqid,omitempty"`
+	RekeyTime     string   `json:"rekey_time"`
+	ReplayWindow  string   `json:"replay_window,omitempty"`
+	Mode          string   `json:"mode"`
+	InstallPolicy string   `json:"policies"`
+	UpDown        string   `json:"updown,omitempty"`
+	Priority      string   `json:"priority,omitempty"`
+	MarkIn        string   `json:"mark_in,omitempty"`
+	MarkOut       string   `json:"mark_out,omitempty"`
+	DpdAction     string   `json:"dpd_action,omitempty"`
+	LifeTime      string   `json:"life_time,omitempty"`
+}
+
+func (c *ClientConn) LoadConn(conn *map[string]IKEConf) error {
+	requestMap := &map[string]interface{}{}
+
+	err := ConvertToGeneral(conn, requestMap)
+
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-conn", *requestMap)
+
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful LoadConn: %v", msg["errmsg"])
+	}
+
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/loadPrivateKey.go

+package goStrongswanVici
+
+import (
+	"crypto/ecdsa"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+)
+
+type keyPayload struct {
+	Typ  string `json:"type"`
+	Data string `json:"data"`
+}
+
+// LoadECDSAPrivateKey encodes a *ecdsa.PrivateKey as a PEM block before sending
+// it to the Vici interface
+func (c *ClientConn) LoadECDSAPrivateKey(key *ecdsa.PrivateKey) error {
+	mk, err := x509.MarshalECPrivateKey(key)
+
+	if err != nil {
+		return err
+	}
+
+	var pemData = pem.EncodeToMemory(&pem.Block{
+		Type:  "ECDSA PRIVATE KEY",
+		Bytes: mk,
+	})
+
+	return c.loadPrivateKey("ECDSA", string(pemData))
+}
+
+// LoadRSAPrivateKey encodes a *rsa.PrivateKey as a PEM block before sending
+// it to the Vici interface
+func (c *ClientConn) LoadRSAPrivateKey(key *rsa.PrivateKey) error {
+	var mk = x509.MarshalPKCS1PrivateKey(key)
+
+	var pemData = pem.EncodeToMemory(&pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: mk,
+	})
+
+	return c.loadPrivateKey("RSA", string(pemData))
+}
+
+// loadPrivateKey expects typ to be (RSA|ECDSA) and a PEM encoded data as a
+// string
+func (c *ClientConn) loadPrivateKey(typ, data string) (err error) {
+	requestMap := &map[string]interface{}{}
+
+	var k = keyPayload{
+		Typ:  typ,
+		Data: data,
+	}
+
+	if err = ConvertToGeneral(k, requestMap); err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-key", *requestMap)
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful loadPrivateKey: %v", msg["success"])
+	}
+
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/marshal.go

+package goStrongswanVici
+
+import (
+	"encoding/json"
+)
+
+//concrete data type to general data type
+// concrete data type like *Version
+// general data type include map[string]interface{} []string string
+// TODO make it faster
+func ConvertToGeneral(concrete interface{}, general interface{}) (err error) {
+	b, err := json.Marshal(concrete)
+	if err != nil {
+		return
+	}
+	return json.Unmarshal(b, general)
+}
+
+// general data type to concrete data type
+// concrete data type like *Version
+// general data type include map[string]interface{} []string string
+// TODO make it faster
+func ConvertFromGeneral(general interface{}, concrete interface{}) (err error) {
+	b, err := json.Marshal(general)
+	if err != nil {
+		return
+	}
+	return json.Unmarshal(b, concrete)
+}

vendor/github.com/bronze1man/goStrongswanVici/monitorSA.go

+package goStrongswanVici
+
+import (
+	"bytes"
+	"encoding/json"
+	"time"
+)
+
+const (
+	EVENT_IKE_UPDOWN   = "ike-updown"
+	EVENT_IKE_REKEY    = "ike-rekey"
+	EVENT_CHILD_UPDOWN = "child-updown"
+	EVENT_CHILD_REKEY  = "child-rekey"
+)
+
+type EventIkeSAUpDown struct {
+	Child_sas     map[string]*EventChildSAUpDown `json:"child-sas"`
+	Dh_group      string                         `json:"dh-group"`
+	Encr_keysize  string                         `json:"encr-keysize"`
+	Encr_alg      string                         `json:"encr-alg"`
+	Established   string                         `json:"established"`
+	Initiator_spi string                         `json:"initiator-spi"`
+	Integ_alg     string                         `json:"integ-alg"`
+	Local_id      string                         `json:"local-id"`
+	Local_host    string                         `json:"local-host"`
+	Local_port    string                         `json:"local-port"`
+	Nat_any       string                         `json:"nat-any"`
+	Nat_remote    string                         `json:"nat-remote"`
+	Prf_alg       string                         `json:"prf-alg"`
+	Rekey_time    string                         `json:"rekey-time"`
+	Remote_id     string                         `json:"remote-id"`
+	Remote_host   string                         `json:"remote-host"`
+	Remote_port   string                         `json:"remote-port"`
+	Responder_spi string                         `json:"responder-spi"`
+	State         string                         `json:"state"`
+	Task_Active   []string                       `json:"tasks-active"`
+	Uniqueid      string                         `json:"uniqueid"`
+	Version       string                         `json:"version"`
+}
+
+type EventChildSAUpDown struct {
+	Bytes_in     string   `json:"bytes-in"`
+	Bytes_out    string   `json:"bytes-out"`
+	Encap        string   `json:"encap"`
+	Encr_alg     string   `json:"encr-alg"`
+	Encr_keysize string   `json:"encr-keysize"`
+	Integ_alg    string   `json:"integ-alg"`
+	Install_time string   `json:"install-time"`
+	Life_time    string   `json:"life-time"`
+	Local_ts     []string `json:"local-ts"`
+	Mode         string   `json:"mode"`
+	Name         string   `json:"name"`
+	Protocol     string   `json:"protocol"`
+	Packets_out  string   `json:"packets-out"`
+	Packets_in   string   `json:"packets-in"`
+	Rekey_time   string   `json:"rekey-time"`
+	Remote_ts    []string `json:"remote-ts"`
+	Reqid        string   `json:"reqid"`
+	Spi_in       string   `json:"spi-in"`
+	Spi_out      string   `json:"spi-out"`
+	State        string   `json:"state"`
+	UniqueId     string   `json:"uniqueid"`
+}
+
+type EventIkeRekeyPair struct {
+	New EventIkeRekeySA `json:"new"`
+	Old EventIkeRekeySA `json:"old"`
+}
+
+type EventIkeRekeySA struct {
+	Child_sas     map[string]*EventChildRekeyPair `json:"child-sas"`
+	Dh_group      string                          `json:"dh-group"`
+	Encr_alg      string                          `json:"encr-alg"`
+	Encr_keysize  string                          `json:"encr-keysize"`
+	Established   string                          `json:"established"`
+	Initiator_spi string                          `json:"initiator-spi"`
+	Integ_alg     string                          `json:"integ-alg"`
+	Local_host    string                          `json:"local-host"`
+	Local_port    string                          `json:"local-port"`
+	Local_id      string                          `json:"local-id"`
+	Nat_any       string                          `json:"nat-any"`
+	Nat_remote    string                          `json:"nat-remote"`
+	Prf_alg       string                          `json:"prf-alg"`
+	Rekey_time    string                          `json:"rekey-time"`
+	Remote_id     string                          `json:"remote-id"`
+	Remote_host   string                          `json:"remote-host"`
+	Remote_port   string                          `json:"remote-port"`
+	Responder_spi string                          `json:"responder-spi"`
+	State         string                          `json:"state"`
+	Task_Active   []string                        `json:"tasks-active"`
+	Task_Passive  []string                        `json:"tasks-passive"`
+	Uniqueid      string                          `json:"uniqueid"`
+	Version       string                          `json:"version"`
+}
+
+type EventChildRekeyPair struct {
+	New EventChildRekeySA `json:"new"`
+	Old EventChildRekeySA `json:"old"`
+}
+
+type EventChildRekeySA struct {
+	Bytes_in     string   `json:"bytes-in"`
+	Bytes_out    string   `json:"bytes-out"`
+	Encap        string   `json:"encap"`
+	Encr_alg     string   `json:"encr-alg"`
+	Encr_keysize string   `json:"encr-keysize"`
+	Integ_alg    string   `json:"integ-alg"`
+	Install_time string   `json:"install-time"`
+	Life_time    string   `json:"life-time"`
+	Local_ts     []string `json:"local-ts"`
+	Mode         string   `json:"mode"`
+	Name         string   `json:"name"`
+	Packets_in   string   `json:"packets-in"`
+	Packets_out  string   `json:"packets-out"`
+	Protocol     string   `json:"protocol"`
+	Remote_ts    []string `json:"remote-ts"`
+	Rekey_time   string   `json:"rekey-time"`
+	Reqid        string   `json:"reqid"`
+	Spi_in       string   `json:"spi-in"`
+	Spi_out      string   `json:"spi-out"`
+	State        string   `json:"state"`
+	Use_in       string   `json:"use-in"`
+	Use_out      string   `json:"use-out"`
+	UniqueId     string   `json:"uniqueid"`
+}
+
+type EventIkeUpDown struct {
+	Up  bool
+	Ike map[string]*EventIkeSAUpDown
+}
+
+type EventIkeRekey struct {
+	Ike map[string]*EventIkeRekeyPair
+}
+
+type EventChildRekey struct {
+	Ike map[string]*EventIkeRekeySA
+}
+
+type EventChildUpDown struct {
+	Up  bool
+	Ike map[string]*EventIkeSAUpDown
+}
+
+type EventIkeSa struct {
+	IkeSa
+	TasksActive []string `json:"tasks-active"`
+}
+
+type EventInfo struct {
+	Up  bool
+	Ike map[string]*EventIkeSa
+}
+
+func prettyprint(b []byte) string {
+	var out bytes.Buffer
+	json.Indent(&out, b, "", "  ")
+	return string(out.Bytes())
+}
+
+type monitorCallBack func(event string, info interface{})
+
+
+func handleIkeUpDown(eventName string, callback monitorCallBack, response map[string]interface{}) {
+	event := &EventIkeUpDown{}
+	event.Ike = map[string]*EventIkeSAUpDown{}
+	//we need to marshall all ikes manual because json uses connections names as key
+	for name := range response {
+		value := response[name]
+		if name == "up" {
+			event.Up = true
+		} else {
+			sa := &EventIkeSAUpDown{}
+			ConvertFromGeneral(value, sa)
+			event.Ike[name] = sa
+		}
+	}
+	callback(eventName, event)
+}
+
+func handleIkeRekey(eventName string, callback monitorCallBack, response map[string]interface{}) {
+	event := &EventIkeRekey{}
+	event.Ike = map[string]*EventIkeRekeyPair{}
+	//we need to marshall all ikes manual because json uses connections names as key
+	for name := range response {
+		value := response[name]
+		sa := &EventIkeRekeyPair{}
+		ConvertFromGeneral(value, sa)
+		event.Ike[name] = sa
+	}
+	callback(eventName, event)
+}
+
+func handleChildUpDown(eventName string, callback monitorCallBack, response map[string]interface{}) {
+	event := &EventChildUpDown{}
+	event.Ike = map[string]*EventIkeSAUpDown{}
+	//we need to marshall all ikes manual because json uses connections names as key
+	for name := range response {
+		value := response[name]
+		if name == "up" {
+			event.Up = true
+		} else {
+			sa := &EventIkeSAUpDown{}
+			ConvertFromGeneral(value, sa)
+			event.Ike[name] = sa
+		}
+	}
+	callback(eventName, event)
+}
+
+func handleChildRekey(eventName string, callback monitorCallBack, response map[string]interface{}) {
+	event := &EventChildRekey{}
+	event.Ike = map[string]*EventIkeRekeySA{}
+	//we need to marshall all ikes manual because json uses connections names as key
+	for name := range response {
+		value := response[name]
+		sa := &EventIkeRekeySA{}
+		ConvertFromGeneral(value, sa)
+		event.Ike[name] = sa
+	}
+	callback(eventName, event)
+}
+
+func (c *ClientConn) MonitorSA(callback monitorCallBack,watchdog time.Duration) (err error) {
+	//register event
+	c.RegisterEvent(EVENT_CHILD_UPDOWN, func(response map[string]interface{}) {
+		//dumpResponse(response)
+		handleChildUpDown(EVENT_CHILD_UPDOWN, callback, response)
+	})
+	c.RegisterEvent(EVENT_CHILD_REKEY, func(response map[string]interface{}) {
+		//dumpResponse(response)
+		handleChildRekey(EVENT_CHILD_REKEY, callback, response)
+	})
+	c.RegisterEvent(EVENT_IKE_UPDOWN, func(response map[string]interface{}) {
+		//dumpResponse(response)
+		handleIkeUpDown(EVENT_IKE_UPDOWN, callback, response)
+	})
+	c.RegisterEvent(EVENT_IKE_REKEY, func(response map[string]interface{}) {
+		//dumpResponse(response)
+		handleIkeRekey(EVENT_IKE_REKEY, callback, response)
+	})
+
+	for {
+		time.Sleep(watchdog)
+		//collect some daemon stats to see if connection is alive
+		if _, err := c.Stats(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/msg.go

+package goStrongswanVici
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"strconv"
+)
+
+type segmentType byte
+
+const (
+	stCMD_REQUEST      segmentType = 0
+	stCMD_RESPONSE                 = 1
+	stCMD_UNKNOWN                  = 2
+	stEVENT_REGISTER               = 3
+	stEVENT_UNREGISTER             = 4
+	stEVENT_CONFIRM                = 5
+	stEVENT_UNKNOWN                = 6
+	stEVENT                        = 7
+)
+
+func (t segmentType) hasName() bool {
+	switch t {
+	case stCMD_REQUEST, stEVENT_REGISTER, stEVENT_UNREGISTER, stEVENT:
+		return true
+	}
+	return false
+}
+func (t segmentType) isValid() bool {
+	switch t {
+	case stCMD_REQUEST, stCMD_RESPONSE, stCMD_UNKNOWN, stEVENT_REGISTER,
+		stEVENT_UNREGISTER, stEVENT_CONFIRM, stEVENT_UNKNOWN, stEVENT:
+		return true
+	}
+	return false
+}
+
+func (t segmentType) hasMsg() bool {
+	switch t {
+	case stCMD_REQUEST, stCMD_RESPONSE, stEVENT:
+		return true
+	}
+	return false
+}
+
+type elementType byte
+
+const (
+	etSECTION_START elementType = 1
+	etSECTION_END               = 2
+	etKEY_VALUE                 = 3
+	etLIST_START                = 4
+	etLIST_ITEM                 = 5
+	etLIST_END                  = 6
+)
+
+type segment struct {
+	typ  segmentType
+	name string
+	msg  map[string]interface{}
+}
+
+//msg 在内部以下列3种类型表示(降低复杂度)
+// string
+// map[string]interface{}
+// []string
+func writeSegment(w io.Writer, msg segment) (err error) {
+	if !msg.typ.isValid() {
+		return fmt.Errorf("[writeSegment] msg.typ %d not defined", msg.typ)
+	}
+	buf := &bytes.Buffer{}
+	buf.WriteByte(byte(msg.typ))
+	//name
+	if msg.typ.hasName() {
+		err = writeString1(buf, msg.name)
+		if err != nil {
+			fmt.Printf("error returned from writeString1i \n")
+			return
+		}
+	}
+
+	if msg.typ.hasMsg() {
+		err = writeMap(buf, msg.msg)
+		if err != nil {
+			fmt.Printf("error retruned from writeMap \n")
+			return
+		}
+	}
+
+	//写长度
+	err = binary.Write(w, binary.BigEndian, uint32(buf.Len()))
+	if err != nil {
+		fmt.Printf("[writeSegment] error writing to binary \n")
+		return
+	}
+
+	_, err = buf.WriteTo(w)
+	if err != nil {
+		fmt.Printf("[writeSegment] error writing to buffer \n")
+		return
+	}
+
+	return nil
+}
+
+func readSegment(inR io.Reader) (msg segment, err error) {
+	//长度
+	var length uint32
+	err = binary.Read(inR, binary.BigEndian, &length)
+	if err != nil {
+		return
+	}
+	r := bufio.NewReader(&io.LimitedReader{
+		R: inR,
+		N: int64(length),
+	})
+	//类型
+	c, err := r.ReadByte()
+	if err != nil {
+		return
+	}
+	msg.typ = segmentType(c)
+	if !msg.typ.isValid() {
+		return msg, fmt.Errorf("[readSegment] msg.typ %d not defined", msg.typ)
+	}
+	if msg.typ.hasName() {
+		msg.name, err = readString1(r)
+		if err != nil {
+			return
+		}
+	}
+	if msg.typ.hasMsg() {
+		msg.msg, err = readMap(r, true)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+//一个字节长度的字符串
+func writeString1(w *bytes.Buffer, s string) (err error) {
+	length := len(s)
+	if length > 255 {
+		return fmt.Errorf("[writeString1] length>255")
+	}
+	w.WriteByte(byte(length))
+	w.WriteString(s)
+	return
+}
+
+func readString1(r *bufio.Reader) (s string, err error) {
+	length, err := r.ReadByte()
+	if err != nil {
+		return
+	}
+	buf := make([]byte, length)
+	_, err = io.ReadFull(r, buf)
+	if err != nil {
+		return
+	}
+	return string(buf), nil
+}
+
+//两个字节长度的字符串
+func writeString2(w *bytes.Buffer, s string) (err error) {
+	length := len(s)
+	if length > 65535 {
+		return fmt.Errorf("[writeString2] length>65535")
+	}
+	binary.Write(w, binary.BigEndian, uint16(length))
+	w.WriteString(s)
+	return
+}
+
+func readString2(r io.Reader) (s string, err error) {
+	var length uint16
+	err = binary.Read(r, binary.BigEndian, &length)
+	if err != nil {
+		return
+	}
+	buf := make([]byte, length)
+	_, err = io.ReadFull(r, buf)
+	if err != nil {
+		return
+	}
+	return string(buf), nil
+}
+
+func writeKeyMap(w *bytes.Buffer, name string, msg map[string]interface{}) (err error) {
+	w.WriteByte(byte(etSECTION_START))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	writeMap(w, msg)
+	w.WriteByte(byte(etSECTION_END))
+	return nil
+}
+
+func writeKeyList(w *bytes.Buffer, name string, msg []string) (err error) {
+	w.WriteByte(byte(etLIST_START))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	for _, s := range msg {
+		w.WriteByte(byte(etLIST_ITEM))
+		err = writeString2(w, s)
+		if err != nil {
+			return
+		}
+	}
+	w.WriteByte(byte(etLIST_END))
+	return nil
+}
+
+func writeKeyString(w *bytes.Buffer, name string, msg string) (err error) {
+	w.WriteByte(byte(etKEY_VALUE))
+	err = writeString1(w, name)
+	if err != nil {
+		return
+	}
+	err = writeString2(w, msg)
+	return
+}
+
+func writeMap(w *bytes.Buffer, msg map[string]interface{}) (err error) {
+	for k, v := range msg {
+		switch t := v.(type) {
+		case map[string]interface{}:
+			writeKeyMap(w, k, t)
+		case []string:
+			writeKeyList(w, k, t)
+		case string:
+			writeKeyString(w, k, t)
+		case []interface{}:
+			str := make([]string, len(t))
+			for i := range t {
+				str[i] = t[i].(string)
+			}
+			writeKeyList(w, k, str)
+		default:
+			return fmt.Errorf("[writeMap] can not write type %T right now", msg)
+		}
+	}
+	return nil
+}
+
+//SECTION_START has been read already.
+func readKeyMap(r *bufio.Reader) (key string, msg map[string]interface{}, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg, err = readMap(r, false)
+	return
+}
+
+//LIST_START has been read already.
+func readKeyList(r *bufio.Reader) (key string, msg []string, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg = []string{}
+	for {
+		var c byte
+		c, err = r.ReadByte()
+		if err != nil {
+			return
+		}
+		switch elementType(c) {
+		case etLIST_ITEM:
+			value, err := readString2(r)
+			if err != nil {
+				return "", nil, err
+			}
+			msg = append(msg, value)
+		case etLIST_END: //end of outer list
+			return key, msg, nil
+		default:
+			return "", nil, fmt.Errorf("[readKeyList] protocol error 2")
+		}
+	}
+	return
+}
+
+//KEY_VALUE has been read already.
+func readKeyString(r *bufio.Reader) (key string, msg string, err error) {
+	key, err = readString1(r)
+	if err != nil {
+		return
+	}
+	msg, err = readString2(r)
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Since the original key chosen can have duplicates,
+// this function is used to map the original key to a new one
+// to make them unique.
+func getNewKeyToHandleDuplicates(key string, msg map[string]interface{}) string {
+	if _, ok := msg[key]; !ok {
+		return key
+	}
+
+	for i := 0; ; i++ {
+		newKey := key + "##" + strconv.Itoa(i)
+		if _, ok := msg[newKey]; !ok {
+			return newKey
+		}
+	}
+}
+
+//SECTION_START has been read already.
+func readMap(r *bufio.Reader, isRoot bool) (msg map[string]interface{}, err error) {
+	msg = map[string]interface{}{}
+	for {
+		c, err := r.ReadByte()
+		if err == io.EOF && isRoot { //may be root section
+			return msg, nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		switch elementType(c) {
+		case etSECTION_START:
+			key, value, err := readKeyMap(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[getNewKeyToHandleDuplicates(key, msg)] = value
+		case etLIST_START:
+			key, value, err := readKeyList(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[getNewKeyToHandleDuplicates(key, msg)] = value
+		case etKEY_VALUE:
+			key, value, err := readKeyString(r)
+			if err != nil {
+				return nil, err
+			}
+			msg[getNewKeyToHandleDuplicates(key, msg)] = value
+		case etSECTION_END: //end of outer section
+			return msg, nil
+		default:
+			panic(fmt.Errorf("[readMap] protocol error 1, %d %#v", c, msg))
+			//return nil, fmt.Errorf("[readMap] protocol error 1, %d",c)
+		}
+	}
+	return
+}

vendor/github.com/bronze1man/goStrongswanVici/pools.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type Pool struct {
+	PoolMapping map[string]interface{} `json:"pools"`
+}
+
+type PoolMapping struct {
+	Addrs              string   `json:"addrs"`
+	DNS                []string `json:"dns,omitempty"`
+	NBNS               []string `json:"nbns,omitempty"`
+	ApplicationVersion []string `json:"7,omitempty"`
+	InternalIPv6Prefix []string `json:"18,omitempty"`
+}
+
+func (c *ClientConn) LoadPool(ph Pool) error {
+	requestMap := map[string]interface{}{}
+
+	err := ConvertToGeneral(ph.PoolMapping, &requestMap)
+
+	if err != nil {
+		fmt.Println(err)
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-pool", requestMap)
+
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful LoadPool: %v", msg["success"])
+	}
+
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/shared.go

+// this file contains the functions for managing shared secrets
+
+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type Key struct {
+	ID     string   `json:"id,omitempty"`
+	Typ    string   `json:"type"`
+	Data   string   `json:"data"`
+	Owners []string `json:"owners"`
+}
+
+type UnloadKeyRequest struct {
+	ID string `json:"id"`
+}
+
+type keyList struct {
+	Keys []string `json:"keys"`
+}
+
+// load a shared secret into the IKE daemon
+func (c *ClientConn) LoadShared(key *Key) error {
+	requestMap := &map[string]interface{}{}
+
+	err := ConvertToGeneral(key, requestMap)
+
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("load-shared", *requestMap)
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful loadSharedKey: %v", msg["errmsg"])
+	}
+
+	return nil
+}
+
+// unload (delete) a shared secret from the IKE daemon
+func (c *ClientConn) UnloadShared(key *UnloadKeyRequest) error {
+	requestMap := &map[string]interface{}{}
+
+	err := ConvertToGeneral(key, requestMap)
+
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	msg, err := c.Request("unload-shared", *requestMap)
+	if msg["success"] != "yes" {
+		return fmt.Errorf("unsuccessful loadSharedKey: %v", msg["errmsg"])
+	}
+
+	return nil
+}
+
+// get a the names of the shared secrets currently loaded
+func (c *ClientConn) GetShared() ([]string, error) {
+	msg, err := c.Request("get-shared", nil)
+	if err != nil {
+		fmt.Errorf("Error making request: %v", err)
+		return nil, err
+	}
+
+	keys := &keyList{}
+
+	err = ConvertFromGeneral(msg, keys)
+	if err != nil {
+		fmt.Errorf("Error converting data: %v", err)
+	}
+
+	return keys.Keys, err
+}

vendor/github.com/bronze1man/goStrongswanVici/stats.go

+package goStrongswanVici
+
+// Stats returns IKE daemon statistics and load information.
+func (c *ClientConn) Stats() (msg map[string]interface{}, err error) {
+	msg, err = c.Request("stats", nil)
+	return
+}

vendor/github.com/bronze1man/goStrongswanVici/terminate.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type TerminateRequest struct {
+	Child    string `json:"child,omitempty"`
+	Ike      string `json:"ike,omitempty"`
+	Child_id string `json:"child-id,omitempty"`
+	Ike_id   string `json:"ike-id,omitempty"`
+	Timeout  string `json:"timeout,omitempty"`
+	Loglevel string `json:"loglevel,omitempty"`
+}
+
+// To be simple, kill a client that is connecting to this server. A client is a sa.
+//Terminates an SA while streaming control-log events.
+func (c *ClientConn) Terminate(r *TerminateRequest) (err error) {
+	err = handlePanic(func() (err error) {
+		reqMap := &map[string]interface{}{}
+		ConvertToGeneral(r, reqMap)
+		msg, err := c.Request("terminate", *reqMap)
+		if err != nil {
+			return
+		}
+		if msg["success"] != "yes" {
+			return fmt.Errorf("[Terminate] %s", msg["errmsg"])
+		}
+		return
+	})
+	return
+}

vendor/github.com/bronze1man/goStrongswanVici/unloadConn.go

+package goStrongswanVici
+
+import (
+	"fmt"
+)
+
+type UnloadConnRequest struct {
+	Name string `json:"name"`
+}
+
+func (c *ClientConn) UnloadConn(r *UnloadConnRequest) error {
+	reqMap := &map[string]interface{}{}
+	ConvertToGeneral(r, reqMap)
+	msg, err := c.Request("unload-conn", *reqMap)
+	if err != nil {
+		return err
+	}
+
+	if msg["success"] != "yes" {
+		return fmt.Errorf("[Unload-Connection] %s", msg["errmsg"])
+	}
+
+	return nil
+}

vendor/github.com/bronze1man/goStrongswanVici/version.go

+package goStrongswanVici
+
+type Version struct {
+	Daemon  string `json:"daemon"`
+	Version string `json:"version"`
+	Sysname string `json:"sysname"`
+	Release string `json:"release"`
+	Machine string `json:"machine"`
+}
+
+func (c *ClientConn) Version() (out *Version, err error) {
+	msg, err := c.Request("version", nil)
+	if err != nil {
+		return
+	}
+	out = &Version{}
+	err = ConvertFromGeneral(msg, out)
+	return
+}

vendor/github.com/coreos/flannel/backend/extension/extension.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extension
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"encoding/json"
+	"os/exec"
+	"sync"
+
+	log "k8s.io/klog"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+	"golang.org/x/net/context"
+)
+
+func init() {
+	backend.Register("extension", New)
+}
+
+type ExtensionBackend struct {
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+	networks map[string]*network
+}
+
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backend, error) {
+	be := &ExtensionBackend{
+		sm:       sm,
+		extIface: extIface,
+		networks: make(map[string]*network),
+	}
+
+	return be, nil
+}
+
+func (_ *ExtensionBackend) Run(ctx context.Context) {
+	<-ctx.Done()
+}
+
+func (be *ExtensionBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
+	n := &network{
+		extIface: be.extIface,
+		sm:       be.sm,
+	}
+
+	// Parse out configuration
+	if len(config.Backend) > 0 {
+		cfg := struct {
+			PreStartupCommand   string
+			PostStartupCommand  string
+			SubnetAddCommand    string
+			SubnetRemoveCommand string
+		}{}
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
+			return nil, fmt.Errorf("error decoding backend config: %v", err)
+		}
+		n.preStartupCommand = cfg.PreStartupCommand
+		n.postStartupCommand = cfg.PostStartupCommand
+		n.subnetAddCommand = cfg.SubnetAddCommand
+		n.subnetRemoveCommand = cfg.SubnetRemoveCommand
+	}
+
+	data := []byte{}
+	if len(n.preStartupCommand) > 0 {
+		cmd_output, err := runCmd([]string{}, "", "sh", "-c", n.preStartupCommand)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run command: %s Err: %v Output: %s", n.preStartupCommand, err, cmd_output)
+		} else {
+			log.Infof("Ran command: %s\n Output: %s", n.preStartupCommand, cmd_output)
+		}
+
+		data, err = json.Marshal(cmd_output)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		log.Infof("No pre startup command configured - skipping")
+	}
+
+	attrs := subnet.LeaseAttrs{
+		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
+		BackendType: "extension",
+		BackendData: data,
+	}
+
+	lease, err := be.sm.AcquireLease(ctx, &attrs)
+	switch err {
+	case nil:
+		n.lease = lease
+
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+
+	if len(n.postStartupCommand) > 0 {
+		cmd_output, err := runCmd([]string{
+			fmt.Sprintf("NETWORK=%s", config.Network),
+			fmt.Sprintf("SUBNET=%s", lease.Subnet),
+			fmt.Sprintf("PUBLIC_IP=%s", attrs.PublicIP)},
+			"", "sh", "-c", n.postStartupCommand)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run command: %s Err: %v Output: %s", n.postStartupCommand, err, cmd_output)
+		} else {
+			log.Infof("Ran command: %s\n Output: %s", n.postStartupCommand, cmd_output)
+		}
+	} else {
+		log.Infof("No post startup command configured - skipping")
+	}
+
+	return n, nil
+}
+
+// Run a cmd, returning a combined stdout and stderr.
+func runCmd(env []string, stdin string, name string, arg ...string) (string, error) {
+	env = append(env, fmt.Sprintf("PATH=%s", os.Getenv("PATH")))
+	cmd := exec.Command(name, arg...)
+	cmd.Env = env
+
+	stdinpipe, err := cmd.StdinPipe()
+	if err != nil {
+		return "", err
+	}
+
+	io.WriteString(stdinpipe, stdin)
+	io.WriteString(stdinpipe, "\n")
+	stdinpipe.Close()
+
+	output, err := cmd.CombinedOutput()
+
+	return strings.TrimSpace(string(output)), err
+}

vendor/github.com/coreos/flannel/backend/extension/extension_network.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package extension
+
+import (
+	"encoding/json"
+	"sync"
+
+	log "k8s.io/klog"
+	"golang.org/x/net/context"
+
+	"fmt"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/subnet"
+)
+
+type network struct {
+	name                string
+	extIface            *backend.ExternalInterface
+	lease               *subnet.Lease
+	sm                  subnet.Manager
+	preStartupCommand   string
+	postStartupCommand  string
+	subnetAddCommand    string
+	subnetRemoveCommand string
+}
+
+func (n *network) Lease() *subnet.Lease {
+	return n.lease
+}
+
+func (n *network) MTU() int {
+	return n.extIface.Iface.MTU
+}
+
+func (n *network) Run(ctx context.Context) {
+	wg := sync.WaitGroup{}
+
+	log.Info("Watching for new subnet leases")
+	evts := make(chan []subnet.Event)
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.lease, evts)
+		wg.Done()
+	}()
+
+	defer wg.Wait()
+
+	for {
+		select {
+		case evtBatch := <-evts:
+			n.handleSubnetEvents(evtBatch)
+
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)
+
+			if evt.Lease.Attrs.BackendType != "extension" {
+				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if len(n.subnetAddCommand) > 0 {
+				backendData := ""
+
+				if len(evt.Lease.Attrs.BackendData) > 0 {
+					if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {
+						log.Errorf("failed to unmarshal BackendData: %v", err)
+						continue
+					}
+				}
+
+				cmd_output, err := runCmd([]string{
+					fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
+					fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
+					backendData,
+					"sh", "-c", n.subnetAddCommand)
+
+				if err != nil {
+					log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetAddCommand, err, cmd_output)
+				} else {
+					log.Infof("Ran command: %s\n Output: %s", n.subnetAddCommand, cmd_output)
+				}
+			}
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "extension" {
+				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if len(n.subnetRemoveCommand) > 0 {
+				backendData := ""
+
+				if len(evt.Lease.Attrs.BackendData) > 0 {
+					if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {
+						log.Errorf("failed to unmarshal BackendData: %v", err)
+						continue
+					}
+				}
+				cmd_output, err := runCmd([]string{
+					fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
+					fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
+					backendData,
+					"sh", "-c", n.subnetRemoveCommand)
+
+				if err != nil {
+					log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetRemoveCommand, err, cmd_output)
+				} else {
+					log.Infof("Ran command: %s\n Output: %s", n.subnetRemoveCommand, cmd_output)
+				}
+			}
+
+		default:
+			log.Error("Internal error: unknown event type: ", int(evt.Type))
+		}
+	}
+}

vendor/github.com/coreos/flannel/backend/ipsec/handle_charon.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// +build !windows
+
+package ipsec
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/bronze1man/goStrongswanVici"
+	"github.com/coreos/flannel/subnet"
+	log "k8s.io/klog"
+	"golang.org/x/net/context"
+)
+
+type Uri struct {
+	network, address string
+}
+
+type CharonIKEDaemon struct {
+	viciUri     Uri
+	espProposal string
+	ctx         context.Context
+}
+
+func NewCharonIKEDaemon(ctx context.Context, wg sync.WaitGroup, espProposal string) (*CharonIKEDaemon, error) {
+
+	charon := &CharonIKEDaemon{ctx: ctx, espProposal: espProposal}
+
+	addr := strings.Split("unix:///var/run/charon.vici", "://")
+	charon.viciUri = Uri{addr[0], addr[1]}
+
+	cmd, err := charon.runBundled("/usr/lib/strongswan/", "charon")
+
+	if err != nil {
+		log.Errorf("Error starting charon daemon: %v", err)
+		return nil, err
+	} else {
+		log.Info("Charon daemon started")
+	}
+	wg.Add(1)
+	go func() {
+		select {
+		case <-ctx.Done():
+			cmd.Process.Signal(syscall.SIGTERM)
+			cmd.Wait()
+			log.Infof("Stopped charon daemon")
+			wg.Done()
+		}
+	}()
+	return charon, nil
+}
+
+func (charon *CharonIKEDaemon) getClient(wait bool) (client *goStrongswanVici.ClientConn, err error) {
+	for {
+		socket_conn, err := net.Dial(charon.viciUri.network, charon.viciUri.address)
+		if err == nil {
+			return goStrongswanVici.NewClientConn(socket_conn), nil
+		} else {
+			if wait {
+				select {
+				case <-charon.ctx.Done():
+					log.Error("Cancel waiting for charon")
+					return nil, err
+				default:
+					log.Errorf("ClientConnection failed: %v", err)
+				}
+
+				log.Info("Retrying in a second ...")
+				time.Sleep(time.Second)
+			} else {
+				return nil, err
+			}
+		}
+	}
+}
+
+func (charon *CharonIKEDaemon) runBundled(staticLocation string, command string) (cmd *exec.Cmd, err error) {
+	path, err := exec.LookPath(command)
+	if err != nil {
+		path = staticLocation + command
+	}
+	cmd = &exec.Cmd{
+		Path: path,
+		SysProcAttr: &syscall.SysProcAttr{
+			Pdeathsig: syscall.SIGTERM,
+		},
+		Stdout: os.Stdout,
+		Stderr: os.Stderr,
+	}
+	err = cmd.Start()
+	return
+}
+
+func (charon *CharonIKEDaemon) LoadSharedKey(remotePublicIP, password string) error {
+	var err error
+	var client *goStrongswanVici.ClientConn
+
+	client, err = charon.getClient(true)
+	if err != nil {
+		log.Errorf("Failed to acquire Vici client: %v", err)
+		return err
+	}
+
+	defer client.Close()
+
+	sharedKey := &goStrongswanVici.Key{
+		Typ:    "IKE",
+		Data:   password,
+		Owners: []string{remotePublicIP},
+	}
+
+	for {
+		err = client.LoadShared(sharedKey)
+		if err != nil {
+			log.Errorf("Failed to load my key. Retrying. %v", err)
+			time.Sleep(time.Second)
+		} else {
+			break
+		}
+	}
+
+	log.Infof("Loaded shared key for: %v", remotePublicIP)
+	return nil
+}
+
+func (charon *CharonIKEDaemon) LoadConnection(localLease, remoteLease *subnet.Lease,
+	reqID, encap string) error {
+	var err error
+	var client *goStrongswanVici.ClientConn
+
+	client, err = charon.getClient(true)
+	if err != nil {
+		log.Errorf("Failed to acquire Vici client: %s", err)
+		return err
+	}
+	defer client.Close()
+
+	childConfMap := make(map[string]goStrongswanVici.ChildSAConf)
+	childSAConf := goStrongswanVici.ChildSAConf{
+		Local_ts:     []string{localLease.Subnet.String()},
+		Remote_ts:    []string{remoteLease.Subnet.String()},
+		ESPProposals: []string{charon.espProposal},
+		StartAction:  "start",
+		CloseAction:  "trap",
+		Mode:         "tunnel",
+		ReqID:        reqID,
+		//		RekeyTime:     rekeyTime,
+		InstallPolicy: "no",
+	}
+
+	childSAConfName := formatChildSAConfName(localLease, remoteLease)
+
+	childConfMap[childSAConfName] = childSAConf
+
+	localAuthConf := goStrongswanVici.AuthConf{
+		AuthMethod: "psk",
+	}
+	remoteAuthConf := goStrongswanVici.AuthConf{
+		AuthMethod: "psk",
+	}
+
+	ikeConf := goStrongswanVici.IKEConf{
+		LocalAddrs:  []string{localLease.Attrs.PublicIP.String()},
+		RemoteAddrs: []string{remoteLease.Attrs.PublicIP.String()},
+		Proposals:   []string{"aes256-sha256-modp4096"},
+		Version:     "2",
+		KeyingTries: "0", //continues to retry
+		LocalAuth:   localAuthConf,
+		RemoteAuth:  remoteAuthConf,
+		Children:    childConfMap,
+		Encap:       encap,
+	}
+	ikeConfMap := make(map[string]goStrongswanVici.IKEConf)
+
+	connectionName := formatConnectionName(localLease, remoteLease)
+	ikeConfMap[connectionName] = ikeConf
+
+	err = client.LoadConn(&ikeConfMap)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Loaded connection: %v", connectionName)
+	return nil
+}
+
+func (charon *CharonIKEDaemon) UnloadCharonConnection(localLease,
+	remoteLease *subnet.Lease) error {
+	client, err := charon.getClient(false)
+	if err != nil {
+		log.Errorf("Failed to acquire Vici client: %s", err)
+		return err
+	}
+	defer client.Close()
+
+	connectionName := formatConnectionName(localLease, remoteLease)
+	unloadConnRequest := &goStrongswanVici.UnloadConnRequest{
+		Name: connectionName,
+	}
+
+	err = client.UnloadConn(unloadConnRequest)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Unloaded connection: %v", connectionName)
+	return nil
+}
+
+func formatConnectionName(localLease, remoteLease *subnet.Lease) string {
+	return fmt.Sprintf("%s-%s-%s-%s", localLease.Attrs.PublicIP,
+		localLease.Subnet, remoteLease.Subnet, remoteLease.Attrs.PublicIP)
+}
+
+func formatChildSAConfName(localLease, remoteLease *subnet.Lease) string {
+	return fmt.Sprintf("%s-%s", localLease.Subnet, remoteLease.Subnet)
+}

vendor/github.com/coreos/flannel/backend/ipsec/handle_xfrm.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// +build !windows
+
+package ipsec
+
+import (
+	"fmt"
+	"net"
+
+	log "k8s.io/klog"
+	"github.com/vishvananda/netlink"
+
+	"github.com/coreos/flannel/subnet"
+)
+
+func AddXFRMPolicy(myLease, remoteLease *subnet.Lease, dir netlink.Dir, reqID int) error {
+	src := myLease.Subnet.ToIPNet()
+
+	dst := remoteLease.Subnet.ToIPNet()
+
+	policy := netlink.XfrmPolicy{
+		Src: src,
+		Dst: dst,
+		Dir: dir,
+	}
+
+	tunnelLeft := myLease.Attrs.PublicIP.ToIP()
+	tunnelRight := remoteLease.Attrs.PublicIP.ToIP()
+
+	tmpl := netlink.XfrmPolicyTmpl{
+		Src:   tunnelLeft,
+		Dst:   tunnelRight,
+		Proto: netlink.XFRM_PROTO_ESP,
+		Mode:  netlink.XFRM_MODE_TUNNEL,
+		Reqid: reqID,
+	}
+
+	log.Infof("Adding ipsec policy: %+v", tmpl)
+
+	policy.Tmpls = append(policy.Tmpls, tmpl)
+
+	if err := netlink.XfrmPolicyAdd(&policy); err != nil {
+		return fmt.Errorf("error adding policy: %+v err: %v", policy, err)
+	}
+
+	return nil
+}
+
+func DeleteXFRMPolicy(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, dir netlink.Dir, reqID int) error {
+	src := localSubnet
+	dst := remoteSubnet
+
+	policy := netlink.XfrmPolicy{
+		Src: src,
+		Dst: dst,
+		Dir: dir,
+	}
+
+	tunnelLeft := localPublicIP
+	tunnelRight := remotePublicIP
+
+	tmpl := netlink.XfrmPolicyTmpl{
+		Src:   tunnelLeft,
+		Dst:   tunnelRight,
+		Proto: netlink.XFRM_PROTO_ESP,
+		Mode:  netlink.XFRM_MODE_TUNNEL,
+		Reqid: reqID,
+	}
+
+	log.Infof("Deleting ipsec policy: %+v", tmpl)
+
+	policy.Tmpls = append(policy.Tmpls, tmpl)
+
+	if err := netlink.XfrmPolicyDel(&policy); err != nil {
+		return fmt.Errorf("error deleting policy: %+v err: %v", policy, err)
+	}
+
+	return nil
+}

vendor/github.com/coreos/flannel/backend/ipsec/ipsec.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// +build !windows
+
+package ipsec
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	log "k8s.io/klog"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet"
+)
+
+/*
+	Flannel's approach to IPSec uses Strongswan to handle the key exchange (using IKEv2) and the kernel to handle the
+	actual encryption.
+
+	Strongswan's "charon" is bundled in the flannel container. Flannel runs it as a child process when the ipsec backend
+	is selected and communicates with it using the "VICI" interface. Strongswan ships a utility "swanctl" which also
+	uses the VICI interface. This utility is bundled in the flannel container and can help with debugging.
+
+	The file "handle_charon.go" contains the logic for working with the charon. It supports creating a "CharonIKEDaemon"
+	which supports loading the PSK into the charon and adding and removing connections.
+
+	The file "handle_xfrm.go" contains functions for adding and removing the ipsec polcies.
+
+	ipsec_network.go ties it all together, loading the PSK for current host on startu and as new hosts are added and
+	removed it, adds/removes the PSK and connection details to strongswan and adds/remove the policy to the kernel.
+*/
+
+const (
+	defaultESPProposal = "aes128gcm16-sha256-prfsha256-ecp256"
+	minPasswordLength  = 96
+)
+
+func init() {
+	backend.Register("ipsec", New)
+}
+
+type IPSECBackend struct {
+	sm       subnet.Manager
+	extIface *backend.ExternalInterface
+}
+
+func New(sm subnet.Manager, extIface *backend.ExternalInterface) (
+	backend.Backend, error) {
+	be := &IPSECBackend{
+		sm:       sm,
+		extIface: extIface,
+	}
+
+	return be, nil
+}
+
+func (be *IPSECBackend) RegisterNetwork(
+	ctx context.Context, wg sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
+
+	cfg := struct {
+		UDPEncap    bool
+		ESPProposal string
+		PSK         string
+	}{
+		UDPEncap:    false,
+		ESPProposal: defaultESPProposal,
+	}
+
+	if len(config.Backend) > 0 {
+		if err := json.Unmarshal(config.Backend, &cfg); err != nil {
+			return nil, fmt.Errorf("error decoding IPSEC backend config: %v", err)
+		}
+	}
+
+	if len(cfg.PSK) < minPasswordLength {
+		return nil, fmt.Errorf(
+			"config error, password should be at least %d characters long",
+			minPasswordLength)
+	}
+
+	log.Infof("IPSec config: UDPEncap=%v ESPProposal=%s", cfg.UDPEncap, cfg.ESPProposal)
+
+	attrs := subnet.LeaseAttrs{
+		PublicIP:    ip.FromIP(be.extIface.ExtAddr),
+		BackendType: "ipsec",
+	}
+
+	l, err := be.sm.AcquireLease(ctx, &attrs)
+
+	switch err {
+	case nil:
+
+	case context.Canceled, context.DeadlineExceeded:
+		return nil, err
+
+	default:
+		return nil, fmt.Errorf("failed to acquire lease: %v", err)
+	}
+
+	ikeDaemon, err := NewCharonIKEDaemon(ctx, wg, cfg.ESPProposal)
+	if err != nil {
+		return nil, fmt.Errorf("error creating CharonIKEDaemon struct: %v", err)
+	}
+
+	return newNetwork(be.sm, be.extIface, cfg.UDPEncap, cfg.PSK, ikeDaemon, l)
+}

vendor/github.com/coreos/flannel/backend/ipsec/ipsec_network.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// +build !windows
+
+package ipsec
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"sync"
+
+	log "k8s.io/klog"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/net/context"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/subnet"
+)
+
+const (
+	/*
+	   New IP header (Tunnel Mode)   : 20
+	   SPI (ESP Header)              : 4
+	   Sequence (ESP Header)         : 4
+	   ESP-AES IV                    : 16
+	   ESP-AES Pad                   : 0-15
+	   Pad length (ESP Trailer)      : 1
+	   Next Header (ESP Trailer)     : 1
+	   ESP-SHA-256 ICV               : 16
+	*/
+	ipsecOverhead    = 77
+	udpEncapOverhead = 8
+
+	defaultReqID = 11
+)
+
+type network struct {
+	backend.SimpleNetwork
+	password string
+	UDPEncap bool
+	sm       subnet.Manager
+	iked     *CharonIKEDaemon
+}
+
+func newNetwork(sm subnet.Manager, extIface *backend.ExternalInterface,
+	UDPEncap bool, password string, ikeDaemon *CharonIKEDaemon,
+	l *subnet.Lease) (*network, error) {
+	n := &network{
+		SimpleNetwork: backend.SimpleNetwork{
+			SubnetLease: l,
+			ExtIface:    extIface,
+		},
+		sm:       sm,
+		iked:     ikeDaemon,
+		password: password,
+		UDPEncap: UDPEncap,
+	}
+
+	return n, nil
+}
+
+func (n *network) Run(ctx context.Context) {
+
+	err := n.iked.LoadSharedKey(n.SimpleNetwork.SubnetLease.Attrs.PublicIP.ToIP().String(), n.password)
+	if err != nil {
+		log.Errorf("Failed to load PSK: %v", err)
+		return
+	}
+
+	wg := sync.WaitGroup{}
+	defer wg.Wait()
+
+	log.Info("Watching for new subnet leases")
+
+	evts := make(chan []subnet.Event)
+
+	wg.Add(1)
+	go func() {
+		subnet.WatchLeases(ctx, n.sm, n.SubnetLease, evts)
+		log.Info("WatchLeases exited")
+		wg.Done()
+	}()
+
+	for {
+		select {
+		case evtsBatch := <-evts:
+			log.Info("Handling event")
+			n.handleSubnetEvents(evtsBatch)
+		case <-ctx.Done():
+			log.Info("Received DONE")
+			return
+		}
+	}
+}
+
+func (n *network) handleSubnetEvents(batch []subnet.Event) {
+	for _, evt := range batch {
+		switch evt.Type {
+		case subnet.EventAdded:
+			log.Info("Subnet added: ", evt.Lease.Subnet)
+
+			if evt.Lease.Attrs.BackendType != "ipsec" {
+				log.Warningf("Ignoring non-ipsec event: type: %v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {
+				log.Warningf("Ignoring own lease add event: %+v", evt.Lease)
+				continue
+			}
+
+			if err := n.AddIPSECPolicies(&evt.Lease, defaultReqID); err != nil {
+				log.Errorf("error adding ipsec policy: %v", err)
+			}
+
+			if err := n.iked.LoadSharedKey(evt.Lease.Attrs.PublicIP.String(), n.password); err != nil {
+				log.Errorf("error loading shared key into IKE daemon: %v", err)
+			}
+
+			if err := n.iked.LoadConnection(n.SubnetLease, &evt.Lease, strconv.Itoa(defaultReqID),
+				strconv.FormatBool(n.UDPEncap)); err != nil {
+				log.Errorf("error loading connection into IKE daemon: %v", err)
+			}
+
+		case subnet.EventRemoved:
+			log.Info("Subnet removed: ", evt.Lease.Subnet)
+			if evt.Lease.Attrs.BackendType != "ipsec" {
+				log.Warningf("Ignoring non-ipsec event: type: %v", evt.Lease.Attrs.BackendType)
+				continue
+			}
+
+			if evt.Lease.Subnet.Equal(n.SubnetLease.Subnet) {
+				log.Warningf("Ignoring own lease remove event: %+v", evt.Lease)
+				continue
+			}
+
+			if err := n.iked.UnloadCharonConnection(n.SubnetLease, &evt.Lease); err != nil {
+				log.Errorf("error unloading charon connections: %v", err)
+			}
+
+			if err := n.DeleteIPSECPolicies(n.SubnetLease.Subnet.ToIPNet(), evt.Lease.Subnet.ToIPNet(),
+				n.SubnetLease.Attrs.PublicIP.ToIP(), evt.Lease.Attrs.PublicIP.ToIP(), defaultReqID); err != nil {
+
+				log.Errorf("error deleting ipsec policies: %v", err)
+			}
+		}
+	}
+}
+
+func (n *network) MTU() int {
+	mtu := n.ExtIface.Iface.MTU - ipsecOverhead
+	if n.UDPEncap {
+		mtu -= udpEncapOverhead
+	}
+
+	return mtu
+}
+
+func (n *network) AddIPSECPolicies(remoteLease *subnet.Lease, reqID int) error {
+	err := AddXFRMPolicy(n.SubnetLease, remoteLease, netlink.XFRM_DIR_OUT, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec out policy: %v", err)
+	}
+
+	err = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_IN, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec in policy: %v", err)
+	}
+
+	err = AddXFRMPolicy(remoteLease, n.SubnetLease, netlink.XFRM_DIR_FWD, reqID)
+	if err != nil {
+		return fmt.Errorf("error adding ipsec fwd policy: %v", err)
+	}
+
+	return nil
+}
+
+func (n *network) DeleteIPSECPolicies(localSubnet, remoteSubnet *net.IPNet, localPublicIP, remotePublicIP net.IP, reqID int) error {
+	err := DeleteXFRMPolicy(localSubnet, remoteSubnet, localPublicIP, remotePublicIP, netlink.XFRM_DIR_OUT, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec out policy: %v", err)
+	}
+
+	err = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_IN, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec in policy: %v", err)
+	}
+
+	err = DeleteXFRMPolicy(remoteSubnet, localSubnet, remotePublicIP, localPublicIP, netlink.XFRM_DIR_FWD, reqID)
+	if err != nil {
+		return fmt.Errorf("error deleting ipsec fwd policy: %v", err)
+	}
+
+	return nil
+}

vendor/github.com/coreos/flannel/backend/ipsec/ipsec_windows.go

+// Copyright 2017 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ipsec
+
+import log "k8s.io/klog"
+
+func init() {
+	log.Infof("ipsec is not supported on this platform")
+}

vendor/github.com/coreos/flannel/backend/vxlan/device.go

@@ -35,6 +35,7 @@ type vxlanDeviceAttrs struct {
 	vtepAddr  net.IP
 	vtepPort  int
 	gbp       bool
+	learning  bool
 }
 
 type vxlanDevice struct {
@@ -51,7 +52,7 @@ func newVXLANDevice(devAttrs *vxlanDeviceAttrs) (*vxlanDevice, error) {
 		VtepDevIndex: devAttrs.vtepIndex,
 		SrcAddr:      devAttrs.vtepAddr,
 		Port:         devAttrs.vtepPort,
-		Learning:     false,
+		Learning:     devAttrs.learning,
 		GBP:          devAttrs.gbp,
 	}
 

vendor/github.com/coreos/flannel/backend/vxlan/device_windows.go

@@ -18,6 +18,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/buger/jsonparser"
 	"github.com/coreos/flannel/pkg/ip"
 	log "k8s.io/klog"
@@ -129,7 +130,30 @@ func ensureNetwork(expectedNetwork *hcsshim.HNSNetwork, expectedVSID int64, expe
 		}
 
 		log.Infof("Created HNSNetwork %s", networkName)
-		return newNetwork, nil
+		existingNetwork = newNetwork
+	}
+
+	existingNetworkV2, err := hcn.GetNetworkByID(existingNetwork.Id)
+	if err != nil {
+		return nil, errors.Annotatef(err, "Could not find vxlan0 in V2")
+	}
+
+	addHostRoute := true
+	for _, policy := range existingNetworkV2.Policies {
+		if policy.Type == hcn.HostRoute {
+			addHostRoute = false
+		}
+	}
+	if addHostRoute {
+		hostRoutePolicy := hcn.NetworkPolicy{
+			Type:     hcn.HostRoute,
+			Settings: []byte("{}"),
+		}
+
+		networkRequest := hcn.PolicyNetworkRequest{
+			Policies: []hcn.NetworkPolicy{hostRoutePolicy},
+		}
+		existingNetworkV2.AddPolicy(networkRequest)
 	}
 
 	return existingNetwork, nil

vendor/github.com/coreos/flannel/backend/vxlan/vxlan.go

@@ -107,6 +107,7 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 		VNI           int
 		Port          int
 		GBP           bool
+		Learning      bool
 		DirectRouting bool
 	}{
 		VNI: defaultVNI,
@@ -117,7 +118,7 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 			return nil, fmt.Errorf("error decoding VXLAN backend config: %v", err)
 		}
 	}
-	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.DirectRouting)
+	log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)
 
 	devAttrs := vxlanDeviceAttrs{
 		vni:       uint32(cfg.VNI),
@@ -126,6 +127,7 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 		vtepAddr:  be.extIface.IfaceAddr,
 		vtepPort:  cfg.Port,
 		gbp:       cfg.GBP,
+		learning:  cfg.Learning,
 	}
 
 	dev, err := newVXLANDevice(&devAttrs)

vendor/github.com/coreos/flannel/backend/vxlan/vxlan_windows.go

@@ -33,7 +33,6 @@ import (
 
 	"golang.org/x/net/context"
 
-	"github.com/Microsoft/hcsshim"
 	"github.com/Microsoft/hcsshim/hcn"
 	"github.com/coreos/flannel/backend"
 	"github.com/coreos/flannel/pkg/ip"
@@ -65,9 +64,13 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
 }
 
 func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
+	var hardwareAddress hardwareAddr
+	if mac != nil {
+		hardwareAddress = hardwareAddr(mac)
+	}
 	leaseAttrs := &vxlanLeaseAttrs{
 		VNI:     vnid,
-		VtepMAC: hardwareAddr(mac),
+		VtepMAC: hardwareAddress,
 	}
 	data, err := json.Marshal(&leaseAttrs)
 	if err != nil {
@@ -123,33 +126,12 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 	}
 	log.Infof("VXLAN config: Name=%s MacPrefix=%s VNI=%d Port=%d GBP=%v DirectRouting=%v", cfg.Name, cfg.MacPrefix, cfg.VNI, cfg.Port, cfg.GBP, cfg.DirectRouting)
 
-	hnsNetworks, err := hcsshim.HNSListNetworkRequest("GET", "", "")
-	if err != nil {
-		log.Infof("Cannot get HNS networks [%+v]", err)
-	}
-
-	var remoteDrMac string
-	for _, hnsnetwork := range hnsNetworks {
-		if hnsnetwork.ManagementIP == be.extIface.ExtAddr.String() {
-			hcnnetwork, err := hcn.GetNetworkByID(hnsnetwork.Id)
-			policies := hcnnetwork.Policies
-			for _, policy := range policies {
-				if policy.Type == hcn.DrMacAddress {
-					policySettings := hcn.DrMacAddressNetworkPolicySetting{}
-					err = json.Unmarshal(policy.Settings, &policySettings)
-					if err != nil {
-						return nil, fmt.Errorf("Failed to unmarshal settings")
-					}
-					remoteDrMac = policySettings.Address
-				}
-			}
-		}
-	}
-	mac, err := net.ParseMAC(string(remoteDrMac))
+	err := hcn.RemoteSubnetSupported()
 	if err != nil {
 		return nil, err
 	}
-	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), mac)
+
+	subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -176,7 +158,41 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg sync.WaitGroup,
 	dev.directRouting = cfg.DirectRouting
 	dev.macPrefix = cfg.MacPrefix
 
-	return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
+	network, err := newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
+	if err != nil {
+		return nil, err
+	}
+
+	hcnNetwork, err := hcn.GetNetworkByName(cfg.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	var newDrMac string
+	for _, policy := range hcnNetwork.Policies {
+		if policy.Type == hcn.DrMacAddress {
+			policySettings := hcn.DrMacAddressNetworkPolicySetting{}
+			err = json.Unmarshal(policy.Settings, &policySettings)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to unmarshal settings")
+			}
+			newDrMac = policySettings.Address
+		}
+	}
+
+	mac, err := net.ParseMAC(string(newDrMac))
+	if err != nil {
+		return nil, fmt.Errorf("Cannot parse DR MAC %v: %+v", newDrMac, err)
+	}
+
+	subnetAttrs, err = newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), mac)
+	if err != nil {
+		return nil, err
+	}
+
+	lease, err = be.subnetMgr.AcquireLease(ctx, subnetAttrs)
+
+	return network, nil
 }
 
 // So we can make it JSON (un)marshalable

vendor/github.com/coreos/flannel/pkg/ip/ipnet.go

@@ -77,12 +77,12 @@ func (ip IP4) StringSep(sep string) string {
 	return fmt.Sprintf("%d%s%d%s%d%s%d", a, sep, b, sep, c, sep, d)
 }
 
-// json.Marshaler impl
+// MarshalJSON: json.Marshaler impl
 func (ip IP4) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf(`"%s"`, ip)), nil
 }
 
-// json.Unmarshaler impl
+// UnmarshalJSON: json.Unmarshaler impl
 func (ip *IP4) UnmarshalJSON(j []byte) error {
 	j = bytes.Trim(j, "\"")
 	if val, err := ParseIP4(string(j)); err != nil {
@@ -163,12 +163,12 @@ func (n IP4Net) Empty() bool {
 	return n.IP == IP4(0) && n.PrefixLen == uint(0)
 }
 
-// json.Marshaler impl
+// MarshalJSON: json.Marshaler impl
 func (n IP4Net) MarshalJSON() ([]byte, error) {
 	return []byte(fmt.Sprintf(`"%s"`, n)), nil
 }
 
-// json.Unmarshaler impl
+// UnmarshalJSON: json.Unmarshaler impl
 func (n *IP4Net) UnmarshalJSON(j []byte) error {
 	j = bytes.Trim(j, "\"")
 	if _, val, err := net.ParseCIDR(string(j)); err != nil {

vendor/github.com/coreos/flannel/subnet/kube/kube.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/coreos/flannel/pkg/ip"
 	"github.com/coreos/flannel/subnet"
+	log "k8s.io/klog"
 	"golang.org/x/net/context"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,7 +39,6 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog"
 )
 
 var (
@@ -48,8 +48,6 @@ var (
 const (
 	resyncPeriod              = 5 * time.Minute
 	nodeControllerSyncTimeout = 10 * time.Minute
-
-	netConfPath = "/etc/kube-flannel/net-conf.json"
 )
 
 type kubeSubnetManager struct {
@@ -62,21 +60,16 @@ type kubeSubnetManager struct {
 	events         chan subnet.Event
 }
 
-func NewSubnetManager(apiUrl, flannelConf, kubeconfig, prefix string) (subnet.Manager, error) {
+func NewSubnetManager(apiUrl, kubeconfig, prefix, netConfPath string) (subnet.Manager, error) {
 
 	var cfg *rest.Config
 	var err error
-	// Use out of cluster config if the URL or kubeconfig have been specified. Otherwise use incluster config.
-	if apiUrl != "" || kubeconfig != "" {
-		cfg, err = clientcmd.BuildConfigFromFlags(apiUrl, kubeconfig)
-		if err != nil {
-			return nil, fmt.Errorf("unable to create k8s config: %v", err)
-		}
-	} else {
-		cfg, err = rest.InClusterConfig()
-		if err != nil {
-			return nil, fmt.Errorf("unable to initialize inclusterconfig: %v", err)
-		}
+	// Try to build kubernetes config from a master url or a kubeconfig filepath. If neither masterUrl
+	// or kubeconfigPath are passed in we fall back to inClusterConfig. If inClusterConfig fails,
+	// we fallback to the default config.
+	cfg, err = clientcmd.BuildConfigFromFlags(apiUrl, kubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("fail to create kubernetes config: %v", err)
 	}
 
 	c, err := clientset.NewForConfig(cfg)
@@ -105,10 +98,7 @@ func NewSubnetManager(apiUrl, flannelConf, kubeconfig, prefix string) (subnet.Ma
 		}
 	}
 
-	if flannelConf == "" {
-		flannelConf = netConfPath
-	}
-	netConf, err := ioutil.ReadFile(flannelConf)
+	netConf, err := ioutil.ReadFile(netConfPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read net conf: %v", err)
 	}
@@ -124,14 +114,14 @@ func NewSubnetManager(apiUrl, flannelConf, kubeconfig, prefix string) (subnet.Ma
 	}
 	go sm.Run(context.Background())
 
-	klog.Infof("Waiting %s for node controller to sync", nodeControllerSyncTimeout)
+	log.Infof("Waiting %s for node controller to sync", nodeControllerSyncTimeout)
 	err = wait.Poll(time.Second, nodeControllerSyncTimeout, func() (bool, error) {
 		return sm.nodeController.HasSynced(), nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error waiting for nodeController to sync state: %v", err)
 	}
-	klog.Infof("Node controller sync successful")
+	log.Infof("Node controller sync successful")
 
 	return sm, nil
 }
@@ -169,12 +159,12 @@ func newKubeSubnetManager(c clientset.Interface, sc *subnet.Config, nodeName, pr
 				if !isNode {
 					deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
 					if !ok {
-						klog.Infof("Error received unexpected object: %v", obj)
+						log.Infof("Error received unexpected object: %v", obj)
 						return
 					}
 					node, ok = deletedState.Obj.(*v1.Node)
 					if !ok {
-						klog.Infof("Error deletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj)
+						log.Infof("Error deletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj)
 						return
 					}
 					obj = node
@@ -197,7 +187,7 @@ func (ksm *kubeSubnetManager) handleAddLeaseEvent(et subnet.EventType, obj inter
 
 	l, err := ksm.nodeToLease(*n)
 	if err != nil {
-		klog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
+		log.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
 		return
 	}
 	ksm.events <- subnet.Event{et, l}
@@ -217,7 +207,7 @@ func (ksm *kubeSubnetManager) handleUpdateLeaseEvent(oldObj, newObj interface{})
 
 	l, err := ksm.nodeToLease(*n)
 	if err != nil {
-		klog.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
+		log.Infof("Error turning node %q to lease: %v", n.ObjectMeta.Name, err)
 		return
 	}
 	ksm.events <- subnet.Event{subnet.EventAdded, l}
@@ -254,7 +244,7 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 		n.Annotations[ksm.annotations.BackendData] = string(bd)
 		if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
 			if n.Annotations[ksm.annotations.BackendPublicIP] != n.Annotations[ksm.annotations.BackendPublicIPOverwrite] {
-				klog.Infof("Overriding public ip with '%s' from node annotation '%s'",
+				log.Infof("Overriding public ip with '%s' from node annotation '%s'",
 					n.Annotations[ksm.annotations.BackendPublicIPOverwrite],
 					ksm.annotations.BackendPublicIPOverwrite)
 				n.Annotations[ksm.annotations.BackendPublicIP] = n.Annotations[ksm.annotations.BackendPublicIPOverwrite]
@@ -284,6 +274,10 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
 			return nil, err
 		}
 	}
+	err = ksm.setNodeNetworkUnavailableFalse()
+	if err != nil {
+		log.Errorf("Unable to set NetworkUnavailable to False for %q: %v", ksm.nodeName, err)
+	}
 	return &subnet.Lease{
 		Subnet:     ip.FromIPNet(cidr),
 		Attrs:      *attrs,
@@ -303,7 +297,7 @@ func (ksm *kubeSubnetManager) WatchLeases(ctx context.Context, cursor interface{
 }
 
 func (ksm *kubeSubnetManager) Run(ctx context.Context) {
-	klog.Infof("Starting kube subnet manager")
+	log.Infof("Starting kube subnet manager")
 	ksm.nodeController.Run(ctx.Done())
 }
 
@@ -325,7 +319,7 @@ func (ksm *kubeSubnetManager) nodeToLease(n v1.Node) (l subnet.Lease, err error)
 	return l, nil
 }
 
-// unimplemented
+// RenewLease: unimplemented
 func (ksm *kubeSubnetManager) RenewLease(ctx context.Context, lease *subnet.Lease) error {
 	return ErrUnimplemented
 }
@@ -337,3 +331,23 @@ func (ksm *kubeSubnetManager) WatchLease(ctx context.Context, sn ip.IP4Net, curs
 func (ksm *kubeSubnetManager) Name() string {
 	return fmt.Sprintf("Kubernetes Subnet Manager - %s", ksm.nodeName)
 }
+
+// Set Kubernetes NodeNetworkUnavailable to false when starting
+// https://kubernetes.io/docs/concepts/architecture/nodes/#condition
+func (ksm *kubeSubnetManager) setNodeNetworkUnavailableFalse() error {
+	condition := v1.NodeCondition{
+		Type:               v1.NodeNetworkUnavailable,
+		Status:             v1.ConditionFalse,
+		Reason:             "FlannelIsUp",
+		Message:            "Flannel is running on this node",
+		LastTransitionTime: metav1.Now(),
+		LastHeartbeatTime:  metav1.Now(),
+	}
+	raw, err := json.Marshal(&[]v1.NodeCondition{condition})
+	if err != nil {
+		return err
+	}
+	patch := []byte(fmt.Sprintf(`{"status":{"conditions":%s}}`, raw))
+	_, err = ksm.client.CoreV1().Nodes().PatchStatus(ksm.nodeName, patch)
+	return err
+}

vendor/modules.txt

@@ -119,6 +119,8 @@ github.com/bhendo/go-powershell/backend
 github.com/bhendo/go-powershell/utils
 # github.com/blang/semver v3.5.0+incompatible
 github.com/blang/semver
+# github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40
+github.com/bronze1man/goStrongswanVici
 # github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23
 github.com/buger/jsonparser
 # github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5
@@ -358,8 +360,10 @@ github.com/coreos/etcd/clientv3/balancer/connectivity
 github.com/coreos/etcd/pkg/systemd
 github.com/coreos/etcd/raft
 github.com/coreos/etcd/raft/raftpb
-# github.com/coreos/flannel v0.11.0 => github.com/ibuildthecloud/flannel v0.10.1-0.20190131215433-823afe66b226
+# github.com/coreos/flannel v0.11.0 => github.com/rancher/flannel v0.11.0-k3s.1
 github.com/coreos/flannel/backend
+github.com/coreos/flannel/backend/extension
+github.com/coreos/flannel/backend/ipsec
 github.com/coreos/flannel/backend/vxlan
 github.com/coreos/flannel/network
 github.com/coreos/flannel/pkg/ip