Merge pull request #1045 from ibuildthecloud/dqlite4

go.mod

@@ -64,6 +64,7 @@ require (
 	github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e // indirect
 	github.com/bronze1man/goStrongswanVici v0.0.0-20190828090544-27d02f80ba40 // indirect
 	github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect
+	github.com/canonical/go-dqlite v1.1.0
 	github.com/containerd/cgroups v0.0.0-20190923161937-abd0b19954a6 // indirect
 	github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69
 	github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect
@@ -75,30 +76,32 @@ require (
 	github.com/containernetworking/plugins v0.8.2 // indirect
 	github.com/coreos/flannel v0.11.0
 	github.com/coreos/go-iptables v0.4.2
-	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/docker/docker v0.7.3-0.20190731001754-589f1dad8dad
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 // indirect
+	github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 // indirect
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gofrs/flock v0.7.1 // indirect
 	github.com/gogo/googleapis v1.3.0 // indirect
 	github.com/google/tcpproxy v0.0.0-20180808230851-dfa16c61dad2
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.4.0
+	github.com/gorilla/websocket v1.4.1
 	github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect
 	github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 // indirect
 	github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
 	github.com/lib/pq v1.1.1
+	github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455
 	github.com/mattn/go-sqlite3 v1.10.0
 	github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 // indirect
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	github.com/opencontainers/runc v1.0.0-rc2.0.20190611121236-6cc515888830
 	github.com/pkg/errors v0.8.1
 	github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 // indirect
-	github.com/rancher/dynamiclistener v0.1.1-0.20191108205817-245f86cc340a
+	github.com/rancher/dynamiclistener v0.1.1-0.20191110035254-aaa5bc0d2a07
 	github.com/rancher/helm-controller v0.2.2
-	github.com/rancher/kine v0.1.2-0.20191107225357-527576e3452f
+	github.com/rancher/kine v0.2.1
 	github.com/rancher/remotedialer v0.2.0
 	github.com/rancher/wrangler v0.2.0
 	github.com/rancher/wrangler-api v0.2.0
@@ -108,11 +111,12 @@ require (
 	github.com/tchap/go-patricia v2.3.0+incompatible // indirect
 	github.com/theckman/go-flock v0.7.1 // indirect
 	github.com/urfave/cli v1.21.0
-	go.etcd.io/bbolt v1.3.3 // indirect
+	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
 	golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc
 	golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3
 	google.golang.org/grpc v1.23.0
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
+	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 // indirect
 	gopkg.in/yaml.v2 v2.2.4
 	k8s.io/api v0.0.0
 	k8s.io/apimachinery v0.0.0

package/Dockerfile

@@ -9,6 +9,8 @@ RUN cd image/bin && \
 
 FROM scratch
 COPY --from=base /image /
+RUN mkdir -p /etc && \
+    echo 'hosts: files dns' > /etc/nsswitch.conf
 RUN chmod 1777 /tmp
 VOLUME /var/lib/kubelet
 VOLUME /var/lib/rancher/k3s

pkg/cli/cmds/dqlite.go

+// +build dqlite
+
+package cmds
+
+const (
+	hideDqlite = false
+)

pkg/cli/cmds/server.go

@@ -130,38 +130,6 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				EnvVar:      "K3S_TOKEN_FILE",
 			},
 			cli.StringFlag{
-				Name:        "agent-token",
-				Usage:       "(cluster) Shared secret used to join agents to the cluster, but not agents",
-				Destination: &ServerConfig.AgentToken,
-				EnvVar:      "K3S_AGENT_TOKEN",
-			},
-			cli.StringFlag{
-				Name:        "agent-token-file",
-				Usage:       "(cluster) File containing the agent secret",
-				Destination: &ServerConfig.AgentTokenFile,
-				EnvVar:      "K3S_AGENT_TOKEN_FILE",
-			},
-			cli.StringFlag{
-				Name:        "server,s",
-				Usage:       "(cluster) Server to connect to, used to join a cluster",
-				EnvVar:      "K3S_URL",
-				Destination: &ServerConfig.ServerURL,
-			},
-			cli.BoolFlag{
-				Name:        "cluster-init",
-				Hidden:      hideDqlite,
-				Usage:       "(cluster) Initialize new cluster master",
-				EnvVar:      "K3S_CLUSTER_INIT",
-				Destination: &ServerConfig.ClusterInit,
-			},
-			cli.BoolFlag{
-				Name:        "cluster-reset",
-				Hidden:      hideDqlite,
-				Usage:       "(cluster) Forget all peers and become a single cluster new cluster master",
-				EnvVar:      "K3S_CLUSTER_RESET",
-				Destination: &ServerConfig.ClusterReset,
-			},
-			cli.StringFlag{
 				Name:        "write-kubeconfig,o",
 				Usage:       "(client) Write kubeconfig for admin client to this file",
 				Destination: &ServerConfig.KubeConfigOutput,
@@ -260,6 +228,38 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command {
 				Usage:       "(experimental) Run rootless",
 				Destination: &ServerConfig.Rootless,
 			},
+			cli.StringFlag{
+				Name:        "agent-token",
+				Usage:       "(experimental/cluster) Shared secret used to join agents to the cluster, but not agents",
+				Destination: &ServerConfig.AgentToken,
+				EnvVar:      "K3S_AGENT_TOKEN",
+			},
+			cli.StringFlag{
+				Name:        "agent-token-file",
+				Usage:       "(experimental/cluster) File containing the agent secret",
+				Destination: &ServerConfig.AgentTokenFile,
+				EnvVar:      "K3S_AGENT_TOKEN_FILE",
+			},
+			cli.StringFlag{
+				Name:        "server,s",
+				Usage:       "(experimental/cluster) Server to connect to, used to join a cluster",
+				EnvVar:      "K3S_URL",
+				Destination: &ServerConfig.ServerURL,
+			},
+			cli.BoolFlag{
+				Name:        "cluster-init",
+				Hidden:      hideDqlite,
+				Usage:       "(experimental/cluster) Initialize new cluster master",
+				EnvVar:      "K3S_CLUSTER_INIT",
+				Destination: &ServerConfig.ClusterInit,
+			},
+			cli.BoolFlag{
+				Name:        "cluster-reset",
+				Hidden:      hideDqlite,
+				Usage:       "(experimental/cluster) Forget all peers and become a single cluster new cluster master",
+				EnvVar:      "K3S_CLUSTER_RESET",
+				Destination: &ServerConfig.ClusterReset,
+			},
 
 			// Hidden/Deprecated flags below
 

pkg/cluster/cluster.go

@@ -2,42 +2,73 @@ package cluster
 
 import (
 	"context"
+	"strings"
 
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/kine/pkg/client"
+	"github.com/rancher/kine/pkg/endpoint"
 )
 
 type Cluster struct {
-	token            string
 	clientAccessInfo *clientaccess.Info
 	config           *config.Control
 	runtime          *config.ControlRuntime
 	db               interface{}
+	runJoin          bool
+	storageStarted   bool
+	etcdConfig       endpoint.ETCDConfig
+	joining          bool
+	saveBootstrap    bool
+	storageClient    client.Client
 }
 
 func (c *Cluster) Start(ctx context.Context) error {
-	join, err := c.shouldJoin()
-	if err != nil {
+	if err := c.startClusterAndHTTPS(ctx); err != nil {
 		return err
 	}
 
-	if join {
-		if err := c.join(); err != nil {
+	if c.runJoin {
+		if err := c.postJoin(ctx); err != nil {
 			return err
 		}
 	}
 
-	if err := c.startClusterAndHTTPS(ctx); err != nil {
+	if err := c.testClusterDB(ctx); err != nil {
 		return err
 	}
 
-	if join {
-		if err := c.postJoin(ctx); err != nil {
+	if c.saveBootstrap {
+		if err := c.save(ctx); err != nil {
+			return err
+		}
+	}
+
+	if c.runJoin {
+		if err := c.joined(); err != nil {
 			return err
 		}
 	}
 
-	return c.joined()
+	return c.startStorage(ctx)
+}
+
+func (c *Cluster) startStorage(ctx context.Context) error {
+	if c.storageStarted {
+		return nil
+	}
+	c.storageStarted = true
+
+	etcdConfig, err := endpoint.Listen(ctx, c.config.Storage)
+	if err != nil {
+		return err
+	}
+
+	c.etcdConfig = etcdConfig
+	c.config.Storage.Config = etcdConfig.TLSConfig
+	c.config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
+	c.config.NoLeaderElect = !etcdConfig.LeaderElect
+	return nil
 }
 
 func New(config *config.Control) *Cluster {

pkg/cluster/dqlite.go

+// +build dqlite
+
+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/rancher/dynamiclistener/factory"
+	"github.com/rancher/k3s/pkg/clientaccess"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/dqlite"
+	"github.com/rancher/kine/pkg/endpoint"
+	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+)
+
+func (c *Cluster) testClusterDB(ctx context.Context) error {
+	if !c.dqliteEnabled() {
+		return nil
+	}
+
+	dqlite := c.db.(*dqlite.DQLite)
+	for {
+		if err := dqlite.Test(ctx); err != nil {
+			logrus.Infof("Failed to test dqlite connection: %v", err)
+		} else {
+			return nil
+		}
+
+		select {
+		case <-time.After(2 * time.Second):
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}
+
+func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
+	if !c.dqliteEnabled() {
+		return l, handler, nil
+	}
+
+	dqlite := dqlite.New(c.config.DataDir, c.config.AdvertiseIP, c.config.AdvertisePort, func() v1.NodeController {
+		if c.runtime.Core == nil {
+			return nil
+		}
+		return c.runtime.Core.Core().V1().Node()
+	})
+
+	certs, err := toGetCerts(c.runtime)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	handler, err = dqlite.Start(ctx, c.config.ClusterInit, c.config.ClusterReset, certs, handler)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if c.config.ClusterReset {
+		if err := dqlite.Reset(ctx); err == nil {
+			logrus.Info("Cluster reset successful, now rejoin members")
+			os.Exit(0)
+		} else {
+			logrus.Fatalf("Cluster reset failed: %v", err)
+		}
+	}
+
+	c.db = dqlite
+	if !strings.HasPrefix(c.config.Storage.Endpoint, "dqlite://") {
+		c.config.Storage = endpoint.Config{
+			Endpoint: dqlite.StorageEndpoint,
+		}
+	}
+
+	return l, handler, err
+}
+
+func (c *Cluster) dqliteEnabled() bool {
+	stamp := filepath.Join(c.config.DataDir, "db", "state.dqlite")
+	if _, err := os.Stat(stamp); err == nil {
+		return true
+	}
+
+	driver, _ := endpoint.ParseStorageEndpoint(c.config.Storage.Endpoint)
+	if driver == endpoint.DQLiteBackend {
+		return true
+	}
+
+	return c.config.Storage.Endpoint == "" && (c.config.ClusterInit || (c.config.Token != "" && c.config.JoinURL != ""))
+}
+
+func (c *Cluster) postJoin(ctx context.Context) error {
+	if !c.dqliteEnabled() {
+		return nil
+	}
+
+	resp, err := clientaccess.Get("/db/info", c.clientAccessInfo)
+	if err != nil {
+		return err
+	}
+
+	dqlite := c.db.(*dqlite.DQLite)
+	var nodes []client.NodeInfo
+
+	if err := json.Unmarshal(resp, &nodes); err != nil {
+		return err
+	}
+
+	return dqlite.Join(ctx, nodes)
+}
+
+func toGetCerts(runtime *config.ControlRuntime) (*dqlite.Certs, error) {
+	clientCA, _, err := factory.LoadCerts(runtime.ClientCA, runtime.ClientCAKey)
+	if err != nil {
+		return nil, err
+	}
+
+	ca, _, err := factory.LoadCerts(runtime.ServerCA, runtime.ServerCAKey)
+	if err != nil {
+		return nil, err
+	}
+
+	clientCert, err := tls.LoadX509KeyPair(runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return &dqlite.Certs{
+		ServerTrust: ca,
+		ClientTrust: clientCA,
+		ClientCert:  clientCert,
+	}, nil
+}

pkg/cluster/encrypt.go

+package cluster
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/rancher/k3s/pkg/token"
+	"golang.org/x/crypto/pbkdf2"
+)
+
+func storageKey(passphrase string) string {
+	d := sha256.New()
+	d.Write([]byte(passphrase))
+	return "/bootstrap/" + hex.EncodeToString(d.Sum(nil)[:])[:12]
+}
+
+func keyHash(passphrase string) string {
+	d := sha256.New()
+	d.Write([]byte(passphrase))
+	return hex.EncodeToString(d.Sum(nil)[:])[:12]
+}
+
+func encrypt(passphrase string, plaintext []byte) ([]byte, error) {
+	salt, err := token.Random(8)
+	if err != nil {
+		return nil, err
+	}
+
+	clearKey := pbkdf2.Key([]byte(passphrase), []byte(salt), 4096, 32, sha1.New)
+	key, err := aes.NewCipher(clearKey)
+	if err != nil {
+		return nil, err
+	}
+
+	gcm, err := cipher.NewGCM(key)
+	if err != nil {
+		return nil, err
+	}
+
+	nonce := make([]byte, gcm.NonceSize())
+	_, err = io.ReadFull(rand.Reader, nonce)
+	if err != nil {
+		return nil, err
+	}
+
+	sealed := gcm.Seal(nonce, nonce, plaintext, nil)
+	return []byte(salt + ":" + base64.StdEncoding.EncodeToString(sealed)), nil
+}
+
+func decrypt(passphrase string, ciphertext []byte) ([]byte, error) {
+	parts := strings.SplitN(string(ciphertext), ":", 2)
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("invalid cipher text, not : delimited")
+	}
+
+	clearKey := pbkdf2.Key([]byte(passphrase), []byte(parts[0]), 4096, 32, sha1.New)
+	key, err := aes.NewCipher(clearKey)
+	if err != nil {
+		return nil, err
+	}
+
+	gcm, err := cipher.NewGCM(key)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := base64.StdEncoding.DecodeString(parts[1])
+	if err != nil {
+		return nil, err
+	}
+
+	return gcm.Open(nil, data[:gcm.NonceSize()], data[gcm.NonceSize():], nil)
+}

pkg/cluster/join.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -11,18 +12,38 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+func (c *Cluster) Join(ctx context.Context) error {
+	runJoin, err := c.shouldJoin()
+	if err != nil {
+		return err
+	}
+	c.runJoin = runJoin
+
+	if runJoin {
+		if err := c.join(ctx); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func (c *Cluster) shouldJoin() (bool, error) {
-	if c.config.JoinURL == "" {
-		return false, nil
+	dqlite := c.dqliteEnabled()
+	if dqlite {
+		c.runtime.HTTPBootstrap = true
+		if c.config.JoinURL == "" {
+			return false, nil
+		}
 	}
 
-	stamp := filepath.Join(c.config.DataDir, "db/joined")
+	stamp := c.joinStamp()
 	if _, err := os.Stat(stamp); err == nil {
-		logrus.Info("Already joined to cluster, not rejoining")
+		logrus.Info("Cluster bootstrap already complete")
 		return false, nil
 	}
 
-	if c.config.Token == "" {
+	if dqlite && c.config.Token == "" {
 		return false, fmt.Errorf("K3S_TOKEN is required to join a cluster")
 	}
 
@@ -46,14 +67,11 @@ func (c *Cluster) joined() error {
 	return f.Close()
 }
 
-func (c *Cluster) join() error {
-	c.runtime.Cluster.Join = true
-
+func (c *Cluster) httpJoin() error {
 	token, err := clientaccess.NormalizeAndValidateTokenForUser(c.config.JoinURL, c.config.Token, "server")
 	if err != nil {
 		return err
 	}
-	c.token = token
 
 	info, err := clientaccess.ParseAndValidateToken(c.config.JoinURL, token)
 	if err != nil {
@@ -69,6 +87,20 @@ func (c *Cluster) join() error {
 	return bootstrap.Read(bytes.NewBuffer(content), &c.runtime.ControlRuntimeBootstrap)
 }
 
+func (c *Cluster) join(ctx context.Context) error {
+	c.joining = true
+
+	if c.runtime.HTTPBootstrap {
+		return c.httpJoin()
+	}
+
+	if err := c.storageJoin(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (c *Cluster) joinStamp() string {
-	return filepath.Join(c.config.DataDir, "db/joined")
+	return filepath.Join(c.config.DataDir, "db/joined-"+keyHash(c.config.Token))
 }

pkg/cluster/nocluster.go

@@ -8,6 +8,10 @@ import (
 	"net/http"
 )
 
+func (c *Cluster) testClusterDB(ctx context.Context) error {
+	return nil
+}
+
 func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
 	return l, handler, nil
 }
@@ -15,3 +19,7 @@ func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler htt
 func (c *Cluster) postJoin(ctx context.Context) error {
 	return nil
 }
+
+func (c *Cluster) dqliteEnabled() bool {
+	return false
+}

pkg/cluster/storage.go

+package cluster
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/rancher/k3s/pkg/bootstrap"
+	"github.com/rancher/kine/pkg/client"
+)
+
+func (c *Cluster) save(ctx context.Context) error {
+	buf := &bytes.Buffer{}
+	if err := bootstrap.Write(buf, &c.runtime.ControlRuntimeBootstrap); err != nil {
+		return err
+	}
+
+	data, err := encrypt(c.config.Token, buf.Bytes())
+	if err != nil {
+		return err
+	}
+
+	return c.storageClient.Create(ctx, storageKey(c.config.Token), data)
+}
+
+func (c *Cluster) storageJoin(ctx context.Context) error {
+	if err := c.startStorage(ctx); err != nil {
+		return err
+	}
+
+	storageClient, err := client.New(c.etcdConfig)
+	if err != nil {
+		return err
+	}
+	c.storageClient = storageClient
+
+	value, err := storageClient.Get(ctx, storageKey(c.config.Token))
+	if err == client.ErrNotFound {
+		c.saveBootstrap = true
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	data, err := decrypt(c.config.Token, value.Data)
+	if err != nil {
+		return err
+	}
+
+	return bootstrap.Read(bytes.NewBuffer(data), &c.runtime.ControlRuntimeBootstrap)
+}

pkg/daemons/config/types.go

@@ -132,6 +132,8 @@ type ControlRuntimeBootstrap struct {
 type ControlRuntime struct {
 	ControlRuntimeBootstrap
 
+	HTTPBootstrap bool
+
 	ClientKubeAPICert string
 	ClientKubeAPIKey  string
 	NodePasswdFile    string
@@ -169,12 +171,7 @@ type ControlRuntime struct {
 	ClientK3sControllerCert   string
 	ClientK3sControllerKey    string
 
-	Cluster ClusterConfig
-	Core    *core.Factory
-}
-
-type ClusterConfig struct {
-	Join bool
+	Core *core.Factory
 }
 
 type ArgString []string

pkg/daemons/control/server.go

@@ -26,14 +26,14 @@ import (
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/passwd"
 	"github.com/rancher/k3s/pkg/token"
-	"github.com/rancher/kine/pkg/endpoint"
 	"github.com/rancher/wrangler-api/pkg/generated/controllers/rbac"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 	ccmapp "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
+	app2 "k8s.io/kubernetes/cmd/controller-manager/app"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
 	sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
@@ -316,11 +316,13 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 	runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt")
 	runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key")
 
-	if err := genCerts(config, runtime); err != nil {
+	cluster := cluster.New(config)
+
+	if err := cluster.Join(ctx); err != nil {
 		return err
 	}
 
-	if err := cluster.New(config).Start(ctx); err != nil {
+	if err := genCerts(config, runtime); err != nil {
 		return err
 	}
 
@@ -336,23 +338,11 @@ func prepare(ctx context.Context, config *config.Control, runtime *config.Contro
 		return err
 	}
 
-	if err := prepareStorageBackend(ctx, config); err != nil {
-		return err
-	}
-
-	return readTokens(runtime)
-}
-
-func prepareStorageBackend(ctx context.Context, config *config.Control) error {
-	etcdConfig, err := endpoint.Listen(ctx, config.Storage)
-	if err != nil {
+	if err := readTokens(runtime); err != nil {
 		return err
 	}
 
-	config.Storage.Config = etcdConfig.TLSConfig
-	config.Storage.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
-	config.NoLeaderElect = !etcdConfig.LeaderElect
-	return nil
+	return cluster.Start(ctx)
 }
 
 func readTokens(runtime *config.ControlRuntime) error {
@@ -858,25 +848,24 @@ func waitForAPIServer(ctx context.Context, runtime *config.ControlRuntime) error
 		return err
 	}
 
-	discoveryclient, err := discovery.NewDiscoveryClientForConfig(restConfig)
+	k8sClient, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
 		return err
 	}
 
-	for i := 0; i < 60; i++ {
-		info, err := discoveryclient.ServerVersion()
-		if err == nil {
-			logrus.Infof("apiserver %s is up and running", info)
-			return nil
-		}
-		logrus.Infof("waiting for apiserver to become available")
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-time.After(time.Second):
-			continue
-		}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-promise(func() error { return app2.WaitForAPIServer(k8sClient, 5*time.Minute) }):
+		return err
 	}
+}
 
-	return fmt.Errorf("timeout waiting for apiserver")
+func promise(f func() error) <-chan error {
+	c := make(chan error, 1)
+	go func() {
+		c <- f()
+		close(c)
+	}()
+	return c
 }

pkg/dqlite/controller/client/controller.go

+package client
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+
+	"github.com/canonical/go-dqlite/client"
+	controllerv1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+const (
+	allKey      = "_all_"
+	nodeID      = "cluster.k3s.cattle.io/node-id"
+	nodeAddress = "cluster.k3s.cattle.io/node-address"
+	master      = "node-role.kubernetes.io/master"
+)
+
+func Register(ctx context.Context, nodeName string, nodeInfo client.NodeInfo,
+	nodeStore client.NodeStore, nodes controllerv1.NodeController, opts []client.Option) {
+	h := &handler{
+		nodeStore:      nodeStore,
+		nodeController: nodes,
+		nodeName:       nodeName,
+		id:             strconv.FormatUint(nodeInfo.ID, 10),
+		address:        nodeInfo.Address,
+		ctx:            ctx,
+		opts:           opts,
+	}
+	nodes.OnChange(ctx, "dqlite-client", h.sync)
+	nodes.OnRemove(ctx, "dqlite-client", h.onRemove)
+}
+
+type handler struct {
+	nodeStore      client.NodeStore
+	nodeController controllerv1.NodeController
+	nodeName       string
+	id             string
+	address        string
+	ctx            context.Context
+	opts           []client.Option
+}
+
+func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) {
+	if key == allKey {
+		return nil, h.updateNodeStore()
+	}
+
+	if node == nil {
+		return nil, nil
+	}
+
+	if key == h.nodeName {
+		return h.handleSelf(node)
+	}
+
+	if node.Labels[master] == "true" {
+		h.nodeController.Enqueue(allKey)
+	}
+
+	return node, nil
+}
+
+func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
+	if node.Annotations[nodeID] == h.id && node.Annotations[nodeAddress] == h.address {
+		return node, nil
+	}
+
+	node = node.DeepCopy()
+	if node.Annotations == nil {
+		node.Annotations = map[string]string{}
+	}
+	node.Annotations[nodeID] = h.id
+	node.Annotations[nodeAddress] = h.address
+
+	return h.nodeController.Update(node)
+}
+
+func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
+	address := node.Annotations[nodeAddress]
+	if address == "" {
+		return node, nil
+	}
+	return node, h.delete(address)
+}
+
+func (h *handler) delete(address string) error {
+	c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...)
+	if err != nil {
+		return err
+	}
+	defer c.Close()
+
+	members, err := c.Cluster(h.ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, member := range members {
+		if member.Address == address {
+			logrus.Infof("Removing %s %d from dqlite", member.Address, member.ID)
+			return c.Remove(h.ctx, member.ID)
+		}
+	}
+
+	return nil
+}
+
+func (h *handler) updateNodeStore() error {
+	nodes, err := h.nodeController.Cache().List(labels.SelectorFromSet(labels.Set{
+		master: "true",
+	}))
+	if err != nil {
+		return err
+	}
+
+	var (
+		nodeInfos []client.NodeInfo
+		seen      = map[string]bool{}
+	)
+
+	for _, node := range nodes {
+		address, ok := node.Annotations[nodeAddress]
+		if !ok {
+			continue
+		}
+
+		nodeIDStr, ok := node.Annotations[nodeID]
+		if !ok {
+			continue
+		}
+
+		id, err := strconv.ParseUint(nodeIDStr, 10, 64)
+		if err != nil {
+			logrus.Errorf("invalid %s=%s, must be a number: %v", nodeID, nodeIDStr, err)
+			continue
+		}
+
+		if !seen[address] {
+			nodeInfos = append(nodeInfos, client.NodeInfo{
+				ID:      id,
+				Address: address,
+			})
+			seen[address] = true
+		}
+	}
+
+	if len(nodeInfos) == 0 {
+		return fmt.Errorf("not setting dqlient NodeStore len to 0")
+	}
+
+	return h.nodeStore.Set(h.ctx, nodeInfos)
+}

pkg/dqlite/dialer/dial.go

+package dialer
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/rancher/k3s/pkg/dqlite/pipe"
+)
+
+func NewHTTPDialer(advertiseAddress, bindAddress string, tls *tls.Config) (client.DialFunc, error) {
+	d := &dialer{
+		advertiseAddress: advertiseAddress,
+		bindAddress:      bindAddress,
+		tls:              tls,
+	}
+
+	return d.dial, nil
+}
+
+type dialer struct {
+	advertiseAddress string
+	bindAddress      string
+	tls              *tls.Config
+}
+
+func (d *dialer) dial(ctx context.Context, address string) (net.Conn, error) {
+	if address == d.advertiseAddress {
+		return net.Dial("unix", d.bindAddress)
+	}
+
+	url := fmt.Sprintf("https://%s/db/connect", address)
+	return pipe.ToHTTP(ctx, url, d.tls)
+}

pkg/dqlite/join.go

+package dqlite
+
+import (
+	"context"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/sirupsen/logrus"
+)
+
+func (d *DQLite) Test(ctx context.Context) error {
+	var ips []string
+	peers, err := d.NodeStore.Get(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, peer := range peers {
+		ips = append(ips, peer.Address)
+	}
+
+	logrus.Infof("Testing connection to peers %v", ips)
+	return d.Join(ctx, nil)
+}
+
+func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error {
+	if len(nodes) > 0 {
+		if err := d.NodeStore.Set(ctx, nodes); err != nil {
+			return err
+		}
+	}
+
+	client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	current, err := client.Cluster(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, testNode := range current {
+		if testNode.Address == d.NodeInfo.Address {
+			return nil
+		}
+	}
+
+	logrus.Infof("Joining dqlite cluster as address=%s, id=%d")
+	return client.Add(ctx, d.NodeInfo)
+}

pkg/dqlite/log.go

+package dqlite
+
+import (
+	"strings"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/sirupsen/logrus"
+)
+
+func log() client.LogFunc {
+	return func(level client.LogLevel, s string, i ...interface{}) {
+		switch level {
+		case client.LogDebug:
+			logrus.Debugf(s, i...)
+		case client.LogError:
+			logrus.Errorf(s, i...)
+		case client.LogInfo:
+			if strings.HasPrefix(s, "connected") {
+				logrus.Debugf(s, i...)
+			} else {
+				logrus.Infof(s, i...)
+			}
+		case client.LogWarn:
+			logrus.Warnf(s, i...)
+		}
+	}
+}

pkg/dqlite/pipe/http.go

+package pipe
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+)
+
+func ToHTTP(ctx context.Context, url string, tlsConfig *tls.Config) (net.Conn, error) {
+	request, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	request = request.WithContext(ctx)
+	netDial := &net.Dialer{}
+
+	if deadline, ok := ctx.Deadline(); ok {
+		netDial.Deadline = deadline
+	}
+
+	conn, err := tls.DialWithDialer(netDial, "tcp", request.URL.Host, tlsConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "tls dial")
+	}
+
+	err = request.Write(conn)
+	if err != nil {
+		return nil, errors.Wrap(err, "request write")
+	}
+
+	response, err := http.ReadResponse(bufio.NewReader(conn), request)
+	if err != nil {
+		return nil, errors.Wrap(err, "read request")
+	}
+	if response.StatusCode != http.StatusSwitchingProtocols {
+		return nil, fmt.Errorf("expected 101 response, got: %d", response.StatusCode)
+	}
+
+	listener, err := net.Listen("unix", "")
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to create unix listener")
+	}
+	defer listener.Close()
+
+	if err := Unix(conn, listener.Addr().String()); err != nil {
+		return nil, err
+	}
+
+	return listener.Accept()
+}

pkg/dqlite/pipe/pipe.go

+package pipe
+
+import (
+	"io"
+	"net"
+
+	"github.com/lxc/lxd/shared/eagain"
+	"github.com/sirupsen/logrus"
+)
+
+func UnixPiper(srcs <-chan net.Conn, bindAddress string) {
+	for src := range srcs {
+		go Unix(src, bindAddress)
+	}
+}
+
+func Unix(src net.Conn, target string) error {
+	dst, err := net.Dial("unix", target)
+	if err != nil {
+		src.Close()
+		return err
+	}
+
+	Connect(src, dst)
+	return nil
+}
+
+func Connect(src net.Conn, dst net.Conn) {
+	go func() {
+		_, err := io.Copy(eagain.Writer{Writer: dst}, eagain.Reader{Reader: src})
+		if err != nil && err != io.EOF {
+			logrus.Debugf("copy pipe src->dst closed: %v", err)
+		}
+		src.Close()
+		dst.Close()
+	}()
+
+	go func() {
+		_, err := io.Copy(eagain.Writer{Writer: src}, eagain.Reader{Reader: dst})
+		if err != nil {
+			logrus.Debugf("copy pipe dst->src closed: %v", err)
+		}
+		src.Close()
+		dst.Close()
+	}()
+}

pkg/dqlite/proxy.go

+package dqlite
+
+import (
+	"context"
+	"net"
+	"net/http"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/dqlite/pipe"
+)
+
+var (
+	upgradeResponse = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: dqlite\r\n\r\n")
+)
+
+type proxy struct {
+	conns chan net.Conn
+}
+
+func newProxy(ctx context.Context, bindAddress string) http.Handler {
+	p := &proxy{
+		conns: make(chan net.Conn, 100),
+	}
+	go func() {
+		<-ctx.Done()
+		close(p.conns)
+	}()
+	go pipe.UnixPiper(p.conns, bindAddress)
+
+	return p
+}
+
+func (h *proxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	hijacker, ok := rw.(http.Hijacker)
+	if !ok {
+		http.Error(rw, "failed to hijack", http.StatusInternalServerError)
+		return
+	}
+
+	conn, _, err := hijacker.Hijack()
+	if err != nil {
+		err := errors.Wrap(err, "Hijack connection")
+		http.Error(rw, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if n, err := conn.Write(upgradeResponse); err != nil || n != len(upgradeResponse) {
+		conn.Close()
+		return
+	}
+
+	h.conns <- conn
+}

pkg/dqlite/reset.go

+package dqlite
+
+import (
+	"context"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/sirupsen/logrus"
+)
+
+func (d *DQLite) Reset(ctx context.Context) error {
+	logrus.Infof("Resetting cluster to single master")
+	return d.node.Recover([]client.NodeInfo{
+		d.NodeInfo,
+	})
+}

pkg/dqlite/router.go

+package dqlite
+
+import (
+	"context"
+	"crypto/x509"
+	"encoding/json"
+	"net/http"
+
+	"github.com/canonical/go-dqlite"
+	"github.com/canonical/go-dqlite/client"
+	"github.com/gorilla/mux"
+)
+
+func router(ctx context.Context, next http.Handler, nodeInfo dqlite.NodeInfo, clientCA *x509.Certificate, clientCN string, bindAddress string) http.Handler {
+	mux := mux.NewRouter()
+	mux.Handle("/db/connect", newChecker(newProxy(ctx, bindAddress), clientCA, clientCN))
+	mux.Handle("/db/info", infoHandler(ctx, nodeInfo, bindAddress))
+	mux.NotFoundHandler = next
+	return mux
+}
+
+func infoHandler(ctx context.Context, nodeInfo dqlite.NodeInfo, bindAddress string) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		client, err := client.New(ctx, bindAddress, client.WithLogFunc(log()))
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		defer client.Close()
+
+		info, err := client.Cluster(ctx)
+		if err != nil {
+			http.Error(rw, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		rw.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(rw).Encode(info)
+	})
+}
+
+type checker struct {
+	next   http.Handler
+	verify x509.VerifyOptions
+	cn     string
+}
+
+func newChecker(next http.Handler, ca *x509.Certificate, cn string) http.Handler {
+	pool := x509.NewCertPool()
+	pool.AddCert(ca)
+	return &checker{
+		next: next,
+		verify: x509.VerifyOptions{
+			Roots: pool,
+			KeyUsages: []x509.ExtKeyUsage{
+				x509.ExtKeyUsageClientAuth,
+			},
+			DNSName: cn,
+		},
+		cn: cn,
+	}
+}
+
+func (c *checker) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	if !c.check(req) {
+		http.Error(rw, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	c.next.ServeHTTP(rw, req)
+}
+
+func (c *checker) check(r *http.Request) bool {
+	for _, cert := range r.TLS.PeerCertificates {
+		_, err := cert.Verify(c.verify)
+		if err == nil {
+			return cert.Subject.CommonName == c.cn
+		}
+	}
+	return false
+}

pkg/dqlite/server.go

+package dqlite
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/canonical/go-dqlite"
+	"github.com/canonical/go-dqlite/client"
+	"github.com/pkg/errors"
+	controllerclient "github.com/rancher/k3s/pkg/dqlite/controller/client"
+	"github.com/rancher/k3s/pkg/dqlite/dialer"
+	dqlitedriver "github.com/rancher/kine/pkg/drivers/dqlite"
+	v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/net"
+)
+
+type Certs struct {
+	ServerTrust *x509.Certificate
+	ClientTrust *x509.Certificate
+	ClientCert  tls.Certificate
+}
+
+type DQLite struct {
+	ClientCA             string
+	ClientCAKey          string
+	ClientCert           string
+	ClientCertKey        string
+	ServerCA             string
+	ServerCAKey          string
+	AdvertiseIP          string
+	AdvertisePort        int
+	DataDir              string
+	NodeStore            client.NodeStore
+	NodeInfo             client.NodeInfo
+	node                 *dqlite.Node
+	StorageEndpoint      string
+	NodeControllerGetter NodeControllerGetter
+	clientOpts           []client.Option
+}
+
+type NodeControllerGetter func() v1.NodeController
+
+func New(dataDir, advertiseIP string, advertisePort int, getter NodeControllerGetter) *DQLite {
+	return &DQLite{
+		AdvertiseIP:          advertiseIP,
+		AdvertisePort:        advertisePort,
+		DataDir:              dataDir,
+		NodeControllerGetter: getter,
+	}
+}
+
+func (d *DQLite) Start(ctx context.Context, initCluster, resetCluster bool, certs *Certs, next http.Handler) (http.Handler, error) {
+	bindAddress := d.getBindAddress()
+
+	clientTLSConfig, err := getClientTLSConfig(certs.ClientCert, certs.ServerTrust)
+	if err != nil {
+		return nil, err
+	}
+
+	advertise, err := getAdvertiseAddress(d.AdvertiseIP, d.AdvertisePort)
+	if err != nil {
+		return nil, errors.Wrap(err, "get advertise address")
+	}
+
+	dial, err := getDialer(advertise, bindAddress, clientTLSConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	dqlitedriver.Dialer = dial
+	dqlitedriver.Logger = log()
+
+	d.clientOpts = append(d.clientOpts, client.WithDialFunc(dial), client.WithLogFunc(log()))
+
+	nodeInfo, node, err := getNode(d.DataDir, advertise, bindAddress, initCluster, dial)
+	if err != nil {
+		return nil, err
+	}
+
+	d.NodeInfo = nodeInfo
+	d.node = node
+
+	go func() {
+		<-ctx.Done()
+		node.Close()
+	}()
+
+	if err := d.nodeStore(ctx, initCluster); err != nil {
+		return nil, err
+	}
+
+	go d.startController(ctx)
+
+	if !resetCluster {
+		if err := node.Start(); err != nil {
+			return nil, err
+		}
+	}
+
+	return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), nil
+}
+
+func (d *DQLite) startController(ctx context.Context) {
+	for {
+		if nc := d.NodeControllerGetter(); nc != nil {
+			if os.Getenv("NODE_NAME") == "" {
+				logrus.Errorf("--disable-agent is not compatible with dqlite")
+			} else {
+				break
+			}
+		}
+		time.Sleep(time.Second)
+	}
+
+	controllerclient.Register(ctx, os.Getenv("NODE_NAME"), d.NodeInfo, d.NodeStore, d.NodeControllerGetter(), d.clientOpts)
+}
+
+func (d *DQLite) nodeStore(ctx context.Context, initCluster bool) error {
+	peerDB := filepath.Join(d.DataDir, "db", "state.dqlite", "peers.db")
+	ns, err := client.DefaultNodeStore(peerDB)
+	if err != nil {
+		return err
+	}
+	d.NodeStore = ns
+	d.StorageEndpoint = fmt.Sprintf("dqlite://?peer-file=%s", peerDB)
+	if initCluster {
+		if err := dqlitedriver.AddPeers(ctx, d.NodeStore, d.NodeInfo); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func getAdvertiseAddress(advertiseIP string, advertisePort int) (string, error) {
+	ip := advertiseIP
+	if ip == "" {
+		ipAddr, err := net.ChooseHostInterface()
+		if err != nil {
+			return "", err
+		}
+		ip = ipAddr.String()
+	}
+
+	return fmt.Sprintf("%s:%d", ip, advertisePort), nil
+}
+
+func getClientTLSConfig(cert tls.Certificate, ca *x509.Certificate) (*tls.Config, error) {
+	tlsConfig := &tls.Config{
+		RootCAs: x509.NewCertPool(),
+		Certificates: []tls.Certificate{
+			cert,
+		},
+		ServerName: "kubernetes",
+	}
+	tlsConfig.RootCAs.AddCert(ca)
+
+	return tlsConfig, nil
+}
+
+func getDialer(advertiseAddress, bindAddress string, tlsConfig *tls.Config) (client.DialFunc, error) {
+	return dialer.NewHTTPDialer(advertiseAddress, bindAddress, tlsConfig)
+}
+
+func getNode(dataDir string, advertiseAddress, bindAddress string, initCluster bool, dial client.DialFunc) (dqlite.NodeInfo, *dqlite.Node, error) {
+	id, err := getClusterID(initCluster, dataDir)
+	if err != nil {
+		return dqlite.NodeInfo{}, nil, errors.Wrap(err, "reading cluster id")
+	}
+
+	dbDir := filepath.Join(dataDir, "db", "state.dqlite")
+
+	node, err := dqlite.New(id, advertiseAddress, dbDir,
+		dqlite.WithBindAddress(bindAddress),
+		dqlite.WithDialFunc(dial),
+		dqlite.WithNetworkLatency(20*time.Millisecond))
+	return dqlite.NodeInfo{
+		ID:      id,
+		Address: advertiseAddress,
+	}, node, err
+}
+
+func getClusterID(initCluster bool, dataDir string) (uint64, error) {
+	idFile := filepath.Join(dataDir, "db/state.dqlite/node-id")
+	content, err := ioutil.ReadFile(idFile)
+	if os.IsNotExist(err) {
+		content = nil
+	} else if err != nil {
+		return 0, err
+	}
+
+	idStr := strings.TrimSpace(string(content))
+	if idStr == "" {
+		if err := os.MkdirAll(filepath.Dir(idFile), 0700); err != nil {
+			return 0, err
+		}
+		id := rand.Uint64()
+		if initCluster {
+			id = 1
+		}
+		return id, ioutil.WriteFile(idFile, []byte(strconv.FormatUint(id, 10)), 0644)
+	}
+
+	return strconv.ParseUint(idStr, 10, 64)
+}
+
+func (d *DQLite) getBindAddress() string {
+	// only anonymous works???
+	return "@" + filepath.Join(d.DataDir, "db", "state.dqlite", "dqlite.sock")
+}

pkg/server/router.go

@@ -49,7 +49,10 @@ func router(serverConfig *config.Control, tunnel http.Handler, ca []byte) http.H
 	serverAuthed := mux.NewRouter()
 	serverAuthed.Use(authMiddleware(serverConfig, "k3s:server"))
 	serverAuthed.NotFoundHandler = nodeAuthed
-	serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap))
+	serverAuthed.Path("/db/info").Handler(nodeAuthed)
+	if serverConfig.Runtime.HTTPBootstrap {
+		serverAuthed.Path("/v1-k3s/server-bootstrap").Handler(bootstrap.Handler(&serverConfig.Runtime.ControlRuntimeBootstrap))
+	}
 
 	staticDir := filepath.Join(serverConfig.DataDir, "static")
 	router := mux.NewRouter()

scripts/build

@@ -48,7 +48,6 @@ rm -f \
     bin/k3s-agent \
     bin/hyperkube \
     bin/containerd \
-    bin/cni \
     bin/runc \
     bin/containerd-shim \
     bin/containerd-shim-runc-v1 \
@@ -65,6 +64,7 @@ cleanup() {
 }
 
 INSTALLBIN=$(pwd)/bin
+if [ ! -x ${INSTALLBIN}/cni ]; then
 (
     echo Building cni
     TMPDIR=$(mktemp -d)
@@ -74,6 +74,7 @@ INSTALLBIN=$(pwd)/bin
     cd $WORKDIR
     GOPATH=$TMPDIR CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
 )
+fi
 # echo Building agent
 # CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
 echo Building server
@@ -83,8 +84,8 @@ ln -s containerd ./bin/k3s-server
 ln -s containerd ./bin/kubectl
 ln -s containerd ./bin/crictl
 ln -s containerd ./bin/ctr
-echo Building hyperkube
-CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/
+#echo Building hyperkube
+#CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/
 #echo Building ctr
 #CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go
 # echo Building containerd

scripts/package-cli

@@ -34,7 +34,6 @@ elif [ ${ARCH} = arm ]; then
     BIN_SUFFIX="-armhf"
 fi
 
-cp -f ./bin/hyperkube dist/artifacts/hyperkube${BIN_SUFFIX}
 CMD_NAME=dist/artifacts/k3s${BIN_SUFFIX}
 
 go generate

scripts/sonobuoy

@@ -212,7 +212,7 @@ echo "Started ${K3S_AGENT}"
 # ---
 
 timeout --foreground 1m bash -c 'wait-for-nodes 2'
-timeout --foreground 1m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'
+timeout --foreground 3m bash -c 'wait-for-services coredns local-path-provisioner metrics-server'
 
 if [ "$ARCH" = 'arm' ]; then
     echo "Aborting sonobuoy tests, images not available for $ARCH"

vendor/github.com/Rican7/retry/.travis.yml

+language: go
+
+go:
+    - 1.6
+    - tip
+
+sudo: false
+
+install:
+    # Get all imported packages
+    - make install-deps install-deps-dev
+
+    # Basic build errors
+    - make build
+
+script:
+    # Lint
+    - make format-lint
+    - make import-lint
+    - make copyright-lint
+
+    # Run tests
+    - make test
+
+matrix:
+    allow_failures:
+        - go: tip
+    fast_finish: true

vendor/github.com/Rican7/retry/LICENSE

+Copyright (C) 2016 Trevor N. Suarez (Rican7)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/Rican7/retry/Makefile

+# Define some VCS context
+PARENT_BRANCH ?= master
+
+# Set a default `min_confidence` value for `golint`
+GOLINT_MIN_CONFIDENCE ?= 0.3
+
+# Set flags for `gofmt`
+GOFMT_FLAGS ?= -s
+
+
+all: install-deps build install
+
+clean:
+	go clean -i -x ./...
+
+build:
+	go build -v ./...
+
+install:
+	go install ./...
+
+install-deps:
+	go get -d -t ./...
+
+install-deps-dev: install-deps
+	go get github.com/golang/lint/golint
+	go get golang.org/x/tools/cmd/goimports
+
+update-deps:
+	go get -d -t -u ./...
+
+update-deps-dev: update-deps
+	go get -u github.com/golang/lint/golint
+	go get -u golang.org/x/tools/cmd/goimports
+
+test:
+	go test -v ./...
+
+test-with-coverage:
+	go test -cover ./...
+
+test-with-coverage-formatted:
+	go test -cover ./... | column -t | sort -r
+
+format-lint:
+	errors=$$(gofmt -l ${GOFMT_FLAGS} .); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
+
+import-lint:
+	errors=$$(goimports -l .); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
+
+style-lint:
+	errors=$$(golint -min_confidence=${GOLINT_MIN_CONFIDENCE} ./...); if [ "$${errors}" != "" ]; then echo "$${errors}"; exit 1; fi
+
+copyright-lint:
+	@old_dates=$$(git diff --diff-filter=ACMRTUXB --name-only "${PARENT_BRANCH}" | xargs grep -E '[Cc]opyright(\s+)[©Cc]?(\s+)[0-9]{4}' | grep -E -v "[Cc]opyright(\s+)[©Cc]?(\s+)$$(date '+%Y')"); if [ "$${old_dates}" != "" ]; then printf "The following files contain outdated copyrights:\n$${old_dates}\n\nThis can be fixed with 'make copyright-fix'\n"; exit 1; fi
+
+lint: install-deps-dev format-lint import-lint style-lint copyright-lint
+
+format-fix:
+	gofmt -w ${GOFMT_FLAGS} .
+
+import-fix:
+	goimports -w .
+
+copyright-fix:
+	@git diff --diff-filter=ACMRTUXB --name-only "${PARENT_BRANCH}" | xargs -I '_FILENAME' -- sh -c 'sed -i.bak "s/\([Cc]opyright\([[:space:]][©Cc]\{0,1\}[[:space:]]*\)\)[0-9]\{4\}/\1"$$(date '+%Y')"/g" _FILENAME && rm _FILENAME.bak'
+
+vet:
+	go vet ./...
+
+
+.PHONY: all clean build install install-deps install-deps-dev update-deps update-deps-dev test test-with-coverage test-with-coverage-formatted format-lint import-lint style-lint copyright-lint lint format-fix import-fix copyright-fix vet

vendor/github.com/Rican7/retry/README.md

+# retry
+
+[![Build Status](https://travis-ci.org/Rican7/retry.svg?branch=master)](https://travis-ci.org/Rican7/retry)
+[![GoDoc](https://godoc.org/github.com/Rican7/retry?status.png)](https://godoc.org/github.com/Rican7/retry)
+[![Go Report Card](https://goreportcard.com/badge/Rican7/retry)](http://goreportcard.com/report/Rican7/retry)
+[![Latest Stable Version](https://img.shields.io/github/release/Rican7/retry.svg?style=flat)](https://github.com/Rican7/retry/releases)
+
+A simple, stateless, functional mechanism to perform actions repetitively until successful.
+
+
+## Project Status
+
+This project is currently in "pre-release". While the code is heavily tested, the API may change.
+Vendor (commit or lock) this dependency if you plan on using it.
+
+
+## Install
+
+`go get github.com/Rican7/retry`
+
+
+## Examples
+
+### Basic
+
+```go
+Retry(func(attempt uint) error {
+	return nil // Do something that may or may not cause an error
+})
+```
+
+### File Open
+
+```go
+const logFilePath = "/var/log/myapp.log"
+
+var logFile *os.File
+
+err := Retry(func(attempt uint) error {
+	var err error
+
+	logFile, err = os.Open(logFilePath)
+
+	return err
+})
+
+if nil != err {
+	log.Fatalf("Unable to open file %q with error %q", logFilePath, err)
+}
+```
+
+### HTTP request with strategies and backoff
+
+```go
+var response *http.Response
+
+action := func(attempt uint) error {
+	var err error
+
+	response, err = http.Get("https://api.github.com/repos/Rican7/retry")
+
+	if nil == err && nil != response && response.StatusCode > 200 {
+		err = fmt.Errorf("failed to fetch (attempt #%d) with status code: %d", attempt, response.StatusCode)
+	}
+
+	return err
+}
+
+err := Retry(
+	action,
+	strategy.Limit(5),
+	strategy.Backoff(backoff.Fibonacci(10*time.Millisecond)),
+)
+
+if nil != err {
+	log.Fatalf("Failed to fetch repository with error %q", err)
+}
+```
+
+### Retry with backoff jitter
+
+```go
+action := func(attempt uint) error {
+	return errors.New("something happened")
+}
+
+seed := time.Now().UnixNano()
+random := rand.New(rand.NewSource(seed))
+
+Retry(
+	action,
+	strategy.Limit(5),
+	strategy.BackoffWithJitter(
+		backoff.BinaryExponential(10*time.Millisecond),
+		jitter.Deviation(random, 0.5),
+	),
+)
+```

vendor/github.com/Rican7/retry/backoff/backoff.go

+// Package backoff provides stateless methods of calculating durations based on
+// a number of attempts made.
+//
+// Copyright © 2016 Trevor N. Suarez (Rican7)
+package backoff
+
+import (
+	"math"
+	"time"
+)
+
+// Algorithm defines a function that calculates a time.Duration based on
+// the given retry attempt number.
+type Algorithm func(attempt uint) time.Duration
+
+// Incremental creates a Algorithm that increments the initial duration
+// by the given increment for each attempt.
+func Incremental(initial, increment time.Duration) Algorithm {
+	return func(attempt uint) time.Duration {
+		return initial + (increment * time.Duration(attempt))
+	}
+}
+
+// Linear creates a Algorithm that linearly multiplies the factor
+// duration by the attempt number for each attempt.
+func Linear(factor time.Duration) Algorithm {
+	return func(attempt uint) time.Duration {
+		return (factor * time.Duration(attempt))
+	}
+}
+
+// Exponential creates a Algorithm that multiplies the factor duration by
+// an exponentially increasing factor for each attempt, where the factor is
+// calculated as the given base raised to the attempt number.
+func Exponential(factor time.Duration, base float64) Algorithm {
+	return func(attempt uint) time.Duration {
+		return (factor * time.Duration(math.Pow(base, float64(attempt))))
+	}
+}
+
+// BinaryExponential creates a Algorithm that multiplies the factor
+// duration by an exponentially increasing factor for each attempt, where the
+// factor is calculated as `2` raised to the attempt number (2^attempt).
+func BinaryExponential(factor time.Duration) Algorithm {
+	return Exponential(factor, 2)
+}
+
+// Fibonacci creates a Algorithm that multiplies the factor duration by
+// an increasing factor for each attempt, where the factor is the Nth number in
+// the Fibonacci sequence.
+func Fibonacci(factor time.Duration) Algorithm {
+	return func(attempt uint) time.Duration {
+		return (factor * time.Duration(fibonacciNumber(attempt)))
+	}
+}
+
+// fibonacciNumber calculates the Fibonacci sequence number for the given
+// sequence position.
+func fibonacciNumber(n uint) uint {
+	if 0 == n {
+		return 0
+	} else if 1 == n {
+		return 1
+	} else {
+		return fibonacciNumber(n-1) + fibonacciNumber(n-2)
+	}
+}

vendor/github.com/Rican7/retry/jitter/jitter.go

+// Package jitter provides methods of transforming durations.
+//
+// Copyright © 2016 Trevor N. Suarez (Rican7)
+package jitter
+
+import (
+	"math"
+	"math/rand"
+	"time"
+)
+
+// Transformation defines a function that calculates a time.Duration based on
+// the given duration.
+type Transformation func(duration time.Duration) time.Duration
+
+// Full creates a Transformation that transforms a duration into a result
+// duration in [0, n) randomly, where n is the given duration.
+//
+// The given generator is what is used to determine the random transformation.
+// If a nil generator is passed, a default one will be provided.
+//
+// Inspired by https://www.awsarchitectureblog.com/2015/03/backoff.html
+func Full(generator *rand.Rand) Transformation {
+	random := fallbackNewRandom(generator)
+
+	return func(duration time.Duration) time.Duration {
+		return time.Duration(random.Int63n(int64(duration)))
+	}
+}
+
+// Equal creates a Transformation that transforms a duration into a result
+// duration in [n/2, n) randomly, where n is the given duration.
+//
+// The given generator is what is used to determine the random transformation.
+// If a nil generator is passed, a default one will be provided.
+//
+// Inspired by https://www.awsarchitectureblog.com/2015/03/backoff.html
+func Equal(generator *rand.Rand) Transformation {
+	random := fallbackNewRandom(generator)
+
+	return func(duration time.Duration) time.Duration {
+		return (duration / 2) + time.Duration(random.Int63n(int64(duration))/2)
+	}
+}
+
+// Deviation creates a Transformation that transforms a duration into a result
+// duration that deviates from the input randomly by a given factor.
+//
+// The given generator is what is used to determine the random transformation.
+// If a nil generator is passed, a default one will be provided.
+//
+// Inspired by https://developers.google.com/api-client-library/java/google-http-java-client/backoff
+func Deviation(generator *rand.Rand, factor float64) Transformation {
+	random := fallbackNewRandom(generator)
+
+	return func(duration time.Duration) time.Duration {
+		min := int64(math.Floor(float64(duration) * (1 - factor)))
+		max := int64(math.Ceil(float64(duration) * (1 + factor)))
+
+		return time.Duration(random.Int63n(max-min) + min)
+	}
+}
+
+// NormalDistribution creates a Transformation that transforms a duration into a
+// result duration based on a normal distribution of the input and the given
+// standard deviation.
+//
+// The given generator is what is used to determine the random transformation.
+// If a nil generator is passed, a default one will be provided.
+func NormalDistribution(generator *rand.Rand, standardDeviation float64) Transformation {
+	random := fallbackNewRandom(generator)
+
+	return func(duration time.Duration) time.Duration {
+		return time.Duration(random.NormFloat64()*standardDeviation + float64(duration))
+	}
+}
+
+// fallbackNewRandom returns the passed in random instance if it's not nil,
+// and otherwise returns a new random instance seeded with the current time.
+func fallbackNewRandom(random *rand.Rand) *rand.Rand {
+	// Return the passed in value if it's already not null
+	if nil != random {
+		return random
+	}
+
+	seed := time.Now().UnixNano()
+
+	return rand.New(rand.NewSource(seed))
+}

vendor/github.com/Rican7/retry/retry.go

+// Package retry provides a simple, stateless, functional mechanism to perform
+// actions repetitively until successful.
+//
+// Copyright © 2016 Trevor N. Suarez (Rican7)
+package retry
+
+import "github.com/Rican7/retry/strategy"
+
+// Action defines a callable function that package retry can handle.
+type Action func(attempt uint) error
+
+// Retry takes an action and performs it, repetitively, until successful.
+//
+// Optionally, strategies may be passed that assess whether or not an attempt
+// should be made.
+func Retry(action Action, strategies ...strategy.Strategy) error {
+	var err error
+
+	for attempt := uint(0); (0 == attempt || nil != err) && shouldAttempt(attempt, strategies...); attempt++ {
+		err = action(attempt)
+	}
+
+	return err
+}
+
+// shouldAttempt evaluates the provided strategies with the given attempt to
+// determine if the Retry loop should make another attempt.
+func shouldAttempt(attempt uint, strategies ...strategy.Strategy) bool {
+	shouldAttempt := true
+
+	for i := 0; shouldAttempt && i < len(strategies); i++ {
+		shouldAttempt = shouldAttempt && strategies[i](attempt)
+	}
+
+	return shouldAttempt
+}

vendor/github.com/Rican7/retry/strategy/strategy.go

+// Package strategy provides a way to change the way that retry is performed.
+//
+// Copyright © 2016 Trevor N. Suarez (Rican7)
+package strategy
+
+import (
+	"time"
+
+	"github.com/Rican7/retry/backoff"
+	"github.com/Rican7/retry/jitter"
+)
+
+// Strategy defines a function that Retry calls before every successive attempt
+// to determine whether it should make the next attempt or not. Returning `true`
+// allows for the next attempt to be made. Returning `false` halts the retrying
+// process and returns the last error returned by the called Action.
+//
+// The strategy will be passed an "attempt" number on each successive retry
+// iteration, starting with a `0` value before the first attempt is actually
+// made. This allows for a pre-action delay, etc.
+type Strategy func(attempt uint) bool
+
+// Limit creates a Strategy that limits the number of attempts that Retry will
+// make.
+func Limit(attemptLimit uint) Strategy {
+	return func(attempt uint) bool {
+		return (attempt <= attemptLimit)
+	}
+}
+
+// Delay creates a Strategy that waits the given duration before the first
+// attempt is made.
+func Delay(duration time.Duration) Strategy {
+	return func(attempt uint) bool {
+		if 0 == attempt {
+			time.Sleep(duration)
+		}
+
+		return true
+	}
+}
+
+// Wait creates a Strategy that waits the given durations for each attempt after
+// the first. If the number of attempts is greater than the number of durations
+// provided, then the strategy uses the last duration provided.
+func Wait(durations ...time.Duration) Strategy {
+	return func(attempt uint) bool {
+		if 0 < attempt && 0 < len(durations) {
+			durationIndex := int(attempt - 1)
+
+			if len(durations) <= durationIndex {
+				durationIndex = len(durations) - 1
+			}
+
+			time.Sleep(durations[durationIndex])
+		}
+
+		return true
+	}
+}
+
+// Backoff creates a Strategy that waits before each attempt, with a duration as
+// defined by the given backoff.Algorithm.
+func Backoff(algorithm backoff.Algorithm) Strategy {
+	return BackoffWithJitter(algorithm, noJitter())
+}
+
+// BackoffWithJitter creates a Strategy that waits before each attempt, with a
+// duration as defined by the given backoff.Algorithm and jitter.Transformation.
+func BackoffWithJitter(algorithm backoff.Algorithm, transformation jitter.Transformation) Strategy {
+	return func(attempt uint) bool {
+		if 0 < attempt {
+			time.Sleep(transformation(algorithm(attempt)))
+		}
+
+		return true
+	}
+}
+
+// noJitter creates a jitter.Transformation that simply returns the input.
+func noJitter() jitter.Transformation {
+	return func(duration time.Duration) time.Duration {
+		return duration
+	}
+}

vendor/github.com/canonical/go-dqlite/.dir-locals.el

+;;; Directory Local Variables
+;;; For more information see (info "(emacs) Directory Variables")
+((go-mode
+  . ((go-test-args . "-tags libsqlite3 -timeout 10s")
+     (eval
+      . (set
+	 (make-local-variable 'flycheck-go-build-tags)
+	 '("libsqlite3"))))))

vendor/github.com/canonical/go-dqlite/.gitignore

+.sqlite
+demo
+profile.coverprofile
+overalls.coverprofile

vendor/github.com/canonical/go-dqlite/.travis.yml

+dist: xenial
+language: go
+
+addons:
+  apt:
+    sources:
+    - sourceline: 'ppa:dqlite/master'
+    packages:
+    - golint
+    - libsqlite3-dev
+    - libuv1-dev
+    - libraft-dev
+    - libco-dev
+    - libdqlite-dev
+
+before_install:
+  - go get github.com/go-playground/overalls
+  - go get github.com/mattn/goveralls
+  - go get github.com/tsenart/deadcode
+
+script:
+  - go get -t -tags libsqlite3 ./...
+  - go vet -tags libsqlite3 ./...
+  - golint
+  - deadcode
+  - project=github.com/canonical/go-dqlite
+  - $GOPATH/bin/overalls -project $project -covermode=count -- -tags libsqlite3 -timeout 240s
+  - $GOPATH/bin/goveralls -coverprofile overalls.coverprofile -service=travis-ci
+
+go:
+  - "1.12"

vendor/github.com/canonical/go-dqlite/AUTHORS

+Free Ekanayaka <free.ekanayaka@canonical.com>

vendor/github.com/canonical/go-dqlite/README.md

+go-dqlite [![Build Status](https://travis-ci.org/canonical/go-dqlite.png)](https://travis-ci.org/canonical/go-dqlite) [![Coverage Status](https://coveralls.io/repos/github/canonical/go-dqlite/badge.svg?branch=master)](https://coveralls.io/github/canonical/go-dqlite?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/canonical/go-dqlite)](https://goreportcard.com/report/github.com/canonical/go-dqlite) [![GoDoc](https://godoc.org/github.com/canonical/go-dqlite?status.svg)](https://godoc.org/github.com/canonical/go-dqlite)
+======
+
+This repository provides the `go-dqlite` Go package, containing bindings for the
+[dqlite](https://github.com/canonical/dqlite) C library and a pure-Go
+client for the dqlite wire [protocol](https://github.com/canonical/dqlite/blob/master/doc/protocol.md).
+
+Usage
+-----
+
+The best way to understand how to use the ```go-dqlite``` package is probably by
+looking at the source code of the [demo
+program](https://github.com/canonical/go-dqlite/tree/master/cmd/dqlite-demo) and
+use it as example.
+
+Build
+-----
+
+In order to use the go-dqlite package in your application, you'll need to have
+the [dqlite](https://github.com/canonical/dqlite) C library installed on your
+system, along with its dependencies. You then need to pass the ```-tags```
+argument to the Go tools when building or testing your packages, for example:
+
+```bash
+go build -tags libsqlite3
+go test -tags libsqlite3
+```
+
+Documentation
+-------------
+
+The documentation for this package can be found on [Godoc](http://godoc.org/github.com/canonical/go-dqlite).
+
+Demo
+----
+
+To see dqlite in action, either install the Debian package from the PPA:
+
+```bash
+sudo add-apt-repository -y ppa:dqlite/v1
+sudo apt install dqlite
+```
+
+or build the dqlite C library and its dependencies from source, as described
+[here](https://github.com/canonical/dqlite#build), and then run:
+
+```
+go install -tags libsqlite3 ./cmd/dqlite-demo
+```
+
+from the top-level directory of this repository.
+
+Once the ```dqlite-demo``` binary is installed, start three nodes of the demo
+application, respectively with IDs ```1```, ```2,``` and ```3```:
+
+```bash
+dqlite-demo start 1 &
+dqlite-demo start 2 &
+dqlite-demo start 3 &
+```
+
+The node with ID ```1``` automatically becomes the leader of a single node
+cluster, while the nodes with IDs ```2``` and ```3``` are waiting to be notified
+what cluster they belong to. Let's make nodes ```2``` and ```3``` join the
+cluster:
+
+```bash
+dqlite-demo add 2
+dqlite-demo add 3
+```
+
+Now we can start using the cluster. The demo application is just a simple
+key/value store that stores data in a SQLite table. Let's insert a key pair:
+
+```bash
+dqlite-demo update my-key my-value
+```
+
+and then retrive it from the database:
+
+```bash
+dqlite-demo query my-key
+```
+
+Currently node ```1``` is the leader. If we stop it and then try to query the
+key again we'll notice that the ```query``` command hangs for a bit waiting for
+the failover to occur and for another node to step up as leader:
+
+```
+kill -TERM %1; sleep 0.1; dqlite-demo query my-key
+```

vendor/github.com/canonical/go-dqlite/client/client.go

+package client
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"net"
+	"strings"
+
+	"github.com/canonical/go-dqlite/internal/protocol"
+	"github.com/pkg/errors"
+)
+
+// DialFunc is a function that can be used to establish a network connection.
+type DialFunc = protocol.DialFunc
+
+// Client speaks the dqlite wire protocol.
+type Client struct {
+	protocol *protocol.Protocol
+}
+
+// Option that can be used to tweak client parameters.
+type Option func(*options)
+
+type options struct {
+	DialFunc DialFunc
+	LogFunc  LogFunc
+}
+
+// WithDialFunc sets a custom dial function for creating the client network
+// connection.
+func WithDialFunc(dial DialFunc) Option {
+	return func(options *options) {
+		options.DialFunc = dial
+	}
+}
+
+// WithLogFunc sets a custom log function.
+// connection.
+func WithLogFunc(log LogFunc) Option {
+	return func(options *options) {
+		options.LogFunc = log
+	}
+}
+
+// New creates a new client connected to the dqlite node with the given
+// address.
+func New(ctx context.Context, address string, options ...Option) (*Client, error) {
+	o := defaultOptions()
+
+	for _, option := range options {
+		option(o)
+	}
+	// Establish the connection.
+	conn, err := o.DialFunc(ctx, address)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to establish network connection")
+	}
+
+	// Latest protocol version.
+	proto := make([]byte, 8)
+	binary.LittleEndian.PutUint64(proto, protocol.VersionOne)
+
+	// Perform the protocol handshake.
+	n, err := conn.Write(proto)
+	if err != nil {
+		conn.Close()
+		return nil, errors.Wrap(err, "failed to send handshake")
+	}
+	if n != 8 {
+		conn.Close()
+		return nil, errors.Wrap(io.ErrShortWrite, "failed to send handshake")
+	}
+
+	client := &Client{protocol: protocol.NewProtocol(protocol.VersionOne, conn)}
+
+	return client, nil
+}
+
+// Leader returns information about the current leader, if any.
+func (c *Client) Leader(ctx context.Context) (*NodeInfo, error) {
+	request := protocol.Message{}
+	request.Init(16)
+	response := protocol.Message{}
+	response.Init(512)
+
+	protocol.EncodeLeader(&request)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return nil, errors.Wrap(err, "failed to send Leader request")
+	}
+
+	id, address, err := protocol.DecodeNode(&response)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse Node response")
+	}
+
+	info := &NodeInfo{ID: id, Address: address}
+
+	return info, nil
+}
+
+// Cluster returns information about all nodes in the cluster.
+func (c *Client) Cluster(ctx context.Context) ([]NodeInfo, error) {
+	request := protocol.Message{}
+	request.Init(16)
+	response := protocol.Message{}
+	response.Init(512)
+
+	protocol.EncodeCluster(&request)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return nil, errors.Wrap(err, "failed to send Cluster request")
+	}
+
+	servers, err := protocol.DecodeNodes(&response)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse Node response")
+	}
+
+	return servers, nil
+}
+
+// File holds the content of a single database file.
+type File struct {
+	Name string
+	Data []byte
+}
+
+// Dump the content of the database with the given name. Two files will be
+// returned, the first is the main database file (which has the same name as
+// the database), the second is the WAL file (which has the same name as the
+// database plus the suffix "-wal").
+func (c *Client) Dump(ctx context.Context, dbname string) ([]File, error) {
+	request := protocol.Message{}
+	request.Init(16)
+	response := protocol.Message{}
+	response.Init(512)
+
+	protocol.EncodeDump(&request, dbname)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return nil, errors.Wrap(err, "failed to send dump request")
+	}
+
+	files, err := protocol.DecodeFiles(&response)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse files response")
+	}
+	defer files.Close()
+
+	dump := make([]File, 0)
+
+	for {
+		name, data := files.Next()
+		if name == "" {
+			break
+		}
+		dump = append(dump, File{Name: name, Data: data})
+	}
+
+	return dump, nil
+}
+
+// Add a node to a cluster.
+func (c *Client) Add(ctx context.Context, node NodeInfo) error {
+	request := protocol.Message{}
+	request.Init(4096)
+	response := protocol.Message{}
+	response.Init(4096)
+
+	protocol.EncodeJoin(&request, node.ID, node.Address)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return err
+	}
+
+	protocol.EncodePromote(&request, node.ID)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Remove a node from the cluster.
+func (c *Client) Remove(ctx context.Context, id uint64) error {
+	request := protocol.Message{}
+	request.Init(4096)
+	response := protocol.Message{}
+	response.Init(4096)
+
+	protocol.EncodeRemove(&request, id)
+
+	if err := c.protocol.Call(ctx, &request, &response); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Close the client.
+func (c *Client) Close() error {
+	return c.protocol.Close()
+}
+
+// Create a client options object with sane defaults.
+func defaultOptions() *options {
+	return &options{
+		DialFunc: DefaultDialFunc,
+		LogFunc:  DefaultLogFunc,
+	}
+}
+
+func DefaultDialFunc(ctx context.Context, address string) (net.Conn, error) {
+	if strings.HasPrefix(address, "@") {
+		return protocol.UnixDial(ctx, address)
+	}
+	return protocol.TCPDial(ctx, address)
+}

vendor/github.com/canonical/go-dqlite/client/leader.go

+package client
+
+import (
+	"context"
+	"time"
+
+	"github.com/Rican7/retry/backoff"
+	"github.com/Rican7/retry/strategy"
+	"github.com/canonical/go-dqlite/internal/protocol"
+)
+
+// FindLeader returns a Client connected to the current cluster leader, if any.
+func FindLeader(ctx context.Context, store NodeStore, options ...Option) (*Client, error) {
+	o := defaultOptions()
+
+	for _, option := range options {
+		option(o)
+	}
+
+	config := protocol.Config{
+		Dial:           o.DialFunc,
+		AttemptTimeout: time.Second,
+		RetryStrategies: []strategy.Strategy{
+			strategy.Backoff(backoff.BinaryExponential(time.Millisecond))},
+	}
+	connector := protocol.NewConnector(0, store, config, o.LogFunc)
+	protocol, err := connector.Connect(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &Client{protocol: protocol}
+
+	return client, nil
+}

vendor/github.com/canonical/go-dqlite/client/log.go

+package client
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/canonical/go-dqlite/internal/logging"
+)
+
+// LogFunc is a function that can be used for logging.
+type LogFunc = logging.Func
+
+// LogLevel defines the logging level.
+type LogLevel = logging.Level
+
+// Available logging levels.
+const (
+	LogDebug = logging.Debug
+	LogInfo  = logging.Info
+	LogWarn  = logging.Warn
+	LogError = logging.Error
+)
+
+// DefaultLogFunc emits messages using the stdlib's logger.
+func DefaultLogFunc(l LogLevel, format string, a ...interface{}) {
+	logger := log.New(os.Stdout, "", log.LstdFlags|log.Lmicroseconds)
+	format = fmt.Sprintf("[%s]: %s", l.String(), format)
+	logger.Printf(format, a...)
+}

vendor/github.com/canonical/go-dqlite/client/store.go

+package client
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/canonical/go-dqlite/internal/protocol"
+	_ "github.com/mattn/go-sqlite3" // Go SQLite bindings
+)
+
+// NodeStore is used by a dqlite client to get an initial list of candidate
+// dqlite nodes that it can dial in order to find a leader dqlite node to use.
+type NodeStore = protocol.NodeStore
+
+// NodeInfo holds information about a single server.
+type NodeInfo = protocol.NodeInfo
+
+// InmemNodeStore keeps the list of target dqlite nodes in memory.
+type InmemNodeStore = protocol.InmemNodeStore
+
+// NewInmemNodeStore creates NodeStore which stores its data in-memory.
+var NewInmemNodeStore = protocol.NewInmemNodeStore
+
+// DatabaseNodeStore persists a list addresses of dqlite nodes in a SQL table.
+type DatabaseNodeStore struct {
+	db     *sql.DB // Database handle to use.
+	schema string  // Name of the schema holding the servers table.
+	table  string  // Name of the servers table.
+	column string  // Column name in the servers table holding the server address.
+}
+
+// DefaultNodeStore creates a new NodeStore using the given filename to
+// open a SQLite database, with default names for the schema, table and column
+// parameters.
+//
+// It also creates the table if it doesn't exist yet.
+func DefaultNodeStore(filename string) (*DatabaseNodeStore, error) {
+	// Open the database.
+	db, err := sql.Open("sqlite3", filename)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open database")
+	}
+
+	// Since we're setting SQLite single-thread mode, we need to have one
+	// connection at most.
+	db.SetMaxOpenConns(1)
+
+	// Create the servers table if it does not exist yet.
+	_, err = db.Exec("CREATE TABLE IF NOT EXISTS servers (address TEXT, UNIQUE(address))")
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create servers table")
+	}
+
+	store := NewNodeStore(db, "main", "servers", "address")
+
+	return store, nil
+}
+
+// NewNodeStore creates a new NodeStore.
+func NewNodeStore(db *sql.DB, schema, table, column string) *DatabaseNodeStore {
+	return &DatabaseNodeStore{
+		db:     db,
+		schema: schema,
+		table:  table,
+		column: column,
+	}
+}
+
+// Get the current servers.
+func (d *DatabaseNodeStore) Get(ctx context.Context) ([]NodeInfo, error) {
+	tx, err := d.db.Begin()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to begin transaction")
+	}
+	defer tx.Rollback()
+
+	query := fmt.Sprintf("SELECT %s FROM %s.%s", d.column, d.schema, d.table)
+	rows, err := tx.QueryContext(ctx, query)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to query servers table")
+	}
+	defer rows.Close()
+
+	servers := make([]NodeInfo, 0)
+	for rows.Next() {
+		var address string
+		err := rows.Scan(&address)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to fetch server address")
+		}
+		servers = append(servers, NodeInfo{ID: 1, Address: address})
+	}
+	if err := rows.Err(); err != nil {
+		return nil, errors.Wrap(err, "result set failure")
+	}
+
+	return servers, nil
+}
+
+// Set the servers addresses.
+func (d *DatabaseNodeStore) Set(ctx context.Context, servers []NodeInfo) error {
+	tx, err := d.db.Begin()
+	if err != nil {
+		return errors.Wrap(err, "failed to begin transaction")
+	}
+
+	query := fmt.Sprintf("DELETE FROM %s.%s", d.schema, d.table)
+	if _, err := tx.ExecContext(ctx, query); err != nil {
+		tx.Rollback()
+		return errors.Wrap(err, "failed to delete existing servers rows")
+	}
+
+	query = fmt.Sprintf("INSERT INTO %s.%s(%s) VALUES (?)", d.schema, d.table, d.column)
+	stmt, err := tx.PrepareContext(ctx, query)
+	if err != nil {
+		tx.Rollback()
+		return errors.Wrap(err, "failed to prepare insert statement")
+	}
+	defer stmt.Close()
+
+	for _, server := range servers {
+		if _, err := stmt.ExecContext(ctx, server.Address); err != nil {
+			tx.Rollback()
+			return errors.Wrapf(err, "failed to insert server %s", server.Address)
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return errors.Wrap(err, "failed to commit transaction")
+	}
+
+	return nil
+}

vendor/github.com/canonical/go-dqlite/internal/bindings/build.go

+package bindings
+
+/*
+#cgo linux LDFLAGS: -lsqlite3 -lraft -lco -ldqlite
+*/
+import "C"

vendor/github.com/canonical/go-dqlite/internal/bindings/errors.go

+package bindings
+
+/*
+#include <sqlite3.h>
+*/
+import "C"
+
+// Error holds information about a SQLite error.
+type Error struct {
+	Code    int
+	Message string
+}
+
+func (e Error) Error() string {
+	if e.Message != "" {
+		return e.Message
+	}
+	return C.GoString(C.sqlite3_errstr(C.int(e.Code)))
+}

vendor/github.com/canonical/go-dqlite/internal/bindings/server.go

+package bindings
+
+/*
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <stdint.h>
+#include <signal.h>
+
+#include <dqlite.h>
+#include <raft.h>
+#include <sqlite3.h>
+
+#define EMIT_BUF_LEN 1024
+
+typedef unsigned long long nanoseconds_t;
+
+// Duplicate a file descriptor and prevent it from being cloned into child processes.
+static int dupCloexec(int oldfd) {
+	int newfd = -1;
+
+	newfd = dup(oldfd);
+	if (newfd < 0) {
+		return -1;
+	}
+
+	if (fcntl(newfd, F_SETFD, FD_CLOEXEC) < 0) {
+		return -1;
+	}
+
+	return newfd;
+}
+
+// C to Go trampoline for custom connect function.
+int connectWithDial(uintptr_t handle, char *address, int *fd);
+
+// Wrapper to call the Go trampoline.
+static int connectTrampoline(void *data, const char *address, int *fd) {
+        uintptr_t handle = (uintptr_t)(data);
+        return connectWithDial(handle, (char*)address, fd);
+}
+
+// Configure a custom connect function.
+static int configConnectFunc(dqlite_node *t, uintptr_t handle) {
+        return dqlite_node_set_connect_func(t, connectTrampoline, (void*)handle);
+}
+
+static int initializeSQLite()
+{
+	int rc;
+
+	// Configure SQLite for single-thread mode. This is a global config.
+	rc = sqlite3_config(SQLITE_CONFIG_SINGLETHREAD);
+	if (rc != SQLITE_OK) {
+		assert(rc == SQLITE_MISUSE);
+		return DQLITE_MISUSE;
+	}
+	return 0;
+}
+
+static dqlite_node_info *makeInfos(int n) {
+	return calloc(n, sizeof(dqlite_node_info));
+}
+
+static void setInfo(dqlite_node_info *infos, unsigned i, unsigned id, const char *address) {
+	dqlite_node_info *info = &infos[i];
+	info->id = id;
+	info->address = address;
+}
+
+*/
+import "C"
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"sync"
+	"time"
+	"unsafe"
+
+	"github.com/canonical/go-dqlite/internal/protocol"
+)
+
+type Node C.dqlite_node
+
+// Init initializes dqlite global state.
+func Init() error {
+	// FIXME: ignore SIGPIPE, see https://github.com/joyent/libuv/issues/1254
+	C.signal(C.SIGPIPE, C.SIG_IGN)
+	// Don't enable single thread mode when running tests. TODO: find a
+	// better way to expose this functionality.
+	if os.Getenv("GO_DQLITE_MULTITHREAD") == "1" {
+		return nil
+	}
+	if rc := C.initializeSQLite(); rc != 0 {
+		return fmt.Errorf("%d", rc)
+	}
+	return nil
+}
+
+// NewNode creates a new Node instance.
+func NewNode(id uint64, address string, dir string) (*Node, error) {
+	var server *C.dqlite_node
+	cid := C.unsigned(id)
+
+	caddress := C.CString(address)
+	defer C.free(unsafe.Pointer(caddress))
+
+	cdir := C.CString(dir)
+	defer C.free(unsafe.Pointer(cdir))
+
+	if rc := C.dqlite_node_create(cid, caddress, cdir, &server); rc != 0 {
+		return nil, fmt.Errorf("failed to create task object")
+	}
+
+	return (*Node)(unsafe.Pointer(server)), nil
+}
+
+func (s *Node) SetDialFunc(dial protocol.DialFunc) error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	connectLock.Lock()
+	defer connectLock.Unlock()
+	connectIndex++
+	connectRegistry[connectIndex] = dial
+	if rc := C.configConnectFunc(server, connectIndex); rc != 0 {
+		return fmt.Errorf("failed to set connect func")
+	}
+	return nil
+}
+
+func (s *Node) SetBindAddress(address string) error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	caddress := C.CString(address)
+	defer C.free(unsafe.Pointer(caddress))
+	if rc := C.dqlite_node_set_bind_address(server, caddress); rc != 0 {
+		return fmt.Errorf("failed to set bind address")
+	}
+	return nil
+}
+
+func (s *Node) SetNetworkLatency(nanoseconds uint64) error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	cnanoseconds := C.nanoseconds_t(nanoseconds)
+	if rc := C.dqlite_node_set_network_latency(server, cnanoseconds); rc != 0 {
+		return fmt.Errorf("failed to set network latency")
+	}
+	return nil
+}
+
+func (s *Node) GetBindAddress() string {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	return C.GoString(C.dqlite_node_get_bind_address(server))
+}
+
+func (s *Node) Start() error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	if rc := C.dqlite_node_start(server); rc != 0 {
+		return fmt.Errorf("failed to start task")
+	}
+	return nil
+}
+
+func (s *Node) Stop() error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	if rc := C.dqlite_node_stop(server); rc != 0 {
+		return fmt.Errorf("task stopped with error code %d", rc)
+	}
+	return nil
+}
+
+// Close the server releasing all used resources.
+func (s *Node) Close() {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	C.dqlite_node_destroy(server)
+}
+
+func (s *Node) Recover(cluster []protocol.NodeInfo) error {
+	server := (*C.dqlite_node)(unsafe.Pointer(s))
+	n := C.int(len(cluster))
+	infos := C.makeInfos(n)
+	defer C.free(unsafe.Pointer(infos))
+	for i, info := range cluster {
+		cid := C.unsigned(info.ID)
+		caddress := C.CString(info.Address)
+		defer C.free(unsafe.Pointer(caddress))
+		C.setInfo(infos, C.unsigned(i), cid, caddress)
+	}
+	if rc := C.dqlite_node_recover(server, infos, n); rc != 0 {
+		return fmt.Errorf("recover failed with error code %d", rc)
+	}
+	return nil
+}
+
+// Extract the underlying socket from a connection.
+func connToSocket(conn net.Conn) (C.int, error) {
+	file, err := conn.(fileConn).File()
+	if err != nil {
+		return C.int(-1), err
+	}
+
+	fd1 := C.int(file.Fd())
+
+	// Duplicate the file descriptor, in order to prevent Go's finalizer to
+	// close it.
+	fd2 := C.dupCloexec(fd1)
+	if fd2 < 0 {
+		return C.int(-1), fmt.Errorf("failed to dup socket fd")
+	}
+
+	conn.Close()
+
+	return fd2, nil
+}
+
+// Interface that net.Conn must implement in order to extract the underlying
+// file descriptor.
+type fileConn interface {
+	File() (*os.File, error)
+}
+
+//export connectWithDial
+func connectWithDial(handle C.uintptr_t, address *C.char, fd *C.int) C.int {
+	connectLock.Lock()
+	defer connectLock.Unlock()
+	dial := connectRegistry[handle]
+	// TODO: make timeout customizable.
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	conn, err := dial(ctx, C.GoString(address))
+	if err != nil {
+		return C.RAFT_NOCONNECTION
+	}
+	socket, err := connToSocket(conn)
+	if err != nil {
+		return C.RAFT_NOCONNECTION
+	}
+	*fd = socket
+	return C.int(0)
+}
+
+// Use handles to avoid passing Go pointers to C.
+var connectRegistry = make(map[C.uintptr_t]protocol.DialFunc)
+var connectIndex C.uintptr_t = 100
+var connectLock = sync.Mutex{}
+
+// ErrNodeStopped is returned by Node.Handle() is the server was stopped.
+var ErrNodeStopped = fmt.Errorf("server was stopped")
+
+// To compare bool values.
+var cfalse C.bool

vendor/github.com/canonical/go-dqlite/internal/logging/func.go

+package logging
+
+import (
+	"fmt"
+	"testing"
+)
+
+// Func is a function that can be used for logging.
+type Func func(Level, string, ...interface{})
+
+// Test returns a logging function that forwards messages to the test logger.
+func Test(t *testing.T) Func {
+	return func(l Level, format string, a ...interface{}) {
+		format = fmt.Sprintf("%s: %s", l.String(), format)
+		t.Logf(format, a...)
+	}
+}
+
+// Stdout returns a logging function that prints log messages on standard
+// output.
+func Stdout() Func {
+	return func(l Level, format string, a ...interface{}) {
+		format = fmt.Sprintf("%s: %s\n", l.String(), format)
+		fmt.Printf(format, a...)
+	}
+}

vendor/github.com/canonical/go-dqlite/internal/logging/level.go

+package logging
+
+// Level defines the logging level.
+type Level int
+
+// Available logging levels.
+const (
+	Debug Level = iota
+	Info
+	Warn
+	Error
+)
+
+func (l Level) String() string {
+	switch l {
+	case Debug:
+		return "DEBUG"
+	case Info:
+		return "INFO"
+	case Warn:
+		return "WARN"
+	case Error:
+		return "ERROR"
+	default:
+		return "UNKNOWN"
+	}
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/buffer.go

+package protocol
+
+// Buffer for reading responses or writing requests.
+type buffer struct {
+	Bytes  []byte
+	Offset int
+}
+
+func (b *buffer) Advance(amount int) {
+	b.Offset += amount
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/config.go

+package protocol
+
+import (
+	"time"
+
+	"github.com/Rican7/retry/strategy"
+)
+
+// Config holds various configuration parameters for a dqlite client.
+type Config struct {
+	Dial            DialFunc            // Network dialer.
+	AttemptTimeout  time.Duration       // Timeout for each individual Dial attempt.
+	RetryStrategies []strategy.Strategy // Strategies used for retrying to connect to a leader.
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/connector.go

+package protocol
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+
+	"github.com/Rican7/retry"
+	"github.com/canonical/go-dqlite/internal/logging"
+	"github.com/pkg/errors"
+)
+
+// DialFunc is a function that can be used to establish a network connection.
+type DialFunc func(context.Context, string) (net.Conn, error)
+
+// Connector is in charge of creating a dqlite SQL client connected to the
+// current leader of a cluster.
+type Connector struct {
+	id     uint64       // Conn ID to use when registering against the server.
+	store  NodeStore    // Used to get and update current cluster servers.
+	config Config       // Connection parameters.
+	log    logging.Func // Logging function.
+}
+
+// NewConnector returns a new connector that can be used by a dqlite driver to
+// create new clients connected to a leader dqlite server.
+func NewConnector(id uint64, store NodeStore, config Config, log logging.Func) *Connector {
+	connector := &Connector{
+		id:     id,
+		store:  store,
+		config: config,
+		log:    log,
+	}
+
+	return connector
+}
+
+// Connect finds the leader server and returns a connection to it.
+//
+// If the connector is stopped before a leader is found, nil is returned.
+func (c *Connector) Connect(ctx context.Context) (*Protocol, error) {
+	var protocol *Protocol
+
+	// The retry strategy should be configured to retry indefinitely, until
+	// the given context is done.
+	err := retry.Retry(func(attempt uint) error {
+		log := func(l logging.Level, format string, a ...interface{}) {
+			format += fmt.Sprintf(" attempt=%d", attempt)
+			c.log(l, fmt.Sprintf(format, a...))
+		}
+
+		select {
+		case <-ctx.Done():
+			// Stop retrying
+			return nil
+		default:
+		}
+
+		var err error
+		protocol, err = c.connectAttemptAll(ctx, log)
+		if err != nil {
+			log(logging.Debug, "connection failed err=%v", err)
+			return err
+		}
+
+		return nil
+	}, c.config.RetryStrategies...)
+
+	if err != nil {
+		// The retry strategy should never give up until success or
+		// context expiration.
+		panic("connect retry aborted unexpectedly")
+	}
+
+	if ctx.Err() != nil {
+		return nil, ErrNoAvailableLeader
+	}
+
+	return protocol, nil
+}
+
+// Make a single attempt to establish a connection to the leader server trying
+// all addresses available in the store.
+func (c *Connector) connectAttemptAll(ctx context.Context, log logging.Func) (*Protocol, error) {
+	servers, err := c.store.Get(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get cluster servers")
+	}
+
+	// Make an attempt for each address until we find the leader.
+	for _, server := range servers {
+		log := func(l logging.Level, format string, a ...interface{}) {
+			format += fmt.Sprintf(" address=%s", server.Address)
+			log(l, fmt.Sprintf(format, a...))
+		}
+
+		ctx, cancel := context.WithTimeout(ctx, c.config.AttemptTimeout)
+		defer cancel()
+
+		version := VersionOne
+		protocol, leader, err := c.connectAttemptOne(ctx, server.Address, version)
+		if err == errBadProtocol {
+			version = VersionLegacy
+			protocol, leader, err = c.connectAttemptOne(ctx, server.Address, version)
+		}
+		if err != nil {
+			// This server is unavailable, try with the next target.
+			log(logging.Debug, "server connection failed err=%v", err)
+			continue
+		}
+		if protocol != nil {
+			// We found the leader
+			log(logging.Info, "connected")
+			return protocol, nil
+		}
+		if leader == "" {
+			// This server does not know who the current leader is,
+			// try with the next target.
+			continue
+		}
+
+		// If we get here, it means this server reported that another
+		// server is the leader, let's close the connection to this
+		// server and try with the suggested one.
+		//logger = logger.With(zap.String("leader", leader))
+		protocol, leader, err = c.connectAttemptOne(ctx, leader, version)
+		if err != nil {
+			// The leader reported by the previous server is
+			// unavailable, try with the next target.
+			//logger.Info("leader server connection failed", zap.String("err", err.Error()))
+			continue
+		}
+		if protocol == nil {
+			// The leader reported by the target server does not consider itself
+			// the leader, try with the next target.
+			//logger.Info("reported leader server is not the leader")
+			continue
+		}
+		log(logging.Info, "connected")
+		return protocol, nil
+	}
+
+	return nil, ErrNoAvailableLeader
+}
+
+// Connect establishes a connection with a dqlite node.
+func Connect(ctx context.Context, dial DialFunc, address string, version uint64) (*Protocol, error) {
+	// Establish the connection.
+	conn, err := dial(ctx, address)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to establish network connection")
+	}
+
+	// Latest protocol version.
+	protocol := make([]byte, 8)
+	binary.LittleEndian.PutUint64(protocol, version)
+
+	// Perform the protocol handshake.
+	n, err := conn.Write(protocol)
+	if err != nil {
+		conn.Close()
+		return nil, errors.Wrap(err, "failed to send handshake")
+	}
+	if n != 8 {
+		conn.Close()
+		return nil, errors.Wrap(io.ErrShortWrite, "failed to send handshake")
+	}
+
+	return NewProtocol(version, conn), nil
+}
+
+// Connect to the given dqlite server and check if it's the leader.
+//
+// Return values:
+//
+// - Any failure is hit:                     -> nil, "", err
+// - Target not leader and no leader known:  -> nil, "", nil
+// - Target not leader and leader known:     -> nil, leader, nil
+// - Target is the leader:                   -> server, "", nil
+//
+func (c *Connector) connectAttemptOne(ctx context.Context, address string, version uint64) (*Protocol, string, error) {
+	protocol, err := Connect(ctx, c.config.Dial, address, version)
+	if err != nil {
+		return nil, "", err
+	}
+
+	// Send the initial Leader request.
+	request := Message{}
+	request.Init(16)
+	response := Message{}
+	response.Init(512)
+
+	EncodeLeader(&request)
+
+	if err := protocol.Call(ctx, &request, &response); err != nil {
+		protocol.Close()
+		cause := errors.Cause(err)
+		// Best-effort detection of a pre-1.0 dqlite node: when sent
+		// version 1 it should close the connection immediately.
+		if _, ok := cause.(*net.OpError); ok || cause == io.EOF {
+			return nil, "", errBadProtocol
+		}
+
+		return nil, "", errors.Wrap(err, "failed to send Leader request")
+	}
+
+	_, leader, err := DecodeNodeCompat(protocol, &response)
+	if err != nil {
+		protocol.Close()
+		return nil, "", errors.Wrap(err, "failed to parse Node response")
+	}
+
+	switch leader {
+	case "":
+		// Currently this server does not know about any leader.
+		protocol.Close()
+		return nil, "", nil
+	case address:
+		// This server is the leader, register ourselves and return.
+		request.Reset()
+		response.Reset()
+
+		EncodeClient(&request, c.id)
+
+		if err := protocol.Call(ctx, &request, &response); err != nil {
+			protocol.Close()
+			return nil, "", errors.Wrap(err, "failed to send Conn request")
+		}
+
+		_, err := DecodeWelcome(&response)
+		if err != nil {
+			protocol.Close()
+			return nil, "", errors.Wrap(err, "failed to parse Welcome response")
+		}
+
+		// TODO: enable heartbeat
+		// protocol.heartbeatTimeout = time.Duration(heartbeatTimeout) * time.Millisecond
+		//go protocol.heartbeat()
+
+		return protocol, "", nil
+	default:
+		// This server claims to know who the current leader is.
+		protocol.Close()
+		return nil, leader, nil
+	}
+}
+
+var errBadProtocol = fmt.Errorf("bad protocol")

vendor/github.com/canonical/go-dqlite/internal/protocol/constants.go

+package protocol
+
+// VersionOne is version 1 of the server protocol.
+const VersionOne = uint64(1)
+
+// VersionLegacy is the pre 1.0 dqlite server protocol version.
+const VersionLegacy = uint64(0x86104dd760433fe5)
+
+// SQLite datatype codes
+const (
+	Integer = 1
+	Float   = 2
+	Text    = 3
+	Blob    = 4
+	Null    = 5
+)
+
+// Special data types for time values.
+const (
+	UnixTime = 9
+	ISO8601  = 10
+	Boolean  = 11
+)
+
+// Request types.
+const (
+	RequestLeader    = 0
+	RequestClient    = 1
+	RequestHeartbeat = 2
+	RequestOpen      = 3
+	RequestPrepare   = 4
+	RequestExec      = 5
+	RequestQuery     = 6
+	RequestFinalize  = 7
+	RequestExecSQL   = 8
+	RequestQuerySQL  = 9
+	RequestInterrupt = 10
+	RequestJoin      = 12
+	RequestPromote   = 13
+	RequestRemove    = 14
+	RequestDump      = 15
+	RequestCluster   = 16
+)
+
+// Response types.
+const (
+	ResponseFailure    = 0
+	ResponseNode       = 1
+	ResponseNodeLegacy = 1
+	ResponseWelcome    = 2
+	ResponseNodes      = 3
+	ResponseDb         = 4
+	ResponseStmt       = 5
+	ResponseResult     = 6
+	ResponseRows       = 7
+	ResponseEmpty      = 8
+	ResponseFiles      = 9
+)

vendor/github.com/canonical/go-dqlite/internal/protocol/dial.go

+package protocol
+
+import (
+	"context"
+	"net"
+)
+
+// TCPDial is a dial function using plain TCP to establish the network
+// connection.
+func TCPDial(ctx context.Context, address string) (net.Conn, error) {
+	dialer := net.Dialer{}
+	return dialer.DialContext(ctx, "tcp", address)
+}
+
+// UnixDial is a dial function using Unix sockets to establish the network
+// connection.
+func UnixDial(ctx context.Context, address string) (net.Conn, error) {
+	dialer := net.Dialer{}
+	return dialer.DialContext(ctx, "unix", address)
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/errors.go

+package protocol
+
+import (
+	"fmt"
+)
+
+// Client errors.
+var (
+	ErrNoAvailableLeader = fmt.Errorf("no available dqlite leader server found")
+	errStop              = fmt.Errorf("connector was stopped")
+	errStaleLeader       = fmt.Errorf("server has lost leadership")
+	errNotClustered      = fmt.Errorf("server is not clustered")
+	errNegativeRead      = fmt.Errorf("reader returned negative count from Read")
+	errMessageEOF        = fmt.Errorf("message eof")
+)
+
+// ErrRequest is returned in case of request failure.
+type ErrRequest struct {
+	Code        uint64
+	Description string
+}
+
+func (e ErrRequest) Error() string {
+	return fmt.Sprintf("%s (%d)", e.Description, e.Code)
+}
+
+// ErrRowsPart is returned when the first batch of a multi-response result
+// batch is done.
+var ErrRowsPart = fmt.Errorf("not all rows were returned in this response")

vendor/github.com/canonical/go-dqlite/internal/protocol/protocol.go

+package protocol
+
+import (
+	"context"
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// Protocol sends and receive the dqlite message on the wire.
+type Protocol struct {
+	version        uint64        // Protocol version
+	conn           net.Conn      // Underlying network connection.
+	contextTimeout time.Duration // Default context timeout.
+	closeCh        chan struct{} // Stops the heartbeat when the connection gets closed
+	mu             sync.Mutex    // Serialize requests
+	netErr         error         // A network error occurred
+}
+
+func NewProtocol(version uint64, conn net.Conn) *Protocol {
+	protocol := &Protocol{
+		version:        version,
+		conn:           conn,
+		closeCh:        make(chan struct{}),
+		contextTimeout: 5 * time.Second,
+	}
+
+	return protocol
+}
+
+// SetContextTimeout sets the default context timeout when no deadline is
+// provided.
+func (p *Protocol) SetContextTimeout(timeout time.Duration) {
+	p.contextTimeout = timeout
+}
+
+// Call invokes a dqlite RPC, sending a request message and receiving a
+// response message.
+func (p *Protocol) Call(ctx context.Context, request, response *Message) (err error) {
+	// We need to take a lock since the dqlite server currently does not
+	// support concurrent requests.
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.netErr != nil {
+		return p.netErr
+	}
+
+	// Honor the ctx deadline, if present, or use a default.
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		deadline = time.Now().Add(p.contextTimeout)
+	}
+
+	p.conn.SetDeadline(deadline)
+
+	if err = p.send(request); err != nil {
+		err = errors.Wrap(err, "failed to send request")
+		goto err
+	}
+
+	if err = p.recv(response); err != nil {
+		err = errors.Wrap(err, "failed to receive response")
+		goto err
+	}
+
+	return
+
+err:
+	switch errors.Cause(err).(type) {
+	case *net.OpError:
+		p.netErr = err
+	}
+	return
+}
+
+// More is used when a request maps to multiple responses.
+func (p *Protocol) More(ctx context.Context, response *Message) error {
+	return p.recv(response)
+}
+
+// Interrupt sends an interrupt request and awaits for the server's empty
+// response.
+func (p *Protocol) Interrupt(ctx context.Context, request *Message, response *Message) error {
+	// We need to take a lock since the dqlite server currently does not
+	// support concurrent requests.
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	// Honor the ctx deadline, if present, or use a default.
+	deadline, ok := ctx.Deadline()
+	if !ok {
+		deadline = time.Now().Add(2 * time.Second)
+	}
+	p.conn.SetDeadline(deadline)
+
+	defer request.Reset()
+
+	EncodeInterrupt(request, 0)
+
+	if err := p.send(request); err != nil {
+		return errors.Wrap(err, "failed to send interrupt request")
+	}
+
+	for {
+		if err := p.recv(response); err != nil {
+			response.Reset()
+			return errors.Wrap(err, "failed to receive response")
+		}
+
+		mtype, _ := response.getHeader()
+		response.Reset()
+
+		if mtype == ResponseEmpty {
+			break
+		}
+	}
+
+	return nil
+}
+
+// Close the client connection.
+func (p *Protocol) Close() error {
+	close(p.closeCh)
+	return p.conn.Close()
+}
+
+func (p *Protocol) send(req *Message) error {
+	if err := p.sendHeader(req); err != nil {
+		return errors.Wrap(err, "failed to send header")
+	}
+
+	if err := p.sendBody(req); err != nil {
+		return errors.Wrap(err, "failed to send body")
+	}
+
+	return nil
+}
+
+func (p *Protocol) sendHeader(req *Message) error {
+	n, err := p.conn.Write(req.header[:])
+	if err != nil {
+		return errors.Wrap(err, "failed to send header")
+	}
+
+	if n != messageHeaderSize {
+		return errors.Wrap(io.ErrShortWrite, "failed to send header")
+	}
+
+	return nil
+}
+
+func (p *Protocol) sendBody(req *Message) error {
+	buf := req.body1.Bytes[:req.body1.Offset]
+	n, err := p.conn.Write(buf)
+	if err != nil {
+		return errors.Wrap(err, "failed to send static body")
+	}
+
+	if n != len(buf) {
+		return errors.Wrap(io.ErrShortWrite, "failed to write body")
+	}
+
+	if req.body2.Bytes == nil {
+		return nil
+	}
+
+	buf = req.body2.Bytes[:req.body2.Offset]
+	n, err = p.conn.Write(buf)
+	if err != nil {
+		return errors.Wrap(err, "failed to send dynamic body")
+	}
+
+	if n != len(buf) {
+		return errors.Wrap(io.ErrShortWrite, "failed to write body")
+	}
+
+	return nil
+}
+
+func (p *Protocol) recv(res *Message) error {
+	if err := p.recvHeader(res); err != nil {
+		return errors.Wrap(err, "failed to receive header")
+	}
+
+	if err := p.recvBody(res); err != nil {
+		return errors.Wrap(err, "failed to receive body")
+	}
+
+	return nil
+}
+
+func (p *Protocol) recvHeader(res *Message) error {
+	if err := p.recvPeek(res.header); err != nil {
+		return errors.Wrap(err, "failed to receive header")
+	}
+
+	res.words = binary.LittleEndian.Uint32(res.header[0:])
+	res.mtype = res.header[4]
+	res.flags = res.header[5]
+	res.extra = binary.LittleEndian.Uint16(res.header[6:])
+
+	return nil
+}
+
+func (p *Protocol) recvBody(res *Message) error {
+	n := int(res.words) * messageWordSize
+	n1 := n
+	n2 := 0
+
+	if n1 > len(res.body1.Bytes) {
+		// We need to allocate the dynamic buffer.
+		n1 = len(res.body1.Bytes)
+		n2 = n - n1
+	}
+
+	buf := res.body1.Bytes[:n1]
+
+	if err := p.recvPeek(buf); err != nil {
+		return errors.Wrap(err, "failed to read body")
+	}
+
+	if n2 > 0 {
+		res.body2.Bytes = make([]byte, n2)
+		res.body2.Offset = 0
+		buf = res.body2.Bytes
+		if err := p.recvPeek(buf); err != nil {
+			return errors.Wrap(err, "failed to read body")
+		}
+	}
+
+	return nil
+}
+
+// Read until buf is full.
+func (p *Protocol) recvPeek(buf []byte) error {
+	for offset := 0; offset < len(buf); {
+		n, err := p.recvFill(buf[offset:])
+		if err != nil {
+			return err
+		}
+		offset += n
+	}
+
+	return nil
+}
+
+// Try to fill buf, but perform at most one read.
+func (p *Protocol) recvFill(buf []byte) (int, error) {
+	// Read new data: try a limited number of times.
+	//
+	// This technique is copied from bufio.Reader.
+	for i := messageMaxConsecutiveEmptyReads; i > 0; i-- {
+		n, err := p.conn.Read(buf)
+		if n < 0 {
+			panic(errNegativeRead)
+		}
+		if err != nil {
+			return -1, err
+		}
+		if n > 0 {
+			return n, nil
+		}
+	}
+	return -1, io.ErrNoProgress
+}
+
+/*
+func (p *Protocol) heartbeat() {
+	request := Message{}
+	request.Init(16)
+	response := Message{}
+	response.Init(512)
+
+	for {
+		delay := c.heartbeatTimeout / 3
+
+		//c.logger.Debug("sending heartbeat", zap.Duration("delay", delay))
+		time.Sleep(delay)
+
+		// Check if we've been closed.
+		select {
+		case <-c.closeCh:
+			return
+		default:
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+
+		EncodeHeartbeat(&request, uint64(time.Now().Unix()))
+
+		err := c.Call(ctx, &request, &response)
+
+		// We bail out upon failures.
+		//
+		// TODO: make the client survive temporary disconnections.
+		if err != nil {
+			cancel()
+			//c.logger.Error("heartbeat failed", zap.Error(err))
+			return
+		}
+
+		//addresses, err := DecodeNodes(&response)
+		_, err = DecodeNodes(&response)
+		if err != nil {
+			cancel()
+			//c.logger.Error("invalid heartbeat response", zap.Error(err))
+			return
+		}
+
+		// if err := c.store.Set(ctx, addresses); err != nil {
+		// 	cancel()
+		// 	c.logger.Error("failed to update servers", zap.Error(err))
+		// 	return
+		// }
+
+		cancel()
+
+		request.Reset()
+		response.Reset()
+	}
+}
+*/
+
+// DecodeNodeCompat handles also pre-1.0 legacy server messages.
+func DecodeNodeCompat(protocol *Protocol, response *Message) (uint64, string, error) {
+	if protocol.version == VersionLegacy {
+		address, err := DecodeNodeLegacy(response)
+		if err != nil {
+			return 0, "", err
+		}
+		return 0, address, nil
+
+	}
+	return DecodeNode(response)
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/request.go

+package protocol
+
+// DO NOT EDIT
+//
+// This file was generated by ./schema.sh
+
+
+// EncodeLeader encodes a Leader request.
+func EncodeLeader(request *Message) {
+	request.putUint64(0)
+
+	request.putHeader(RequestLeader)
+}
+
+// EncodeClient encodes a Client request.
+func EncodeClient(request *Message, id uint64) {
+	request.putUint64(id)
+
+	request.putHeader(RequestClient)
+}
+
+// EncodeHeartbeat encodes a Heartbeat request.
+func EncodeHeartbeat(request *Message, timestamp uint64) {
+	request.putUint64(timestamp)
+
+	request.putHeader(RequestHeartbeat)
+}
+
+// EncodeOpen encodes a Open request.
+func EncodeOpen(request *Message, name string, flags uint64, vfs string) {
+	request.putString(name)
+	request.putUint64(flags)
+	request.putString(vfs)
+
+	request.putHeader(RequestOpen)
+}
+
+// EncodePrepare encodes a Prepare request.
+func EncodePrepare(request *Message, db uint64, sql string) {
+	request.putUint64(db)
+	request.putString(sql)
+
+	request.putHeader(RequestPrepare)
+}
+
+// EncodeExec encodes a Exec request.
+func EncodeExec(request *Message, db uint32, stmt uint32, values NamedValues) {
+	request.putUint32(db)
+	request.putUint32(stmt)
+	request.putNamedValues(values)
+
+	request.putHeader(RequestExec)
+}
+
+// EncodeQuery encodes a Query request.
+func EncodeQuery(request *Message, db uint32, stmt uint32, values NamedValues) {
+	request.putUint32(db)
+	request.putUint32(stmt)
+	request.putNamedValues(values)
+
+	request.putHeader(RequestQuery)
+}
+
+// EncodeFinalize encodes a Finalize request.
+func EncodeFinalize(request *Message, db uint32, stmt uint32) {
+	request.putUint32(db)
+	request.putUint32(stmt)
+
+	request.putHeader(RequestFinalize)
+}
+
+// EncodeExecSQL encodes a ExecSQL request.
+func EncodeExecSQL(request *Message, db uint64, sql string, values NamedValues) {
+	request.putUint64(db)
+	request.putString(sql)
+	request.putNamedValues(values)
+
+	request.putHeader(RequestExecSQL)
+}
+
+// EncodeQuerySQL encodes a QuerySQL request.
+func EncodeQuerySQL(request *Message, db uint64, sql string, values NamedValues) {
+	request.putUint64(db)
+	request.putString(sql)
+	request.putNamedValues(values)
+
+	request.putHeader(RequestQuerySQL)
+}
+
+// EncodeInterrupt encodes a Interrupt request.
+func EncodeInterrupt(request *Message, db uint64) {
+	request.putUint64(db)
+
+	request.putHeader(RequestInterrupt)
+}
+
+// EncodeJoin encodes a Join request.
+func EncodeJoin(request *Message, id uint64, address string) {
+	request.putUint64(id)
+	request.putString(address)
+
+	request.putHeader(RequestJoin)
+}
+
+// EncodePromote encodes a Promote request.
+func EncodePromote(request *Message, id uint64) {
+	request.putUint64(id)
+
+	request.putHeader(RequestPromote)
+}
+
+// EncodeRemove encodes a Remove request.
+func EncodeRemove(request *Message, id uint64) {
+	request.putUint64(id)
+
+	request.putHeader(RequestRemove)
+}
+
+// EncodeDump encodes a Dump request.
+func EncodeDump(request *Message, name string) {
+	request.putString(name)
+
+	request.putHeader(RequestDump)
+}
+
+// EncodeCluster encodes a Cluster request.
+func EncodeCluster(request *Message) {
+	request.putUint64(0)
+
+	request.putHeader(RequestCluster)
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/response.go

+package protocol
+
+// DO NOT EDIT
+//
+// This file was generated by ./schema.sh
+
+import "fmt"
+
+// DecodeFailure decodes a Failure response.
+func DecodeFailure(response *Message) (code uint64, message string, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseFailure {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	code = response.getUint64()
+	message = response.getString()
+
+	return
+}
+
+// DecodeWelcome decodes a Welcome response.
+func DecodeWelcome(response *Message) (heartbeatTimeout uint64, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseWelcome {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	heartbeatTimeout = response.getUint64()
+
+	return
+}
+
+// DecodeNodeLegacy decodes a NodeLegacy response.
+func DecodeNodeLegacy(response *Message) (address string, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseNodeLegacy {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	address = response.getString()
+
+	return
+}
+
+// DecodeNode decodes a Node response.
+func DecodeNode(response *Message) (id uint64, address string, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseNode {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	id = response.getUint64()
+	address = response.getString()
+
+	return
+}
+
+// DecodeNodes decodes a Nodes response.
+func DecodeNodes(response *Message) (servers Nodes, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseNodes {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	servers = response.getNodes()
+
+	return
+}
+
+// DecodeDb decodes a Db response.
+func DecodeDb(response *Message) (id uint32, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseDb {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	id = response.getUint32()
+	response.getUint32()
+
+	return
+}
+
+// DecodeStmt decodes a Stmt response.
+func DecodeStmt(response *Message) (db uint32, id uint32, params uint64, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseStmt {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	db = response.getUint32()
+	id = response.getUint32()
+	params = response.getUint64()
+
+	return
+}
+
+// DecodeEmpty decodes a Empty response.
+func DecodeEmpty(response *Message) (err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseEmpty {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	response.getUint64()
+
+	return
+}
+
+// DecodeResult decodes a Result response.
+func DecodeResult(response *Message) (result Result, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseResult {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	result = response.getResult()
+
+	return
+}
+
+// DecodeRows decodes a Rows response.
+func DecodeRows(response *Message) (rows Rows, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseRows {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	rows = response.getRows()
+
+	return
+}
+
+// DecodeFiles decodes a Files response.
+func DecodeFiles(response *Message) (files Files, err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+		err = e
+		return
+	}
+
+	if mtype != ResponseFiles {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+		return
+	}
+
+	files = response.getFiles()
+
+	return
+}

vendor/github.com/canonical/go-dqlite/internal/protocol/schema.go

+package protocol
+
+//go:generate ./schema.sh --request init
+
+//go:generate ./schema.sh --request Leader    unused:uint64
+//go:generate ./schema.sh --request Client    id:uint64
+//go:generate ./schema.sh --request Heartbeat timestamp:uint64
+//go:generate ./schema.sh --request Open      name:string flags:uint64 vfs:string
+//go:generate ./schema.sh --request Prepare   db:uint64 sql:string
+//go:generate ./schema.sh --request Exec      db:uint32 stmt:uint32 values:NamedValues
+//go:generate ./schema.sh --request Query     db:uint32 stmt:uint32 values:NamedValues
+//go:generate ./schema.sh --request Finalize  db:uint32 stmt:uint32
+//go:generate ./schema.sh --request ExecSQL   db:uint64 sql:string values:NamedValues
+//go:generate ./schema.sh --request QuerySQL  db:uint64 sql:string values:NamedValues
+//go:generate ./schema.sh --request Interrupt  db:uint64
+//go:generate ./schema.sh --request Join  id:uint64 address:string
+//go:generate ./schema.sh --request Promote   id:uint64
+//go:generate ./schema.sh --request Remove    id:uint64
+//go:generate ./schema.sh --request Dump      name:string
+//go:generate ./schema.sh --request Cluster   unused:uint64
+
+//go:generate ./schema.sh --response init
+//go:generate ./schema.sh --response Failure  code:uint64 message:string
+//go:generate ./schema.sh --response Welcome  heartbeatTimeout:uint64
+//go:generate ./schema.sh --response NodeLegacy  address:string
+//go:generate ./schema.sh --response Node   id:uint64 address:string
+//go:generate ./schema.sh --response Nodes  servers:Nodes
+//go:generate ./schema.sh --response Db       id:uint32 unused:uint32
+//go:generate ./schema.sh --response Stmt     db:uint32 id:uint32 params:uint64
+//go:generate ./schema.sh --response Empty    unused:uint64
+//go:generate ./schema.sh --response Result   result:Result
+//go:generate ./schema.sh --response Rows     rows:Rows
+//go:generate ./schema.sh --response Files    files:Files

vendor/github.com/canonical/go-dqlite/internal/protocol/schema.sh

+#!/bin/bash
+
+request_init() {
+    cat > request.go <<EOF
+package protocol
+
+// DO NOT EDIT
+//
+// This file was generated by ./schema.sh
+
+EOF
+}
+
+response_init() {
+    cat > response.go <<EOF
+package protocol
+
+// DO NOT EDIT
+//
+// This file was generated by ./schema.sh
+
+import "fmt"
+
+EOF
+}
+
+entity=$1
+shift
+
+cmd=$1
+shift
+
+if [ "$entity" = "--request" ]; then
+    if [ "$cmd" = "init" ]; then
+	request_init
+	exit
+    fi
+
+    args=""
+
+    for i in "${@}"
+    do
+	name=$(echo "$i" | cut -f 1 -d :)
+	type=$(echo "$i" | cut -f 2 -d :)
+
+	if [ "$name" = "unused" ]; then
+	    continue
+	fi
+
+	args=$(echo "${args}, ${name} ${type}")
+    done
+
+    cat >> request.go <<EOF
+
+// Encode${cmd} encodes a $cmd request.
+func Encode${cmd}(request *Message${args}) {
+EOF
+
+    for i in "${@}"
+    do
+	name=$(echo "$i" | cut -f 1 -d :)
+	type=$(echo "$i" | cut -f 2 -d :)
+
+	if [ "$name" = "unused" ]; then
+	    name=$(echo "0")
+	fi
+
+	cat >> request.go <<EOF
+	request.put${type^}(${name})
+EOF
+    done
+
+    cat >> request.go <<EOF
+
+	request.putHeader(Request${cmd})
+}
+EOF
+
+fi
+
+if [ "$entity" = "--response" ]; then
+    if [ "$cmd" = "init" ]; then
+	response_init
+	exit
+    fi
+
+    returns=""
+
+    for i in "${@}"
+    do
+	name=$(echo "$i" | cut -f 1 -d :)
+	type=$(echo "$i" | cut -f 2 -d :)
+
+	if [ "$name" = "unused" ]; then
+	    continue
+	fi
+
+	returns=$(echo "${returns}${name} ${type}, ")
+    done
+
+    cat >> response.go <<EOF
+
+// Decode${cmd} decodes a $cmd response.
+func Decode${cmd}(response *Message) (${returns}err error) {
+	mtype, _ := response.getHeader()
+
+	if mtype == ResponseFailure {
+		e := ErrRequest{}
+		e.Code = response.getUint64()
+		e.Description = response.getString()
+                err = e
+                return
+	}
+
+	if mtype != Response${cmd} {
+		err = fmt.Errorf("unexpected response type %d", mtype)
+                return
+	}
+
+EOF
+
+    for i in "${@}"
+    do
+	name=$(echo "$i" | cut -f 1 -d :)
+	type=$(echo "$i" | cut -f 2 -d :)
+
+	assign=$(echo "${name} = ")
+
+	if [ "$name" = "unused" ]; then
+	    assign=$(echo "")
+	fi
+
+	cat >> response.go <<EOF
+	${assign}response.get${type^}()
+EOF
+    done
+
+    cat >> response.go <<EOF
+
+	return
+}
+EOF
+
+fi

vendor/github.com/canonical/go-dqlite/internal/protocol/store.go

+package protocol
+
+import (
+	"context"
+)
+
+// NodeInfo holds information about a single server.
+type NodeInfo struct {
+	ID      uint64
+	Address string
+}
+
+// NodeStore is used by a dqlite client to get an initial list of candidate
+// dqlite servers that it can dial in order to find a leader server to connect
+// to.
+//
+// Once connected, the client periodically updates the server addresses in the
+// store by querying the leader about changes in the cluster (such as servers
+// being added or removed).
+type NodeStore interface {
+	// Get return the list of known servers.
+	Get(context.Context) ([]NodeInfo, error)
+
+	// Set updates the list of known cluster servers.
+	Set(context.Context, []NodeInfo) error
+}
+
+// InmemNodeStore keeps the list of servers in memory.
+type InmemNodeStore struct {
+	servers []NodeInfo
+}
+
+// NewInmemNodeStore creates NodeStore which stores its data in-memory.
+func NewInmemNodeStore() *InmemNodeStore {
+	return &InmemNodeStore{
+		servers: make([]NodeInfo, 0),
+	}
+}
+
+// Get the current servers.
+func (i *InmemNodeStore) Get(ctx context.Context) ([]NodeInfo, error) {
+	return i.servers, nil
+}
+
+// Set the servers.
+func (i *InmemNodeStore) Set(ctx context.Context, servers []NodeInfo) error {
+	i.servers = servers
+	return nil
+}

vendor/github.com/canonical/go-dqlite/node.go

+package dqlite
+
+import (
+	"time"
+
+	"github.com/canonical/go-dqlite/client"
+	"github.com/canonical/go-dqlite/internal/bindings"
+	"github.com/pkg/errors"
+)
+
+// Node runs a dqlite node.
+type Node struct {
+	log         client.LogFunc // Logger
+	server      *bindings.Node // Low-level C implementation
+	acceptCh    chan error     // Receives connection handling errors
+	id          uint64
+	address     string
+	bindAddress string
+}
+
+// NodeInfo is a convenience alias for client.NodeInfo.
+type NodeInfo = client.NodeInfo
+
+// Option can be used to tweak node parameters.
+type Option func(*options)
+
+// WithDialFunc sets a custom dial function for the server.
+func WithDialFunc(dial client.DialFunc) Option {
+	return func(options *options) {
+		options.DialFunc = dial
+	}
+}
+
+// WithBindAddress sets a custom bind address for the server.
+func WithBindAddress(address string) Option {
+	return func(options *options) {
+		options.BindAddress = address
+	}
+}
+
+// WithNetworkLatency sets the average one-way network latency.
+func WithNetworkLatency(latency time.Duration) Option {
+	return func(options *options) {
+		options.NetworkLatency = uint64(latency.Nanoseconds())
+	}
+}
+
+// New creates a new Node instance.
+func New(id uint64, address string, dir string, options ...Option) (*Node, error) {
+	o := defaultOptions()
+
+	for _, option := range options {
+		option(o)
+	}
+
+	server, err := bindings.NewNode(id, address, dir)
+	if err != nil {
+		return nil, err
+	}
+	if o.DialFunc != nil {
+		if err := server.SetDialFunc(o.DialFunc); err != nil {
+			return nil, err
+		}
+	}
+	if o.BindAddress != "" {
+		if err := server.SetBindAddress(o.BindAddress); err != nil {
+			return nil, err
+		}
+	}
+	if o.NetworkLatency != 0 {
+		if err := server.SetNetworkLatency(o.NetworkLatency); err != nil {
+			return nil, err
+		}
+	}
+	s := &Node{
+		server:      server,
+		acceptCh:    make(chan error, 1),
+		id:          id,
+		address:     address,
+		bindAddress: o.BindAddress,
+	}
+
+	return s, nil
+}
+
+// BindAddress returns the network address the node is listening to.
+func (s *Node) BindAddress() string {
+	return s.server.GetBindAddress()
+}
+
+// Start serving requests.
+func (s *Node) Start() error {
+	return s.server.Start()
+}
+
+// Recover a node by forcing a new cluster configuration.
+func (s *Node) Recover(cluster []NodeInfo) error {
+	return s.server.Recover(cluster)
+}
+
+// Hold configuration options for a dqlite server.
+type options struct {
+	Log            client.LogFunc
+	DialFunc       client.DialFunc
+	BindAddress    string
+	NetworkLatency uint64
+}
+
+// Close the server, releasing all resources it created.
+func (s *Node) Close() error {
+	// Send a stop signal to the dqlite event loop.
+	if err := s.server.Stop(); err != nil {
+		return errors.Wrap(err, "server failed to stop")
+	}
+
+	s.server.Close()
+
+	return nil
+}
+
+// Create a options object with sane defaults.
+func defaultOptions() *options {
+	return &options{
+		DialFunc: client.DefaultDialFunc,
+	}
+}

vendor/github.com/coreos/pkg/capnslog/init.go

@@ -32,7 +32,7 @@ import (
 func init() {
 	initHijack()
 
-	// Go `log` pacakge uses os.Stderr.
+	// Go `log` package uses os.Stderr.
 	SetFormatter(NewDefaultFormatter(os.Stderr))
 	SetGlobalLogLevel(INFO)
 }

vendor/github.com/flosch/pongo2/.gitattributes

+* text eol=lf

vendor/github.com/flosch/pongo2/.gitignore

+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+.idea
+.vscode
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+
+.project
+EBNF.txt
+test1.tpl
+pongo2_internal_test.go
+tpl-error.out
+/count.out
+/cover.out
+*.swp
+*.iml
+/cpu.out
+/mem.out
+/pongo2.test
+*.error
+/profile
+/coverage.out
+/pongo2_internal_test.ignore
+go.sum

vendor/github.com/flosch/pongo2/.travis.yml

+language: go
+os:
+  - linux
+  - osx
+go:
+  - 1.12
+script:
+  - go test -v

vendor/github.com/flosch/pongo2/AUTHORS

+Main author and maintainer of pongo2:
+
+* Florian Schlachter <flori@n-schlachter.de>
+
+Contributors (in no specific order):
+
+* @romanoaugusto88
+* @vitalbh
+* @blaubaer
+
+Feel free to add yourself to the list or to modify your entry if you did a contribution.

vendor/github.com/flosch/pongo2/LICENSE

+The MIT License (MIT)
+
+Copyright (c) 2013-2014 Florian Schlachter
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/flosch/pongo2/context.go

+package pongo2
+
+import (
+	"regexp"
+
+	"github.com/juju/errors"
+)
+
+var reIdentifiers = regexp.MustCompile("^[a-zA-Z0-9_]+$")
+
+var autoescape = true
+
+func SetAutoescape(newValue bool) {
+	autoescape = newValue
+}
+
+// A Context type provides constants, variables, instances or functions to a template.
+//
+// pongo2 automatically provides meta-information or functions through the "pongo2"-key.
+// Currently, context["pongo2"] contains the following keys:
+//  1. version: returns the version string
+//
+// Template examples for accessing items from your context:
+//     {{ myconstant }}
+//     {{ myfunc("test", 42) }}
+//     {{ user.name }}
+//     {{ pongo2.version }}
+type Context map[string]interface{}
+
+func (c Context) checkForValidIdentifiers() *Error {
+	for k, v := range c {
+		if !reIdentifiers.MatchString(k) {
+			return &Error{
+				Sender:    "checkForValidIdentifiers",
+				OrigError: errors.Errorf("context-key '%s' (value: '%+v') is not a valid identifier", k, v),
+			}
+		}
+	}
+	return nil
+}
+
+// Update updates this context with the key/value-pairs from another context.
+func (c Context) Update(other Context) Context {
+	for k, v := range other {
+		c[k] = v
+	}
+	return c
+}
+
+// ExecutionContext contains all data important for the current rendering state.
+//
+// If you're writing a custom tag, your tag's Execute()-function will
+// have access to the ExecutionContext. This struct stores anything
+// about the current rendering process's Context including
+// the Context provided by the user (field Public).
+// You can safely use the Private context to provide data to the user's
+// template (like a 'forloop'-information). The Shared-context is used
+// to share data between tags. All ExecutionContexts share this context.
+//
+// Please be careful when accessing the Public data.
+// PLEASE DO NOT MODIFY THE PUBLIC CONTEXT (read-only).
+//
+// To create your own execution context within tags, use the
+// NewChildExecutionContext(parent) function.
+type ExecutionContext struct {
+	template *Template
+
+	Autoescape bool
+	Public     Context
+	Private    Context
+	Shared     Context
+}
+
+var pongo2MetaContext = Context{
+	"version": Version,
+}
+
+func newExecutionContext(tpl *Template, ctx Context) *ExecutionContext {
+	privateCtx := make(Context)
+
+	// Make the pongo2-related funcs/vars available to the context
+	privateCtx["pongo2"] = pongo2MetaContext
+
+	return &ExecutionContext{
+		template: tpl,
+
+		Public:     ctx,
+		Private:    privateCtx,
+		Autoescape: autoescape,
+	}
+}
+
+func NewChildExecutionContext(parent *ExecutionContext) *ExecutionContext {
+	newctx := &ExecutionContext{
+		template: parent.template,
+
+		Public:     parent.Public,
+		Private:    make(Context),
+		Autoescape: parent.Autoescape,
+	}
+	newctx.Shared = parent.Shared
+
+	// Copy all existing private items
+	newctx.Private.Update(parent.Private)
+
+	return newctx
+}
+
+func (ctx *ExecutionContext) Error(msg string, token *Token) *Error {
+	return ctx.OrigError(errors.New(msg), token)
+}
+
+func (ctx *ExecutionContext) OrigError(err error, token *Token) *Error {
+	filename := ctx.template.name
+	var line, col int
+	if token != nil {
+		// No tokens available
+		// TODO: Add location (from where?)
+		filename = token.Filename
+		line = token.Line
+		col = token.Col
+	}
+	return &Error{
+		Template:  ctx.template,
+		Filename:  filename,
+		Line:      line,
+		Column:    col,
+		Token:     token,
+		Sender:    "execution",
+		OrigError: err,
+	}
+}
+
+func (ctx *ExecutionContext) Logf(format string, args ...interface{}) {
+	ctx.template.set.logf(format, args...)
+}

vendor/github.com/flosch/pongo2/doc.go

+// A Django-syntax like template-engine
+//
+// Blog posts about pongo2 (including introduction and migration):
+// https://www.florian-schlachter.de/?tag=pongo2
+//
+// Complete documentation on the template language:
+// https://docs.djangoproject.com/en/dev/topics/templates/
+//
+// Try out pongo2 live in the pongo2 playground:
+// https://www.florian-schlachter.de/pongo2/
+//
+// Make sure to read README.md in the repository as well.
+//
+// A tiny example with template strings:
+//
+// (Snippet on playground: https://www.florian-schlachter.de/pongo2/?id=1206546277)
+//
+//     // Compile the template first (i. e. creating the AST)
+//     tpl, err := pongo2.FromString("Hello {{ name|capfirst }}!")
+//     if err != nil {
+//         panic(err)
+//     }
+//     // Now you can render the template with the given
+//     // pongo2.Context how often you want to.
+//     out, err := tpl.Execute(pongo2.Context{"name": "fred"})
+//     if err != nil {
+//         panic(err)
+//     }
+//     fmt.Println(out) // Output: Hello Fred!
+//
+package pongo2

vendor/github.com/flosch/pongo2/error.go

+package pongo2
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+)
+
+// The Error type is being used to address an error during lexing, parsing or
+// execution. If you want to return an error object (for example in your own
+// tag or filter) fill this object with as much information as you have.
+// Make sure "Sender" is always given (if you're returning an error within
+// a filter, make Sender equals 'filter:yourfilter'; same goes for tags: 'tag:mytag').
+// It's okay if you only fill in ErrorMsg if you don't have any other details at hand.
+type Error struct {
+	Template  *Template
+	Filename  string
+	Line      int
+	Column    int
+	Token     *Token
+	Sender    string
+	OrigError error
+}
+
+func (e *Error) updateFromTokenIfNeeded(template *Template, t *Token) *Error {
+	if e.Template == nil {
+		e.Template = template
+	}
+
+	if e.Token == nil {
+		e.Token = t
+		if e.Line <= 0 {
+			e.Line = t.Line
+			e.Column = t.Col
+		}
+	}
+
+	return e
+}
+
+// Returns a nice formatted error string.
+func (e *Error) Error() string {
+	s := "[Error"
+	if e.Sender != "" {
+		s += " (where: " + e.Sender + ")"
+	}
+	if e.Filename != "" {
+		s += " in " + e.Filename
+	}
+	if e.Line > 0 {
+		s += fmt.Sprintf(" | Line %d Col %d", e.Line, e.Column)
+		if e.Token != nil {
+			s += fmt.Sprintf(" near '%s'", e.Token.Val)
+		}
+	}
+	s += "] "
+	s += e.OrigError.Error()
+	return s
+}
+
+// RawLine returns the affected line from the original template, if available.
+func (e *Error) RawLine() (line string, available bool, outErr error) {
+	if e.Line <= 0 || e.Filename == "<string>" {
+		return "", false, nil
+	}
+
+	filename := e.Filename
+	if e.Template != nil {
+		filename = e.Template.set.resolveFilename(e.Template, e.Filename)
+	}
+	file, err := os.Open(filename)
+	if err != nil {
+		return "", false, err
+	}
+	defer func() {
+		err := file.Close()
+		if err != nil && outErr == nil {
+			outErr = err
+		}
+	}()
+
+	scanner := bufio.NewScanner(file)
+	l := 0
+	for scanner.Scan() {
+		l++
+		if l == e.Line {
+			return scanner.Text(), true, nil
+		}
+	}
+	return "", false, nil
+}

vendor/github.com/flosch/pongo2/filters.go

+package pongo2
+
+import (
+	"fmt"
+
+	"github.com/juju/errors"
+)
+
+// FilterFunction is the type filter functions must fulfil
+type FilterFunction func(in *Value, param *Value) (out *Value, err *Error)
+
+var filters map[string]FilterFunction
+
+func init() {
+	filters = make(map[string]FilterFunction)
+}
+
+// FilterExists returns true if the given filter is already registered
+func FilterExists(name string) bool {
+	_, existing := filters[name]
+	return existing
+}
+
+// RegisterFilter registers a new filter. If there's already a filter with the same
+// name, RegisterFilter will panic. You usually want to call this
+// function in the filter's init() function:
+// http://golang.org/doc/effective_go.html#init
+//
+// See http://www.florian-schlachter.de/post/pongo2/ for more about
+// writing filters and tags.
+func RegisterFilter(name string, fn FilterFunction) error {
+	if FilterExists(name) {
+		return errors.Errorf("filter with name '%s' is already registered", name)
+	}
+	filters[name] = fn
+	return nil
+}
+
+// ReplaceFilter replaces an already registered filter with a new implementation. Use this
+// function with caution since it allows you to change existing filter behaviour.
+func ReplaceFilter(name string, fn FilterFunction) error {
+	if !FilterExists(name) {
+		return errors.Errorf("filter with name '%s' does not exist (therefore cannot be overridden)", name)
+	}
+	filters[name] = fn
+	return nil
+}
+
+// MustApplyFilter behaves like ApplyFilter, but panics on an error.
+func MustApplyFilter(name string, value *Value, param *Value) *Value {
+	val, err := ApplyFilter(name, value, param)
+	if err != nil {
+		panic(err)
+	}
+	return val
+}
+
+// ApplyFilter applies a filter to a given value using the given parameters.
+// Returns a *pongo2.Value or an error.
+func ApplyFilter(name string, value *Value, param *Value) (*Value, *Error) {
+	fn, existing := filters[name]
+	if !existing {
+		return nil, &Error{
+			Sender:    "applyfilter",
+			OrigError: errors.Errorf("Filter with name '%s' not found.", name),
+		}
+	}
+
+	// Make sure param is a *Value
+	if param == nil {
+		param = AsValue(nil)
+	}
+
+	return fn(value, param)
+}
+
+type filterCall struct {
+	token *Token
+
+	name      string
+	parameter IEvaluator
+
+	filterFunc FilterFunction
+}
+
+func (fc *filterCall) Execute(v *Value, ctx *ExecutionContext) (*Value, *Error) {
+	var param *Value
+	var err *Error
+
+	if fc.parameter != nil {
+		param, err = fc.parameter.Evaluate(ctx)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		param = AsValue(nil)
+	}
+
+	filteredValue, err := fc.filterFunc(v, param)
+	if err != nil {
+		return nil, err.updateFromTokenIfNeeded(ctx.template, fc.token)
+	}
+	return filteredValue, nil
+}
+
+// Filter = IDENT | IDENT ":" FilterArg | IDENT "|" Filter
+func (p *Parser) parseFilter() (*filterCall, *Error) {
+	identToken := p.MatchType(TokenIdentifier)
+
+	// Check filter ident
+	if identToken == nil {
+		return nil, p.Error("Filter name must be an identifier.", nil)
+	}
+
+	filter := &filterCall{
+		token: identToken,
+		name:  identToken.Val,
+	}
+
+	// Get the appropriate filter function and bind it
+	filterFn, exists := filters[identToken.Val]
+	if !exists {
+		return nil, p.Error(fmt.Sprintf("Filter '%s' does not exist.", identToken.Val), identToken)
+	}
+
+	filter.filterFunc = filterFn
+
+	// Check for filter-argument (2 tokens needed: ':' ARG)
+	if p.Match(TokenSymbol, ":") != nil {
+		if p.Peek(TokenSymbol, "}}") != nil {
+			return nil, p.Error("Filter parameter required after ':'.", nil)
+		}
+
+		// Get filter argument expression
+		v, err := p.parseVariableOrLiteral()
+		if err != nil {
+			return nil, err
+		}
+		filter.parameter = v
+	}
+
+	return filter, nil
+}