Skip to content

K
k3s
Unverified Commit 1a964eff authored by Brian Downs's avatar Brian Downs Committed by GitHub
Browse files
Options

Merge pull request #3509 from briandowns/backport_issue-3414_release_1.20

[Backport 1.20] - fix possible race where bootstrap data might not save
parents 3e376093 a36c460b
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 238 additions and 95 deletions
cmd/etcdsnapshot/main.go
View file @ 1a964eff
......@@ -13,7 +13,7 @@ import (
func main() {
app := cmds.NewApp()
app.Commands = []cli.Command{
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
}
if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {
......
cmd/k3s/main.go
View file @ 1a964eff
......@@ -33,6 +33,8 @@ func main() {
return
}
etcdsnapshotCommand := internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)
// Handle subcommand invocation (k3s server, k3s crictl, etc)
app := cmds.NewApp()
app.Commands = []cli.Command{
......@@ -42,7 +44,7 @@ func main() {
cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
cmds.NewEtcdSnapshotCommand(internalCLIAction(version.Program+"-"+cmds.EtcdSnapshotCommand, dataDir, os.Args)),
cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand, cmds.NewEtcdSnapshotSubcommands(etcdsnapshotCommand)),
}
if err := app.Run(os.Args); err != nil {
......
cmd/server/main.go
View file @ 1a964eff
......@@ -43,7 +43,7 @@ func main() {
cmds.NewKubectlCommand(kubectl.Run),
cmds.NewCRICTL(crictl.Run),
cmds.NewCtrCommand(ctr.Run),
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
}
err := app.Run(configfilearg.MustParse(os.Args))
......
main.go
View file @ 1a964eff
......@@ -27,7 +27,7 @@ func main() {
cmds.NewAgentCommand(agent.Run),
cmds.NewKubectlCommand(kubectl.Run),
cmds.NewCRICTL(crictl.Run),
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run),
cmds.NewEtcdSnapshotCommand(etcdsnapshot.Run, cmds.NewEtcdSnapshotSubcommands(etcdsnapshot.Delete)),
}
if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {
......
pkg/cli/cmds/etcd_snapshot.go
View file @ 1a964eff
......@@ -7,82 +7,103 @@ import (
const EtcdSnapshotCommand = "etcd-snapshot"
func NewEtcdSnapshotCommand(action func(*cli.Context) error) cli.Command {
var EtcdSnapshotFlags = []cli.Flag{
DebugFlag,
LogFile,
AlsoLogToStderr,
cli.StringFlag{
Name: "node-name",
Usage: "(agent/node) Node name",
EnvVar: version.ProgramUpper + "_NODE_NAME",
Destination: &AgentConfig.NodeName,
},
cli.StringFlag{
Name: "data-dir,d",
Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
Destination: &ServerConfig.DataDir,
},
&cli.StringFlag{
Name: "name",
Usage: "(db) Set the base name of the etcd on-demand snapshot (appended with UNIX timestamp).",
Destination: &ServerConfig.EtcdSnapshotName,
Value: "on-demand",
},
&cli.BoolFlag{
Name: "s3",
Usage: "(db) Enable backup to S3",
Destination: &ServerConfig.EtcdS3,
},
&cli.StringFlag{
Name: "s3-endpoint",
Usage: "(db) S3 endpoint url",
Destination: &ServerConfig.EtcdS3Endpoint,
Value: "s3.amazonaws.com",
},
&cli.StringFlag{
Name: "s3-endpoint-ca",
Usage: "(db) S3 custom CA cert to connect to S3 endpoint",
Destination: &ServerConfig.EtcdS3EndpointCA,
},
&cli.BoolFlag{
Name: "s3-skip-ssl-verify",
Usage: "(db) Disables S3 SSL certificate validation",
Destination: &ServerConfig.EtcdS3SkipSSLVerify,
},
&cli.StringFlag{
Name: "s3-access-key",
Usage: "(db) S3 access key",
EnvVar: "AWS_ACCESS_KEY_ID",
Destination: &ServerConfig.EtcdS3AccessKey,
},
&cli.StringFlag{
Name: "s3-secret-key",
Usage: "(db) S3 secret key",
EnvVar: "AWS_SECRET_ACCESS_KEY",
Destination: &ServerConfig.EtcdS3SecretKey,
},
&cli.StringFlag{
Name: "s3-bucket",
Usage: "(db) S3 bucket name",
Destination: &ServerConfig.EtcdS3BucketName,
},
&cli.StringFlag{
Name: "s3-region",
Usage: "(db) S3 region / bucket location (optional)",
Destination: &ServerConfig.EtcdS3Region,
Value: "us-east-1",
},
&cli.StringFlag{
Name: "s3-folder",
Usage: "(db) S3 folder",
Destination: &ServerConfig.EtcdS3Folder,
},
}
func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command {
return cli.Command{
Name: EtcdSnapshotCommand,
Usage: "Trigger an immediate etcd snapshot",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: action,
Flags: []cli.Flag{
DebugFlag,
LogFile,
AlsoLogToStderr,
cli.StringFlag{
Name: "data-dir,d",
Usage: "(data) Folder to hold state default /var/lib/rancher/" + version.Program + " or ${HOME}/.rancher/" + version.Program + " if not root",
Destination: &ServerConfig.DataDir,
},
&cli.StringFlag{
Name: "name",
Usage: "(db) Set the base name of the etcd on-demand snapshot (appended with UNIX timestamp).",
Destination: &ServerConfig.EtcdSnapshotName,
Value: "on-demand",
},
&cli.StringFlag{
Name: "dir",
Usage: "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
Destination: &ServerConfig.EtcdSnapshotDir,
},
&cli.BoolFlag{
Name: "s3",
Usage: "(db) Enable backup to S3",
Destination: &ServerConfig.EtcdS3,
},
&cli.StringFlag{
Name: "s3-endpoint",
Usage: "(db) S3 endpoint url",
Destination: &ServerConfig.EtcdS3Endpoint,
Value: "s3.amazonaws.com",
},
&cli.StringFlag{
Name: "s3-endpoint-ca",
Usage: "(db) S3 custom CA cert to connect to S3 endpoint",
Destination: &ServerConfig.EtcdS3EndpointCA,
},
&cli.BoolFlag{
Name: "s3-skip-ssl-verify",
Usage: "(db) Disables S3 SSL certificate validation",
Destination: &ServerConfig.EtcdS3SkipSSLVerify,
},
&cli.StringFlag{
Name: "s3-access-key",
Usage: "(db) S3 access key",
EnvVar: "AWS_ACCESS_KEY_ID",
Destination: &ServerConfig.EtcdS3AccessKey,
},
&cli.StringFlag{
Name: "s3-secret-key",
Usage: "(db) S3 secret key",
EnvVar: "AWS_SECRET_ACCESS_KEY",
Destination: &ServerConfig.EtcdS3SecretKey,
},
&cli.StringFlag{
Name: "s3-bucket",
Usage: "(db) S3 bucket name",
Destination: &ServerConfig.EtcdS3BucketName,
},
&cli.StringFlag{
Name: "s3-region",
Usage: "(db) S3 region / bucket location (optional)",
Destination: &ServerConfig.EtcdS3Region,
Value: "us-east-1",
},
&cli.StringFlag{
Name: "s3-folder",
Usage: "(db) S3 folder",
Destination: &ServerConfig.EtcdS3Folder,
},
Subcommands: subcommands,
Flags: append(EtcdSnapshotFlags, &cli.StringFlag{
Name: "dir",
Usage: "(db) Directory to save etcd on-demand snapshot. (default: ${data-dir}/db/snapshots)",
Destination: &ServerConfig.EtcdSnapshotDir,
}),
}
}
func NewEtcdSnapshotSubcommands(delete func(ctx *cli.Context) error) []cli.Command {
return []cli.Command{
{
Name: "delete",
Usage: "Delete given snapshot(s)",
SkipFlagParsing: false,
SkipArgReorder: true,
Action: delete,
Flags: EtcdSnapshotFlags,
},
}
}
pkg/cli/cmds/root.go
View file @ 1a964eff
package cmds
import (
"errors"
"fmt"
"os"
"runtime"
......@@ -20,6 +21,8 @@ var (
}
)
var ErrCommandNoArgs = errors.New("this command does not take any arguments")
func init() {
// hack - force "file,dns" lookup order if go dns is used
if os.Getenv("RES_OPTIONS") == "" {
......
pkg/cli/etcdsnapshot/etcd_snapshot.go
View file @ 1a964eff
......@@ -16,6 +16,25 @@ import (
"github.com/urfave/cli"
)
// commandSetup setups up common things needed
// for each etcd command.
func commandSetup(app *cli.Context, cfg *cmds.Server) (string, error) {
gspt.SetProcTitle(os.Args[0])
nodeName := app.String("node-name")
if nodeName == "" {
h, err := os.Hostname()
if err != nil {
return "", err
}
nodeName = h
}
os.Setenv("NODE_NAME", nodeName)
return server.ResolveDataDir(cfg.DataDir)
}
func Run(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
......@@ -24,13 +43,15 @@ func Run(app *cli.Context) error {
}
func run(app *cli.Context, cfg *cmds.Server) error {
gspt.SetProcTitle(os.Args[0])
dataDir, err := server.ResolveDataDir(cfg.DataDir)
dataDir, err := commandSetup(app, cfg)
if err != nil {
return err
}
if len(app.Args()) > 0 {
return cmds.ErrCommandNoArgs
}
var serverConfig server.Config
serverConfig.DisableAgent = true
serverConfig.ControlConfig.DataDir = dataDir
......@@ -50,10 +71,13 @@ func run(app *cli.Context, cfg *cmds.Server) error {
serverConfig.ControlConfig.Runtime.ETCDServerCA = filepath.Join(dataDir, "tls", "etcd", "server-ca.crt")
serverConfig.ControlConfig.Runtime.ClientETCDCert = filepath.Join(dataDir, "tls", "etcd", "client.crt")
serverConfig.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
ctx := signals.SetupSignalHandler(context.Background())
e := etcd.NewETCD()
e.SetControlConfig(&serverConfig.ControlConfig)
initialized, err := etcd.NewETCD().IsInitialized(ctx, &serverConfig.ControlConfig)
initialized, err := e.IsInitialized(ctx, &serverConfig.ControlConfig)
if err != nil {
return err
}
......@@ -67,5 +91,59 @@ func run(app *cli.Context, cfg *cmds.Server) error {
return err
}
sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return err
}
serverConfig.ControlConfig.Runtime.Core = sc.Core
return cluster.Snapshot(ctx, &serverConfig.ControlConfig)
}
func Delete(app *cli.Context) error {
if err := cmds.InitLogging(); err != nil {
return err
}
return delete(app, &cmds.ServerConfig)
}
func delete(app *cli.Context, cfg *cmds.Server) error {
dataDir, err := commandSetup(app, cfg)
if err != nil {
return err
}
snapshots := app.Args()
if len(snapshots) == 0 {
return errors.New("no snapshots given for removal")
}
var serverConfig server.Config
serverConfig.DisableAgent = true
serverConfig.ControlConfig.DataDir = dataDir
serverConfig.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
serverConfig.ControlConfig.EtcdS3Endpoint = cfg.EtcdS3Endpoint
serverConfig.ControlConfig.EtcdS3EndpointCA = cfg.EtcdS3EndpointCA
serverConfig.ControlConfig.EtcdS3SkipSSLVerify = cfg.EtcdS3SkipSSLVerify
serverConfig.ControlConfig.EtcdS3AccessKey = cfg.EtcdS3AccessKey
serverConfig.ControlConfig.EtcdS3SecretKey = cfg.EtcdS3SecretKey
serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName
serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region
serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder
serverConfig.ControlConfig.Runtime = &config.ControlRuntime{}
serverConfig.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
ctx := signals.SetupSignalHandler(context.Background())
e := etcd.NewETCD()
e.SetControlConfig(&serverConfig.ControlConfig)
sc, err := server.NewContext(ctx, serverConfig.ControlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return err
}
serverConfig.ControlConfig.Runtime.Core = sc.Core
return e.DeleteSnapshots(ctx, app.Args())
}
pkg/cli/server/server.go
View file @ 1a964eff
......@@ -131,6 +131,10 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
serverConfig.ControlConfig.EtcdSnapshotCron = cfg.EtcdSnapshotCron
serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
if cfg.EtcdDisableSnapshots {
logrus.Info("ETCD snapshots are disabled")
}
serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics
serverConfig.ControlConfig.EtcdS3 = cfg.EtcdS3
......
pkg/cluster/cluster.go
View file @ 1a964eff
......@@ -3,6 +3,7 @@ package cluster
import (
"context"
"net/url"
"runtime"
"strings"
"github.com/k3s-io/kine/pkg/endpoint"
......@@ -102,9 +103,20 @@ func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
// snapshots will be empty.
if c.managedDB != nil {
go func() {
for range ready {
if err := c.save(ctx); err != nil {
panic(err)
for {
select {
case <-ready:
if err := c.save(ctx); err != nil {
panic(err)
}
if err := c.managedDB.StoreSnapshotData(ctx); err != nil {
logrus.Errorf("Failed to record snapshots for cluster: %v", err)
}
return
default:
runtime.Gosched()
}
}
}()
......
pkg/cluster/managed/drivers.go
View file @ 1a964eff
......@@ -21,9 +21,10 @@ type Driver interface {
Test(ctx context.Context) error
Restore(ctx context.Context) error
EndpointName() string
Snapshot(ctx context.Context, config *config.Control) error
StoreSnapshotData(ctx context.Context) error
GetMembersClientURLs(ctx context.Context) ([]string, error)
RemoveSelf(ctx context.Context) error
Snapshot(ctx context.Context, config *config.Control) error
}
func RegisterDriver(d Driver) {
......
pkg/etcd/s3.go
View file @ 1a964eff
......@@ -14,6 +14,7 @@ import (
"path/filepath"
"sort"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
......@@ -22,6 +23,8 @@ import (
"github.com/sirupsen/logrus"
)
const defaultS3OpTimeout = time.Second * 30
// s3 maintains state for S3 functionality.
type s3 struct {
config *config.Control
......@@ -61,6 +64,10 @@ func newS3(ctx context.Context, config *config.Control) (*s3, error) {
}
logrus.Infof("Checking if S3 bucket %s exists", config.EtcdS3BucketName)
ctx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
defer cancel()
exists, err := c.BucketExists(ctx, config.EtcdS3BucketName)
if err != nil {
return nil, err
......@@ -87,11 +94,13 @@ func (s *s3) upload(ctx context.Context, snapshot string) error {
snapshotFileName = basename
}
toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
defer cancel()
opts := minio.PutObjectOptions{
ContentType: "application/zip",
NumThreads: 2,
}
if _, err := s.client.FPutObject(ctx, s.config.EtcdS3BucketName, snapshotFileName, snapshot, opts); err != nil {
if _, err := s.client.FPutObject(toCtx, s.config.EtcdS3BucketName, snapshotFileName, snapshot, opts); err != nil {
logrus.Errorf("Error received in attempt to upload snapshot to S3: %s", err)
}
......@@ -109,7 +118,10 @@ func (s *s3) download(ctx context.Context) error {
}
logrus.Debugf("retrieving snapshot: %s", remotePath)
r, err := s.client.GetObject(ctx, s.config.EtcdS3BucketName, remotePath, minio.GetObjectOptions{})
toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
defer cancel()
r, err := s.client.GetObject(toCtx, s.config.EtcdS3BucketName, remotePath, minio.GetObjectOptions{})
if err != nil {
return nil
}
......@@ -141,23 +153,33 @@ func (s *s3) download(ctx context.Context) error {
return os.Chmod(fullSnapshotPath, 0600)
}
// snapshotPrefix returns the prefix used in the
// naming of the snapshots.
func (s *s3) snapshotPrefix() string {
nodeName := os.Getenv("NODE_NAME")
fullSnapshotPrefix := snapshotPrefix + nodeName
var prefix string
if s.config.EtcdS3Folder != "" {
prefix = filepath.Join(s.config.EtcdS3Folder, fullSnapshotPrefix)
} else {
prefix = fullSnapshotPrefix
}
return prefix
}
// snapshotRetention deletes the given snapshot from the configured S3
// compatible backend.
func (s *s3) snapshotRetention(ctx context.Context) error {
var snapshotFiles []minio.ObjectInfo
var prefix string
if s.config.EtcdS3Folder != "" {
prefix = filepath.Join(s.config.EtcdS3Folder, snapshotPrefix)
} else {
prefix = snapshotPrefix
}
toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout)
defer cancel()
loo := minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
Prefix: s.snapshotPrefix(),
}
for info := range s.client.ListObjects(ctx, s.config.EtcdS3BucketName, loo) {
for info := range s.client.ListObjects(toCtx, s.config.EtcdS3BucketName, loo) {
if info.Err != nil {
return info.Err
}
......
pkg/server/context.go
View file @ 1a964eff
......@@ -32,7 +32,7 @@ func (c *Context) Start(ctx context.Context) error {
return start.All(ctx, 5, c.K3s, c.Helm, c.Apps, c.Auth, c.Batch, c.Core)
}
func newContext(ctx context.Context, cfg string) (*Context, error) {
func NewContext(ctx context.Context, cfg string) (*Context, error) {
restConfig, err := clientcmd.BuildConfigFromFlags("", cfg)
if err != nil {
return nil, err
......
pkg/server/etcd.go
View file @ 1a964eff
......@@ -22,7 +22,7 @@ func setETCDLabelsAndAnnotations(ctx context.Context, config *Config) error {
for range t.C {
controlConfig := &config.ControlConfig
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
logrus.Infof("Failed to set etcd role label: %v", err)
continue
......
pkg/server/server.go
View file @ 1a964eff
......@@ -107,7 +107,7 @@ func startOnAPIServerReady(ctx context.Context, config *Config) {
func runControllers(ctx context.Context, config *Config) error {
controlConfig := &config.ControlConfig
sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
sc, err := NewContext(ctx, controlConfig.Runtime.KubeConfigAdmin)
if err != nil {
return err
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment