Dynamic Kubelet Configuration

cmd/hyperkube/kubelet.go

@@ -23,8 +23,12 @@ import (
 
 // NewKubelet creates a new hyperkube Server object that includes the
 // description and flags.
-func NewKubelet() *Server {
-	s := options.NewKubeletServer()
+func NewKubelet() (*Server, error) {
+	s, err := options.NewKubeletServer()
+	if err != nil {
+		return nil, err
+	}
+
 	hks := Server{
 		name:        "kubelet",
 		SimpleUsage: "kubelet",
@@ -39,5 +43,5 @@ func NewKubelet() *Server {
 		},
 	}
 	s.AddFlags(hks.Flags())
-	return &hks
+	return &hks, nil
 }

cmd/hyperkube/main.go

@@ -20,6 +20,7 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"os"
 
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
@@ -36,7 +37,12 @@ func main() {
 	hk.AddServer(NewKubeAPIServer())
 	hk.AddServer(NewKubeControllerManager())
 	hk.AddServer(NewScheduler())
-	hk.AddServer(NewKubelet())
+	if kubelet, err := NewKubelet(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	} else {
+		hk.AddServer(kubelet)
+	}
 	hk.AddServer(NewKubeProxy())
 	hk.AddServer(NewKubeAggregator())
 

cmd/kubelet/BUILD

@@ -23,10 +23,14 @@ go_library(
     deps = [
         "//cmd/kubelet/app:go_default_library",
         "//cmd/kubelet/app/options:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/version/prometheus:go_default_library",
         "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

cmd/kubelet/app/BUILD

@@ -52,6 +52,7 @@ go_library(
         "//pkg/kubelet/dockershim/remote:go_default_library",
         "//pkg/kubelet/eviction:go_default_library",
         "//pkg/kubelet/eviction/api:go_default_library",
+        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/network/cni:go_default_library",
         "//pkg/kubelet/network/kubenet:go_default_library",
@@ -100,8 +101,6 @@ go_library(
         "//vendor/golang.org/x/exp/inotify:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",

cmd/kubelet/app/options/BUILD

@@ -19,6 +19,8 @@ go_library(
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/apis/componentconfig/install:go_default_library",
         "//pkg/apis/componentconfig/v1alpha1:go_default_library",
+        "//pkg/apis/componentconfig/validation:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/util/taints:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",

cmd/kubelet/kubelet.go

@@ -24,36 +24,88 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/spf13/pflag"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kubelet/app"
 	"k8s.io/kubernetes/cmd/kubelet/app/options"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
-	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
+	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
 	"k8s.io/kubernetes/pkg/version/verflag"
-
-	"github.com/spf13/pflag"
 )
 
+func die(err error) {
+	fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	os.Exit(1)
+}
+
 func main() {
-	s := options.NewKubeletServer()
-	s.AddFlags(pflag.CommandLine)
+	// construct KubeletFlags object and register command line flags mapping
+	kubeletFlags := options.NewKubeletFlags()
+	kubeletFlags.AddFlags(pflag.CommandLine)
+
+	// construct KubeletConfiguration object and register command line flags mapping
+	defaultConfig, err := options.NewKubeletConfiguration()
+	if err != nil {
+		die(err)
+	}
+	options.AddKubeletConfigFlags(pflag.CommandLine, defaultConfig)
 
+	// parse the command line flags into the respective objects
 	flag.InitFlags()
+
+	// initialize logging and defer flush
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
+	// short-circuit on verflag
 	verflag.PrintAndExitIfRequested()
 
-	if s.ExperimentalDockershim {
-		if err := app.RunDockershim(&s.KubeletConfiguration, &s.ContainerRuntimeOptions); err != nil {
-			fmt.Fprintf(os.Stderr, "error: %v\n", err)
-			os.Exit(1)
+	// TODO(mtaufen): won't need this this once dynamic config is GA
+	// set feature gates so we can check if dynamic config is enabled
+	if err := utilfeature.DefaultFeatureGate.Set(defaultConfig.FeatureGates); err != nil {
+		die(err)
+	}
+	// validate the initial KubeletFlags, to make sure the dynamic-config-related flags aren't used unless the feature gate is on
+	if err := options.ValidateKubeletFlags(kubeletFlags); err != nil {
+		die(err)
+	}
+	// if dynamic kubelet config is enabled, bootstrap the kubelet config controller
+	var kubeletConfig *componentconfig.KubeletConfiguration
+	var kubeletConfigController *kubeletconfig.Controller
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		var err error
+		kubeletConfig, kubeletConfigController, err = app.BootstrapKubeletConfigController(kubeletFlags, defaultConfig)
+		if err != nil {
+			die(err)
+		}
+	} else if kubeletConfig == nil {
+		kubeletConfig = defaultConfig
+	}
+
+	// construct a KubeletServer from kubeletFlags and kubeletConfig
+	kubeletServer := &options.KubeletServer{KubeletFlags: *kubeletFlags, KubeletConfiguration: *kubeletConfig}
+
+	// use kubeletServer to construct the default KubeletDeps
+	kubeletDeps, err := app.UnsecuredDependencies(kubeletServer)
+
+	// add the kubelet config controller to kubeletDeps
+	kubeletDeps.KubeletConfigController = kubeletConfigController
+
+	// start the experimental docker shim, if enabled
+	if kubeletFlags.ExperimentalDockershim {
+		if err := app.RunDockershim(kubeletConfig, &kubeletFlags.ContainerRuntimeOptions); err != nil {
+			die(err)
 		}
 	}
 
-	if err := app.Run(s, nil); err != nil {
-		fmt.Fprintf(os.Stderr, "error: %v\n", err)
-		os.Exit(1)
+	// run the kubelet
+	if err := app.Run(kubeletServer, kubeletDeps); err != nil {
+		die(err)
 	}
 }

hack/.golint_failures

@@ -793,7 +793,6 @@ staging/src/k8s.io/sample-apiserver/pkg/apiserver
 staging/src/k8s.io/sample-apiserver/pkg/client/informers_generated/externalversions/internalinterfaces
 staging/src/k8s.io/sample-apiserver/pkg/client/informers_generated/internalversion/internalinterfaces
 staging/src/k8s.io/sample-apiserver/pkg/cmd/server
-staging/src/k8s.io/sample-apiserver/pkg/registry
 staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/fischer
 staging/src/k8s.io/sample-apiserver/pkg/registry/wardle/flunder
 test/e2e

hack/verify-flags/known-flags.txt

@@ -193,6 +193,7 @@ drop-embedded-fields
 dry-run
 dump-logs-on-failure
 duration-sec
+dynamic-config-dir
 e2e-output-dir
 e2e-verify-service-account
 enable-aggregator-routing
@@ -347,6 +348,7 @@ image-service-endpoint
 included-types-overrides
 include-extended-apis
 include-extended-apis
+init-config-dir
 initial-sync-timeout
 input-base
 input-dirs
@@ -490,6 +492,7 @@ network-plugin
 network-plugin-dir
 network-plugin-mtu
 node-cidr-mask-size
+node-config-dir
 node-eviction-rate
 node-instance-group
 node-ip

pkg/api/register.go

@@ -85,6 +85,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&ServiceProxyOptions{},
 		&NodeList{},
 		&Node{},
+		&NodeConfigSource{},
 		&NodeProxyOptions{},
 		&Endpoints{},
 		&EndpointsList{},

pkg/api/types.go

@@ -2913,6 +2913,19 @@ type NodeSpec struct {
 	// If specified, the node's taints.
 	// +optional
 	Taints []Taint
+
+	// If specified, the source to get node configuration from
+	// The DynamicKubeletConfig feature gate must be enabled for the Kubelet to use this field
+	// +optional
+	ConfigSource *NodeConfigSource
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// NodeConfigSource specifies a source of node configuration. Exactly one subfield must be non-nil.
+type NodeConfigSource struct {
+	metav1.TypeMeta
+	ConfigMapRef *ObjectReference
 }
 
 // DaemonEndpoint contains information about a single Daemon endpoint.

pkg/api/validation/validation.go

@@ -3338,6 +3338,11 @@ func ValidateNode(node *api.Node) field.ErrorList {
 		allErrs = append(allErrs, field.Required(field.NewPath("spec", "externalID"), ""))
 	}
 
+	// Only allow Node.Spec.ConfigSource to be set if the DynamicKubeletConfig feature gate is enabled
+	if node.Spec.ConfigSource != nil && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "configSource"), "configSource may only be set if the DynamicKubeletConfig feature gate is enabled)"))
+	}
+
 	// TODO(rjnagal): Ignore PodCIDR till its completely implemented.
 	return allErrs
 }
@@ -3365,7 +3370,7 @@ func ValidateNodeUpdate(node, oldNode *api.Node) field.ErrorList {
 		allErrs = append(allErrs, ValidateResourceQuantityValue(string(k), v, resPath)...)
 	}
 
-	// Validte no duplicate addresses in node status.
+	// Validate no duplicate addresses in node status.
 	addresses := make(map[api.NodeAddress]bool)
 	for i, address := range node.Status.Addresses {
 		if _, ok := addresses[address]; ok {
@@ -3398,10 +3403,16 @@ func ValidateNodeUpdate(node, oldNode *api.Node) field.ErrorList {
 	}
 	oldNode.Spec.Taints = node.Spec.Taints
 
+	// Allow updates to Node.Spec.ConfigSource if DynamicKubeletConfig feature gate is enabled
+	if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
+		oldNode.Spec.ConfigSource = node.Spec.ConfigSource
+	}
+
+	// We made allowed changes to oldNode, and now we compare oldNode to node. Any remaining differences indicate changes to protected fields.
 	// TODO: Add a 'real' error type for this error and provide print actual diffs.
 	if !apiequality.Semantic.DeepEqual(oldNode, node) {
 		glog.V(4).Infof("Update failed validation %#v vs %#v", oldNode, node)
-		allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels, taints or capacity"))
+		allErrs = append(allErrs, field.Forbidden(field.NewPath(""), "node updates may only change labels, taints, or capacity (or configSource, if the DynamicKubeletConfig feature gate is enabled)"))
 	}
 
 	return allErrs

pkg/apis/componentconfig/BUILD

@@ -50,6 +50,7 @@ filegroup(
         ":package-srcs",
         "//pkg/apis/componentconfig/install:all-srcs",
         "//pkg/apis/componentconfig/v1alpha1:all-srcs",
+        "//pkg/apis/componentconfig/validation:all-srcs",
     ],
     tags = ["automanaged"],
 )

pkg/apis/componentconfig/types.go

@@ -180,6 +180,18 @@ const (
 type KubeletConfiguration struct {
 	metav1.TypeMeta
 
+	// Only used for dynamic configuration.
+	// The length of the trial period for this configuration. If the Kubelet records CrashLoopThreshold or
+	// more startups during this period, the current configuration will be marked bad and the
+	// Kubelet will roll-back to the last-known-good. Default 10 minutes.
+	ConfigTrialDuration metav1.Duration
+	// Only used for dynamic configuration.
+	// If this number of Kubelet "crashes" during ConfigTrialDuration meets this threshold,
+	// the configuration fails the trial and the Kubelet rolls back to its last-known-good config.
+	// Crash-loops are detected by counting Kubelet startups, so one startup is implicitly added
+	// to this threshold to always allow a single restart per config change.
+	// Default 10, mimimum allowed is 0, maximum allowed is 10.
+	CrashLoopThreshold int32
 	// podManifestPath is the path to the directory containing pod manifests to
 	// run, or the path to a single manifest file
 	PodManifestPath string
@@ -215,17 +227,10 @@ type KubeletConfiguration struct {
 	// tlsPrivateKeyFile is the ile containing x509 private key matching
 	// tlsCertFile.
 	TLSPrivateKeyFile string
-	// certDirectory is the directory where the TLS certs are located (by
-	// default /var/run/kubernetes). If tlsCertFile and tlsPrivateKeyFile
-	// are provided, this flag will be ignored.
-	CertDirectory string
 	// authentication specifies how requests to the Kubelet's server are authenticated
 	Authentication KubeletAuthentication
 	// authorization specifies how requests to the Kubelet's server are authorized
 	Authorization KubeletAuthorization
-	// rootDirectory is the directory path to place kubelet files (volume
-	// mounts,etc).
-	RootDirectory string
 	// seccompProfileRoot is the directory path for seccomp profiles.
 	SeccompProfileRoot string
 	// allowPrivileged enables containers to request privileged mode.
@@ -314,12 +319,6 @@ type KubeletConfiguration struct {
 	// volumePluginDir is the full path of the directory in which to search
 	// for additional third party volume plugins
 	VolumePluginDir string
-	// cloudProvider is the provider for cloud services.
-	// +optional
-	CloudProvider string
-	// cloudConfigFile is the path to the cloud provider configuration file.
-	// +optional
-	CloudConfigFile string
 	// KubeletCgroups is the absolute name of cgroups to isolate the kubelet in.
 	// +optional
 	KubeletCgroups string

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	defaultRootDir = "/var/lib/kubelet"
+	DefaultRootDir = "/var/lib/kubelet"
 
 	AutoDetectCloudProvider = "auto-detect"
 
@@ -192,6 +192,13 @@ func SetDefaults_LeaderElectionConfiguration(obj *LeaderElectionConfiguration) {
 }
 
 func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
+	// pointer because the zeroDuration is valid - if you want to skip the trial period
+	if obj.ConfigTrialDuration == nil {
+		obj.ConfigTrialDuration = &metav1.Duration{Duration: 10 * time.Minute}
+	}
+	if obj.CrashLoopThreshold == nil {
+		obj.CrashLoopThreshold = utilpointer.Int32Ptr(10)
+	}
 	if obj.Authentication.Anonymous.Enabled == nil {
 		obj.Authentication.Anonymous.Enabled = boolVar(true)
 	}
@@ -214,18 +221,12 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.Address == "" {
 		obj.Address = "0.0.0.0"
 	}
-	if obj.CloudProvider == "" {
-		obj.CloudProvider = AutoDetectCloudProvider
-	}
 	if obj.CAdvisorPort == nil {
 		obj.CAdvisorPort = utilpointer.Int32Ptr(4194)
 	}
 	if obj.VolumeStatsAggPeriod == zeroDuration {
 		obj.VolumeStatsAggPeriod = metav1.Duration{Duration: time.Minute}
 	}
-	if obj.CertDirectory == "" {
-		obj.CertDirectory = "/var/run/kubernetes"
-	}
 	if obj.ContainerRuntime == "" {
 		obj.ContainerRuntime = "docker"
 	}
@@ -338,14 +339,11 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 	if obj.ResolverConfig == "" {
 		obj.ResolverConfig = kubetypes.ResolvConfDefault
 	}
-	if obj.RootDirectory == "" {
-		obj.RootDirectory = defaultRootDir
-	}
 	if obj.SerializeImagePulls == nil {
 		obj.SerializeImagePulls = boolVar(true)
 	}
 	if obj.SeccompProfileRoot == "" {
-		obj.SeccompProfileRoot = filepath.Join(defaultRootDir, "seccomp")
+		obj.SeccompProfileRoot = filepath.Join(DefaultRootDir, "seccomp")
 	}
 	if obj.StreamingConnectionIdleTimeout == zeroDuration {
 		obj.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 4 * time.Hour}

pkg/apis/componentconfig/v1alpha1/types.go

@@ -256,6 +256,18 @@ type LeaderElectionConfiguration struct {
 type KubeletConfiguration struct {
 	metav1.TypeMeta `json:",inline"`
 
+	// Only used for dynamic configuration.
+	// The length of the trial period for this configuration. If the Kubelet records CrashLoopThreshold or
+	// more startups during this period, the current configuration will be marked bad and the
+	// Kubelet will roll-back to the last-known-good. Default 10 minutes.
+	ConfigTrialDuration *metav1.Duration `json:"configTrialDuration"`
+	// Only used for dynamic configuration.
+	// If this number of Kubelet "crashes" during ConfigTrialDuration meets this threshold,
+	// the configuration fails the trial and the Kubelet rolls back to its last-known-good config.
+	// Crash-loops are detected by counting Kubelet startups, so one startup is implicitly added
+	// to this threshold to always allow a single restart per config change.
+	// Default 10, mimimum allowed is 0, maximum allowed is 10.
+	CrashLoopThreshold *int32 `json:"crashLoopThreshold"`
 	// podManifestPath is the path to the directory containing pod manifests to
 	// run, or the path to a single manifest file
 	PodManifestPath string `json:"podManifestPath"`
@@ -291,17 +303,10 @@ type KubeletConfiguration struct {
 	// tlsPrivateKeyFile is the ile containing x509 private key matching
 	// tlsCertFile.
 	TLSPrivateKeyFile string `json:"tlsPrivateKeyFile"`
-	// certDirectory is the directory where the TLS certs are located (by
-	// default /var/run/kubernetes). If tlsCertFile and tlsPrivateKeyFile
-	// are provided, this flag will be ignored.
-	CertDirectory string `json:"certDirectory"`
 	// authentication specifies how requests to the Kubelet's server are authenticated
 	Authentication KubeletAuthentication `json:"authentication"`
 	// authorization specifies how requests to the Kubelet's server are authorized
 	Authorization KubeletAuthorization `json:"authorization"`
-	// rootDirectory is the directory path to place kubelet files (volume
-	// mounts,etc).
-	RootDirectory string `json:"rootDirectory"`
 	// seccompProfileRoot is the directory path for seccomp profiles.
 	SeccompProfileRoot string `json:"seccompProfileRoot"`
 	// allowPrivileged enables containers to request privileged mode.
@@ -391,10 +396,6 @@ type KubeletConfiguration struct {
 	// volumePluginDir is the full path of the directory in which to search
 	// for additional third party volume plugins
 	VolumePluginDir string `json:"volumePluginDir"`
-	// cloudProvider is the provider for cloud services.
-	CloudProvider string `json:"cloudProvider"`
-	// cloudConfigFile is the path to the cloud provider configuration file.
-	CloudConfigFile string `json:"cloudConfigFile"`
 	// kubeletCgroups is the absolute name of cgroups to isolate the kubelet in.
 	KubeletCgroups string `json:"kubeletCgroups"`
 	// runtimeCgroups are cgroups that container runtime is expected to be isolated in.

pkg/apis/componentconfig/validation/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["validation.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/kubelet/cm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/apis/componentconfig/validation/validation.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	containermanager "k8s.io/kubernetes/pkg/kubelet/cm"
+)
+
+// MaxCrashLoopThreshold is the maximum allowed KubeletConfiguraiton.CrashLoopThreshold
+const MaxCrashLoopThreshold = 10
+
+// ValidateKubeletConfiguration validates `kc` and returns an error if it is invalid
+func ValidateKubeletConfiguration(kc *componentconfig.KubeletConfiguration) error {
+	// restrict crashloop threshold to between 0 and `maxCrashLoopThreshold`, inclusive
+	// more than `maxStartups=maxCrashLoopThreshold` adds unnecessary bloat to the .startups.json file,
+	// and negative values would be silly.
+	if kc.CrashLoopThreshold < 0 || kc.CrashLoopThreshold > MaxCrashLoopThreshold {
+		return fmt.Errorf("field `CrashLoopThreshold` must be between 0 and %d, inclusive", MaxCrashLoopThreshold)
+	}
+
+	if !kc.CgroupsPerQOS && len(kc.EnforceNodeAllocatable) > 0 {
+		return fmt.Errorf("node allocatable enforcement is not supported unless Cgroups Per QOS feature is turned on")
+	}
+	if kc.SystemCgroups != "" && kc.CgroupRoot == "" {
+		return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
+	}
+	for _, val := range kc.EnforceNodeAllocatable {
+		switch val {
+		case containermanager.NodeAllocatableEnforcementKey:
+		case containermanager.SystemReservedEnforcementKey:
+		case containermanager.KubeReservedEnforcementKey:
+			continue
+		default:
+			return fmt.Errorf("invalid option %q specified for EnforceNodeAllocatable setting. Valid options are %q, %q or %q",
+				val, containermanager.NodeAllocatableEnforcementKey, containermanager.SystemReservedEnforcementKey, containermanager.KubeReservedEnforcementKey)
+		}
+	}
+	return nil
+}

pkg/kubelet/BUILD

@@ -64,6 +64,7 @@ go_library(
         "//pkg/kubelet/gpu:go_default_library",
         "//pkg/kubelet/gpu/nvidia:go_default_library",
         "//pkg/kubelet/images:go_default_library",
+        "//pkg/kubelet/kubeletconfig:go_default_library",
         "//pkg/kubelet/kuberuntime:go_default_library",
         "//pkg/kubelet/lifecycle:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
@@ -256,6 +257,7 @@ filegroup(
         "//pkg/kubelet/eviction:all-srcs",
         "//pkg/kubelet/gpu:all-srcs",
         "//pkg/kubelet/images:all-srcs",
+        "//pkg/kubelet/kubeletconfig:all-srcs",
         "//pkg/kubelet/kuberuntime:all-srcs",
         "//pkg/kubelet/leaky:all-srcs",
         "//pkg/kubelet/lifecycle:all-srcs",

pkg/kubelet/certificate/kubelet.go

@@ -31,15 +31,15 @@ import (
 
 // NewKubeletServerCertificateManager creates a certificate manager for the kubelet when retrieving a server certificate
 // or returns an error.
-func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string) (Manager, error) {
+func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg *componentconfig.KubeletConfiguration, nodeName types.NodeName, ips []net.IP, hostnames []string, certDirectory string) (Manager, error) {
 	var certSigningRequestClient clientcertificates.CertificateSigningRequestInterface
 	if kubeClient != nil && kubeClient.Certificates() != nil {
 		certSigningRequestClient = kubeClient.Certificates().CertificateSigningRequests()
 	}
 	certificateStore, err := NewFileStore(
 		"kubelet-server",
-		kubeCfg.CertDirectory,
-		kubeCfg.CertDirectory,
+		certDirectory,
+		certDirectory,
 		kubeCfg.TLSCertFile,
 		kubeCfg.TLSPrivateKeyFile)
 	if err != nil {

pkg/kubelet/kubelet.go

@@ -74,6 +74,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/gpu"
 	"k8s.io/kubernetes/pkg/kubelet/gpu/nvidia"
 	"k8s.io/kubernetes/pkg/kubelet/images"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig"
 	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
 	"k8s.io/kubernetes/pkg/kubelet/metrics"
@@ -189,7 +190,15 @@ type Bootstrap interface {
 }
 
 // Builder creates and initializes a Kubelet instance
-type Builder func(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, hostnameOverride, nodeIP, providerID string) (Bootstrap, error)
+type Builder func(kubeCfg *componentconfig.KubeletConfiguration,
+	kubeDeps *Dependencies,
+	crOptions *options.ContainerRuntimeOptions,
+	hostnameOverride,
+	nodeIP,
+	providerID,
+	cloudProvider,
+	certDirectory,
+	rootDirectory string) (Bootstrap, error)
 
 // Dependencies is a bin for things we might consider "injected dependencies" -- objects constructed
 // at runtime that are necessary for running the Kubelet. This is a temporary solution for grouping
@@ -218,23 +227,24 @@ type Dependencies struct {
 	Options                 []Option
 
 	// Injected Dependencies
-	Auth               server.AuthInterface
-	CAdvisorInterface  cadvisor.Interface
-	Cloud              cloudprovider.Interface
-	ContainerManager   cm.ContainerManager
-	DockerClient       libdocker.Interface
-	EventClient        v1core.EventsGetter
-	KubeClient         clientset.Interface
-	ExternalKubeClient clientgoclientset.Interface
-	Mounter            mount.Interface
-	NetworkPlugins     []network.NetworkPlugin
-	OOMAdjuster        *oom.OOMAdjuster
-	OSInterface        kubecontainer.OSInterface
-	PodConfig          *config.PodConfig
-	Recorder           record.EventRecorder
-	Writer             kubeio.Writer
-	VolumePlugins      []volume.VolumePlugin
-	TLSOptions         *server.TLSOptions
+	Auth                    server.AuthInterface
+	CAdvisorInterface       cadvisor.Interface
+	Cloud                   cloudprovider.Interface
+	ContainerManager        cm.ContainerManager
+	DockerClient            libdocker.Interface
+	EventClient             v1core.EventsGetter
+	KubeClient              clientset.Interface
+	ExternalKubeClient      clientgoclientset.Interface
+	Mounter                 mount.Interface
+	NetworkPlugins          []network.NetworkPlugin
+	OOMAdjuster             *oom.OOMAdjuster
+	OSInterface             kubecontainer.OSInterface
+	PodConfig               *config.PodConfig
+	Recorder                record.EventRecorder
+	Writer                  kubeio.Writer
+	VolumePlugins           []volume.VolumePlugin
+	TLSOptions              *server.TLSOptions
+	KubeletConfigController *kubeletconfig.Controller
 }
 
 // makePodSourceConfig creates a config.PodConfig from the given
@@ -284,9 +294,17 @@ func getRuntimeAndImageServices(config *componentconfig.KubeletConfiguration) (i
 
 // NewMainKubelet instantiates a new Kubelet object along with all the required internal modules.
 // No initialization of Kubelet and its modules should happen here.
-func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dependencies, crOptions *options.ContainerRuntimeOptions, hostnameOverride, nodeIP, providerID string) (*Kubelet, error) {
-	if kubeCfg.RootDirectory == "" {
-		return nil, fmt.Errorf("invalid root directory %q", kubeCfg.RootDirectory)
+func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration,
+	kubeDeps *Dependencies,
+	crOptions *options.ContainerRuntimeOptions,
+	hostnameOverride,
+	nodeIP,
+	providerID,
+	cloudProvider,
+	certDirectory,
+	rootDirectory string) (*Kubelet, error) {
+	if rootDirectory == "" {
+		return nil, fmt.Errorf("invalid root directory %q", rootDirectory)
 	}
 	if kubeCfg.SyncFrequency.Duration <= 0 {
 		return nil, fmt.Errorf("invalid sync frequency %d", kubeCfg.SyncFrequency.Duration)
@@ -429,7 +447,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
 		hostname:                       hostname,
 		nodeName:                       nodeName,
 		kubeClient:                     kubeDeps.KubeClient,
-		rootDirectory:                  kubeCfg.RootDirectory,
+		rootDirectory:                  rootDirectory,
 		resyncInterval:                 kubeCfg.SyncFrequency.Duration,
 		sourcesReady:                   config.NewSourcesReady(kubeDeps.PodConfig.SeenAllSources),
 		registerNode:                   kubeCfg.RegisterNode,
@@ -443,8 +461,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
 		recorder:                       kubeDeps.Recorder,
 		cadvisor:                       kubeDeps.CAdvisorInterface,
 		cloud:                          kubeDeps.Cloud,
-		autoDetectCloudProvider:   (componentconfigv1alpha1.AutoDetectCloudProvider == kubeCfg.CloudProvider),
-		externalCloudProvider:     cloudprovider.IsExternal(kubeCfg.CloudProvider),
+		autoDetectCloudProvider:   (componentconfigv1alpha1.AutoDetectCloudProvider == cloudProvider),
+		externalCloudProvider:     cloudprovider.IsExternal(cloudProvider),
 		providerID:                providerID,
 		nodeRef:                   nodeRef,
 		nodeLabels:                kubeCfg.NodeLabels,
@@ -696,7 +714,7 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
 
 		ips = append(ips, cloudIPs...)
 		names := append([]string{klet.GetHostname(), hostnameOverride}, cloudNames...)
-		klet.serverCertificateManager, err = certificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names)
+		klet.serverCertificateManager, err = certificate.NewKubeletServerCertificateManager(klet.kubeClient, kubeCfg, klet.nodeName, ips, names, certDirectory)
 		if err != nil {
 			return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
 		}

pkg/kubelet/kubeletconfig/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "configsync.go",
+        "controller.go",
+        "rollback.go",
+        "watch.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/validation:go_default_library",
+        "//pkg/kubelet/kubeletconfig/badconfig:go_default_library",
+        "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
+        "//pkg/kubelet/kubeletconfig/checkpoint/store:go_default_library",
+        "//pkg/kubelet/kubeletconfig/configfiles:go_default_library",
+        "//pkg/kubelet/kubeletconfig/startups:go_default_library",
+        "//pkg/kubelet/kubeletconfig/status:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/equal:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/panic:go_default_library",
+        "//pkg/version:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/kubeletconfig/badconfig:all-srcs",
+        "//pkg/kubelet/kubeletconfig/checkpoint:all-srcs",
+        "//pkg/kubelet/kubeletconfig/configfiles:all-srcs",
+        "//pkg/kubelet/kubeletconfig/startups:all-srcs",
+        "//pkg/kubelet/kubeletconfig/status:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/codec:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/equal:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/files:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/log:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/panic:all-srcs",
+        "//pkg/kubelet/kubeletconfig/util/test:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/OWNERS

+approvers:
+- mtaufen
+- dchen1107
+reviewers:
+- sig-node-reviewers

pkg/kubelet/kubeletconfig/badconfig/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "badconfig_test.go",
+        "fstracker_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/test:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "badconfig.go",
+        "fstracker.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/badconfig/badconfig.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package badconfig
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+// Tracker tracks "bad" configurations in a storage layer
+type Tracker interface {
+	// Initialize sets up the storage layer
+	Initialize() error
+	// MarkBad marks `uid` as a bad config and records `reason` as the reason for marking it bad
+	MarkBad(uid, reason string) error
+	// Entry returns the Entry for `uid` if it exists in the tracker, otherise nil
+	Entry(uid string) (*Entry, error)
+}
+
+// Entry describes when a configuration was marked bad and why
+type Entry struct {
+	Time   string `json:"time"`
+	Reason string `json:"reason"`
+}
+
+// markBad makes an entry in `m` for the config with `uid` and reason `reason`
+func markBad(m map[string]Entry, uid, reason string) {
+	now := time.Now()
+	entry := Entry{
+		Time:   now.Format(time.RFC3339), // use RFC3339 time format
+		Reason: reason,
+	}
+	m[uid] = entry
+}
+
+// getEntry returns the Entry for `uid` in `m`, or nil if no such entry exists
+func getEntry(m map[string]Entry, uid string) *Entry {
+	entry, ok := m[uid]
+	if ok {
+		return &entry
+	}
+	return nil
+}
+
+// encode retuns a []byte representation of `m`, for saving `m` to a storage layer
+func encode(m map[string]Entry) ([]byte, error) {
+	data, err := json.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+// decode transforms a []byte into a `map[string]Entry`, or returns an error if it can't produce said map
+// if `data` is empty, returns an empty map
+func decode(data []byte) (map[string]Entry, error) {
+	// create the map
+	m := map[string]Entry{}
+	// if the data is empty, just return the empty map
+	if len(data) == 0 {
+		return m, nil
+	}
+	// otherwise unmarshal the json
+	if err := json.Unmarshal(data, &m); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal, error: %v", err)
+	}
+	return m, nil
+}

pkg/kubelet/kubeletconfig/badconfig/badconfig_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package badconfig
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestMarkBad(t *testing.T) {
+	// build a map with one entry
+	m := map[string]Entry{}
+	uid := "uid"
+	reason := "reason"
+	markBad(m, uid, reason)
+
+	// the entry should exist for uid
+	entry, ok := m[uid]
+	if !ok {
+		t.Fatalf("expect entry for uid %q, but none exists", uid)
+	}
+
+	// the entry's reason should match the reason it was marked bad with
+	if entry.Reason != reason {
+		t.Errorf("expect Entry.Reason %q, but got %q", reason, entry.Reason)
+	}
+
+	// the entry's timestamp should be in RFC3339 format
+	if err := assertRFC3339(entry.Time); err != nil {
+		t.Errorf("expect Entry.Time to use RFC3339 format, but got %q, error: %v", entry.Time, err)
+	}
+
+	// it should be the only entry in the map thus far
+	if n := len(m); n != 1 {
+		t.Errorf("expect one entry in the map, but got %d", n)
+	}
+
+}
+
+func TestGetEntry(t *testing.T) {
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	expect := &Entry{
+		Time:   nowstamp,
+		Reason: "reason",
+	}
+	m := map[string]Entry{uid: *expect}
+
+	// should return nil for entries that don't exist
+	bogus := "bogus-uid"
+	if e := getEntry(m, bogus); e != nil {
+		t.Errorf("expect nil for entries that don't exist (uid: %q), but got %#v", bogus, e)
+	}
+
+	// should return non-nil for entries that exist
+	if e := getEntry(m, uid); e == nil {
+		t.Errorf("expect non-nil for entries that exist (uid: %q), but got nil", uid)
+	} else if !reflect.DeepEqual(expect, e) {
+		// entry should match what we inserted for the given UID
+		t.Errorf("expect entry for uid %q to match %#v, but got %#v", uid, expect, e)
+	}
+}
+
+func TestEncode(t *testing.T) {
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	expect := fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp)
+	m := map[string]Entry{uid: {
+		Time:   nowstamp,
+		Reason: "reason",
+	}}
+
+	data, err := encode(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	json := string(data)
+
+	if json != expect {
+		t.Errorf("expect encoding of %#v to match %q, but got %q", m, expect, json)
+	}
+}
+
+func TestDecode(t *testing.T) {
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	valid := []byte(fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp))
+	expect := map[string]Entry{uid: {
+		Time:   nowstamp,
+		Reason: "reason",
+	}}
+
+	// decoding valid json should result in an object with the correct values
+	if m, err := decode(valid); err != nil {
+		t.Errorf("expect decoding valid json %q to produce a map, but got error: %v", valid, err)
+	} else if !reflect.DeepEqual(expect, m) {
+		// m should equal expected decoded object
+		t.Errorf("expect decoding valid json %q to produce %#v, but got %#v", valid, expect, m)
+	}
+
+	// decoding invalid json should return an error
+	invalid := []byte(`invalid`)
+	if m, err := decode(invalid); err == nil {
+		t.Errorf("expect decoding invalid json %q to return an error, but decoded to %#v", invalid, m)
+	}
+}
+
+func TestRoundTrip(t *testing.T) {
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	expect := map[string]Entry{uid: {
+		Time:   nowstamp,
+		Reason: "reason",
+	}}
+
+	// test that encoding and decoding an object results in the same value
+	data, err := encode(expect)
+	if err != nil {
+		t.Fatalf("failed to encode %#v, error: %v", expect, err)
+	}
+	after, err := decode(data)
+	if err != nil {
+		t.Fatalf("failed to decode %q, error: %v", string(data), err)
+	}
+	if !reflect.DeepEqual(expect, after) {
+		t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after)
+	}
+}
+
+func assertRFC3339(s string) error {
+	tm, err := time.Parse(time.RFC3339, s)
+	if err != nil {
+		return fmt.Errorf("expect RFC3339 format, but failed to parse, error: %v", err)
+	}
+	// parsing succeeded, now finish round-trip and compare
+	rt := tm.Format(time.RFC3339)
+	if rt != s {
+		return fmt.Errorf("expect RFC3339 format, but failed to round trip unchanged, original %q, round-trip %q", s, rt)
+	}
+	return nil
+}

pkg/kubelet/kubeletconfig/badconfig/fstracker.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package badconfig
+
+import (
+	"path/filepath"
+
+	utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+const (
+	badConfigsFile = "bad-configs.json"
+)
+
+// fsTracker tracks bad config in the local filesystem
+type fsTracker struct {
+	// fs is the filesystem to use for storage operations; can be mocked for testing
+	fs utilfs.Filesystem
+	// trackingDir is the absolute path to the storage directory for fsTracker
+	trackingDir string
+}
+
+// NewFsTracker returns a new Tracker that will store information in the `trackingDir`
+func NewFsTracker(fs utilfs.Filesystem, trackingDir string) Tracker {
+	return &fsTracker{
+		fs:          fs,
+		trackingDir: trackingDir,
+	}
+}
+
+func (tracker *fsTracker) Initialize() error {
+	utillog.Infof("initializing bad config tracking directory %q", tracker.trackingDir)
+	if err := utilfiles.EnsureDir(tracker.fs, tracker.trackingDir); err != nil {
+		return err
+	}
+	if err := utilfiles.EnsureFile(tracker.fs, filepath.Join(tracker.trackingDir, badConfigsFile)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tracker *fsTracker) MarkBad(uid, reason string) error {
+	m, err := tracker.load()
+	if err != nil {
+		return err
+	}
+	// create the bad config entry in the map
+	markBad(m, uid, reason)
+	// save the file
+	if err := tracker.save(m); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tracker *fsTracker) Entry(uid string) (*Entry, error) {
+	m, err := tracker.load()
+	if err != nil {
+		return nil, err
+	}
+	// return the entry, or nil if it doesn't exist
+	return getEntry(m, uid), nil
+}
+
+// load loads the bad-config-tracking file from disk and decodes the map encoding it contains
+func (tracker *fsTracker) load() (map[string]Entry, error) {
+	path := filepath.Join(tracker.trackingDir, badConfigsFile)
+	// load the file
+	data, err := tracker.fs.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return decode(data)
+}
+
+// save replaces the contents of the bad-config-tracking file with the encoding of `m`
+func (tracker *fsTracker) save(m map[string]Entry) error {
+	// encode the map
+	data, err := encode(m)
+	if err != nil {
+		return err
+	}
+	// save the file
+	path := filepath.Join(tracker.trackingDir, badConfigsFile)
+	if err := utilfiles.ReplaceFile(tracker.fs, path, data); err != nil {
+		return err
+	}
+	return nil
+}

pkg/kubelet/kubeletconfig/badconfig/fstracker_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package badconfig
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+const testTrackingDir = "/test-tracking-dir"
+
+// TODO(mtaufen): this file reuses a lot of test code from badconfig_test.go, should consolidate
+
+func newInitializedFakeFsTracker() (*fsTracker, error) {
+	fs := utilfs.NewFakeFs()
+	tracker := NewFsTracker(fs, testTrackingDir)
+	if err := tracker.Initialize(); err != nil {
+		return nil, err
+	}
+	return tracker.(*fsTracker), nil
+}
+
+func TestFsTrackerInitialize(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("fsTracker.Initialize() failed with error: %v", err)
+	}
+
+	// check that testTrackingDir exists
+	_, err = tracker.fs.Stat(testTrackingDir)
+	if err != nil {
+		t.Fatalf("expect %q to exist, but stat failed with error: %v", testTrackingDir, err)
+	}
+
+	// check that testTrackingDir contains the badConfigsFile
+	path := filepath.Join(testTrackingDir, badConfigsFile)
+	_, err = tracker.fs.Stat(path)
+	if err != nil {
+		t.Fatalf("expect %q to exist, but stat failed with error: %v", path, err)
+	}
+}
+
+func TestFsTrackerMarkBad(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	// create a bad config entry in the fs
+	uid := "uid"
+	reason := "reason"
+	tracker.MarkBad(uid, reason)
+
+	// load the map from the fs
+	m, err := tracker.load()
+	if err != nil {
+		t.Fatalf("failed to load bad-config data, error: %v", err)
+	}
+
+	// the entry should exist for uid
+	entry, ok := m[uid]
+	if !ok {
+		t.Fatalf("expect entry for uid %q, but none exists", uid)
+	}
+
+	// the entry's reason should match the reason it was marked bad with
+	if entry.Reason != reason {
+		t.Errorf("expect Entry.Reason %q, but got %q", reason, entry.Reason)
+	}
+
+	// the entry's timestamp should be in RFC3339 format
+	if err := assertRFC3339(entry.Time); err != nil {
+		t.Errorf("expect Entry.Time to use RFC3339 format, but got %q, error: %v", entry.Time, err)
+	}
+
+	// it should be the only entry in the map thus far
+	if n := len(m); n != 1 {
+		t.Errorf("expect one entry in the map, but got %d", n)
+	}
+}
+
+func TestFsTrackerEntry(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	// manually save a correct entry to fs
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	expect := &Entry{
+		Time:   nowstamp,
+		Reason: "reason",
+	}
+	m := map[string]Entry{uid: *expect}
+	err = tracker.save(m)
+	if err != nil {
+		t.Fatalf("failed to save bad-config data, error: %v", err)
+	}
+
+	// should return nil for entries that don't exist
+	bogus := "bogus-uid"
+	e, err := tracker.Entry(bogus)
+	if err != nil {
+		t.Errorf("expect nil for entries that don't exist (uid: %q), but got error: %v", bogus, err)
+	} else if e != nil {
+		t.Errorf("expect nil for entries that don't exist (uid: %q), but got %#v", bogus, e)
+	}
+
+	// should return non-nil for entries that exist
+	e, err = tracker.Entry(uid)
+	if err != nil {
+		t.Errorf("expect non-nil for entries that exist (uid: %q), but got error: %v", uid, err)
+	} else if e == nil {
+		t.Errorf("expect non-nil for entries that exist (uid: %q), but got nil", uid)
+	} else if !reflect.DeepEqual(expect, e) {
+		// entry should match what we inserted for the given UID
+		t.Errorf("expect entry for uid %q to match %#v, but got %#v", uid, expect, e)
+	}
+}
+
+// TODO(mtaufen): test loading invalid json (see startups/fstracker_test.go for example)
+func TestFsTrackerLoad(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	uid := "uid"
+	nowstamp := time.Now().Format(time.RFC3339)
+	cases := []struct {
+		desc   string
+		data   []byte
+		expect map[string]Entry
+		err    string
+	}{
+		// empty file
+		{"empty file", []byte(""), map[string]Entry{}, ""},
+		// empty map
+		{"empty map", []byte("{}"), map[string]Entry{}, ""},
+		// valid json
+		{"valid json", []byte(fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp)),
+			map[string]Entry{uid: {
+				Time:   nowstamp,
+				Reason: "reason",
+			}}, ""},
+		// invalid json
+		{"invalid json", []byte(`*`), map[string]Entry{}, "failed to unmarshal"},
+	}
+
+	for _, c := range cases {
+		// save a file containing the correct serialization
+		utilfiles.ReplaceFile(tracker.fs, filepath.Join(testTrackingDir, badConfigsFile), c.data)
+
+		// loading valid json should result in an object with the correct values
+		m, err := tracker.load()
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		if !reflect.DeepEqual(c.expect, m) {
+			// m should equal expected decoded object
+			t.Errorf("case %q, expect %#v but got %#v", c.desc, c.expect, m)
+		}
+	}
+}
+
+func TestFsTrackerSave(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	uid := "uid"
+	nowstamp := time.Now().Format(time.RFC3339)
+	cases := []struct {
+		desc   string
+		m      map[string]Entry
+		expect string
+		err    string
+	}{
+		// empty map
+		{"empty map", map[string]Entry{}, "{}", ""},
+		// 1-entry map
+		{"1-entry map",
+			map[string]Entry{uid: {
+				Time:   nowstamp,
+				Reason: "reason",
+			}},
+			fmt.Sprintf(`{"%s":{"time":"%s","reason":"reason"}}`, uid, nowstamp), ""},
+	}
+
+	for _, c := range cases {
+		if err := tracker.save(c.m); utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+
+		data, err := tracker.fs.ReadFile(filepath.Join(testTrackingDir, badConfigsFile))
+		if err != nil {
+			t.Fatalf("failed to read bad-config file, error: %v", err)
+		}
+		json := string(data)
+
+		if json != c.expect {
+			t.Errorf("case %q, expect %q but got %q", c.desc, c.expect, json)
+		}
+	}
+}
+
+func TestFsTrackerRoundTrip(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	nowstamp := time.Now().Format(time.RFC3339)
+	uid := "uid"
+	expect := map[string]Entry{uid: {
+		Time:   nowstamp,
+		Reason: "reason",
+	}}
+
+	// test that saving and loading an object results in the same value
+	err = tracker.save(expect)
+	if err != nil {
+		t.Fatalf("failed to save bad-config data, error: %v", err)
+	}
+	after, err := tracker.load()
+	if err != nil {
+		t.Fatalf("failed to load bad-config data, error: %v", err)
+	}
+	if !reflect.DeepEqual(expect, after) {
+		t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after)
+	}
+}

pkg/kubelet/kubeletconfig/checkpoint/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "checkpoint_test.go",
+        "configmap_test.go",
+        "download_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/test:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "checkpoint.go",
+        "configmap.go",
+        "download.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/kubeletconfig/checkpoint/store:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"fmt"
+
+	apiv1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// Checkpoint represents a local copy of a config source (payload) object
+type Checkpoint interface {
+	// UID returns the UID of the config source object behind the Checkpoint
+	UID() string
+	// Parse parses the checkpoint into the internal KubeletConfiguration type
+	Parse() (*componentconfig.KubeletConfiguration, error)
+	// Encode returns a []byte representation of the config source object behind the Checkpoint
+	Encode() ([]byte, error)
+
+	// object returns the underlying checkpointed object. If you want to compare sources for equality, use EqualCheckpoints,
+	// which compares the underlying checkpointed objects for semantic API equality.
+	object() interface{}
+}
+
+// DecodeCheckpoint is a helper for using the apimachinery to decode serialized checkpoints
+func DecodeCheckpoint(data []byte) (Checkpoint, error) {
+	// decode the checkpoint
+	obj, err := runtime.Decode(api.Codecs.UniversalDecoder(), data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode, error: %v", err)
+	}
+
+	// TODO(mtaufen): for now we assume we are trying to load a ConfigMap checkpoint, may need to extend this if we allow other checkpoint types
+
+	// convert it to the external ConfigMap type, so we're consistently working with the external type outside of the on-disk representation
+	cm := &apiv1.ConfigMap{}
+	err = api.Scheme.Convert(obj, cm, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert decoded object into a v1 ConfigMap, error: %v", err)
+	}
+	return NewConfigMapCheckpoint(cm)
+}
+
+// EqualCheckpoints compares two Checkpoints for equality, if their underlying objects are equal, so are the Checkpoints
+func EqualCheckpoints(a, b Checkpoint) bool {
+	if a != nil && b != nil {
+		return apiequality.Semantic.DeepEqual(a.object(), b.object())
+	}
+	if a == nil && b == nil {
+		return true
+	}
+	return false
+}

pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	apiv1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+// newUnsupportedEncoded returns an encoding of an object that does not have a Checkpoint implementation
+func newUnsupportedEncoded(t *testing.T) []byte {
+	encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
+	if err != nil {
+		t.Fatalf("could not create an encoder, error: %v", err)
+	}
+	unsupported := &apiv1.Node{}
+	data, err := runtime.Encode(encoder, unsupported)
+	if err != nil {
+		t.Fatalf("could not encode object, error: %v", err)
+	}
+	return data
+}
+
+func TestDecodeCheckpoint(t *testing.T) {
+	// generate correct Checkpoint for v1/ConfigMap test case
+	cm, err := NewConfigMapCheckpoint(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: types.UID("uid")}})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	// generate unsupported object encoding for unsupported type test case
+	unsupported := newUnsupportedEncoded(t)
+
+	// test cases
+	cases := []struct {
+		desc   string
+		data   []byte
+		expect Checkpoint // expect a deeply-equal Checkpoint to be returned from Decode
+		err    string     // expect error to contain this substring
+	}{
+		// v1/ConfigMap
+		{"v1/ConfigMap", []byte(`{"apiVersion": "v1","kind": "ConfigMap","metadata": {"uid": "uid"}}`), cm, ""},
+		// malformed
+		{"malformed", []byte("malformed"), nil, "failed to decode"},
+		// no UID
+		{"no UID", []byte(`{"apiVersion": "v1","kind": "ConfigMap"}`), nil, "ConfigMap must have a UID"},
+		// well-formed, but unsupported type
+		{"well-formed, but unsupported encoded type", unsupported, nil, "failed to convert"},
+	}
+
+	for _, c := range cases {
+		cpt, err := DecodeCheckpoint(c.data)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		// Unfortunately reflect.DeepEqual treats nil data structures as != empty data structures, so
+		// we have to settle for semantic equality of the underlying checkpointed API objects.
+		// If additional fields are added to the object that implements the Checkpoint interface,
+		// they should be added to a named sub-object to facilitate a DeepEquals comparison
+		// of the extra fields.
+		// decoded checkpoint should match expected checkpoint
+		if !apiequality.Semantic.DeepEqual(cpt.object(), c.expect.object()) {
+			t.Errorf("case %q, expect checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
+		}
+	}
+}

pkg/kubelet/kubeletconfig/checkpoint/configmap.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"fmt"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
+)
+
+const configMapConfigKey = "kubelet"
+
+// configMapCheckpoint implements Checkpoint, backed by a v1/ConfigMap config source object
+type configMapCheckpoint struct {
+	configMap *apiv1.ConfigMap
+}
+
+// NewConfigMapCheckpoint returns a Checkpoint backed by `cm`. `cm` must be non-nil
+// and have a non-empty ObjectMeta.UID, or an error will be returned.
+func NewConfigMapCheckpoint(cm *apiv1.ConfigMap) (Checkpoint, error) {
+	if cm == nil {
+		return nil, fmt.Errorf("ConfigMap must be non-nil to be treated as a Checkpoint")
+	} else if len(cm.ObjectMeta.UID) == 0 {
+		return nil, fmt.Errorf("ConfigMap must have a UID to be treated as a Checkpoint")
+	}
+	return &configMapCheckpoint{cm}, nil
+}
+
+// UID returns the UID of a configMapCheckpoint
+func (c *configMapCheckpoint) UID() string {
+	return string(c.configMap.UID)
+}
+
+// implements Parse for v1/ConfigMap checkpoints
+func (c *configMapCheckpoint) Parse() (*componentconfig.KubeletConfiguration, error) {
+	const emptyCfgErr = "config was empty, but some parameters are required"
+
+	cm := c.configMap
+
+	if len(cm.Data) == 0 {
+		return nil, fmt.Errorf(emptyCfgErr)
+	}
+
+	// TODO(mtaufen): Once the KubeletConfiguration type is decomposed, extend this to a key for each sub-object
+	config, ok := cm.Data[configMapConfigKey]
+	if !ok {
+		return nil, fmt.Errorf("key %q not found in ConfigMap", configMapConfigKey)
+	} else if len(config) == 0 {
+		return nil, fmt.Errorf(emptyCfgErr)
+	}
+
+	return utilcodec.DecodeKubeletConfiguration([]byte(config))
+}
+
+// Encode encodes a configMapCheckpoint
+func (c *configMapCheckpoint) Encode() ([]byte, error) {
+	cm := c.configMap
+	encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
+	if err != nil {
+		return nil, err
+	}
+	data, err := runtime.Encode(encoder, cm)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (c *configMapCheckpoint) object() interface{} {
+	return c.configMap
+}

pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	apiv1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	ccv1a1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+func TestNewConfigMapCheckpoint(t *testing.T) {
+	cases := []struct {
+		desc string
+		cm   *apiv1.ConfigMap
+		err  string
+	}{
+		{"nil v1/ConfigMap", nil, "must be non-nil"},
+		{"empty v1/ConfigMap", &apiv1.ConfigMap{}, "must have a UID"},
+		{"populated v1/ConfigMap",
+			&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "name",
+					UID:  types.UID("uid"),
+				},
+				Data: map[string]string{
+					"key1": "value1",
+					"key2": "value2",
+				},
+			}, ""},
+	}
+
+	for _, c := range cases {
+		cpt, err := NewConfigMapCheckpoint(c.cm)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		// underlying object should match the object passed in
+		if !apiequality.Semantic.DeepEqual(cpt.object(), c.cm) {
+			t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.cm), spew.Sdump(cpt))
+		}
+	}
+}
+
+func TestConfigMapCheckpointUID(t *testing.T) {
+	cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
+	for _, uidIn := range cases {
+		cpt := &configMapCheckpoint{
+			&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{UID: types.UID(uidIn)},
+			},
+		}
+		// UID method should return the correct value of the UID
+		uidOut := cpt.UID()
+		if uidIn != uidOut {
+			t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
+		}
+	}
+}
+
+func TestConfigMapCheckpointParse(t *testing.T) {
+	// get the built-in default configuration
+	external := &ccv1a1.KubeletConfiguration{}
+	api.Scheme.Default(external)
+	defaultConfig := &componentconfig.KubeletConfiguration{}
+	err := api.Scheme.Convert(external, defaultConfig, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	cases := []struct {
+		desc   string
+		cm     *apiv1.ConfigMap
+		expect *componentconfig.KubeletConfiguration
+		err    string
+	}{
+		{"empty data", &apiv1.ConfigMap{}, nil, "config was empty"},
+		// missing kubelet key
+		{"missing kubelet key", &apiv1.ConfigMap{Data: map[string]string{
+			"bogus": "stuff"}}, nil, fmt.Sprintf("key %q not found", configMapConfigKey)},
+		// invalid format
+		{"invalid yaml", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": "*"}}, nil, "failed to decode"},
+		{"invalid json", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": "{*"}}, nil, "failed to decode"},
+		// invalid object
+		{"missing kind", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `{"apiVersion":"componentconfig/v1alpha1"}`}}, nil, "failed to decode"},
+		{"missing version", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `{"kind":"KubeletConfiguration"}`}}, nil, "failed to decode"},
+		{"unregistered kind", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `{"kind":"BogusKind","apiVersion":"componentconfig/v1alpha1"}`}}, nil, "failed to decode"},
+		{"unregistered version", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"bogusversion"}`}}, nil, "failed to decode"},
+		// empty object with correct kind and version should result in the defaults for that kind and version
+		{"default from yaml", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `kind: KubeletConfiguration
+apiVersion: componentconfig/v1alpha1`}}, defaultConfig, ""},
+		{"default from json", &apiv1.ConfigMap{Data: map[string]string{
+			"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"componentconfig/v1alpha1"}`}}, defaultConfig, ""},
+	}
+	for _, c := range cases {
+		cpt := &configMapCheckpoint{c.cm}
+		kc, err := cpt.Parse()
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		// we expect the parsed configuration to match what we described in the ConfigMap
+		if !apiequality.Semantic.DeepEqual(c.expect, kc) {
+			t.Errorf("case %q, expect config %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(kc))
+		}
+	}
+}
+
+func TestConfigMapCheckpointEncode(t *testing.T) {
+	// only one case, based on output from the existing encoder, and since
+	// this is hard to test (key order isn't guaranteed), we should probably
+	// just stick to this test case and mostly rely on the round-trip test.
+	cases := []struct {
+		desc   string
+		cpt    *configMapCheckpoint
+		expect string
+	}{
+		// we expect Checkpoints to be encoded as a json representation of the underlying API object
+		{"one-key",
+			&configMapCheckpoint{&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{Name: "one-key"},
+				Data:       map[string]string{"one": ""}}},
+			`{"kind":"ConfigMap","apiVersion":"v1","metadata":{"name":"one-key","creationTimestamp":null},"data":{"one":""}}
+`},
+	}
+
+	for _, c := range cases {
+		data, err := c.cpt.Encode()
+		// we don't expect any errors from encoding
+		if utiltest.SkipRest(t, c.desc, err, "") {
+			continue
+		}
+		if string(data) != c.expect {
+			t.Errorf("case %q, expect encoding %q but got %q", c.desc, c.expect, string(data))
+		}
+	}
+}
+
+func TestConfigMapCheckpointRoundTrip(t *testing.T) {
+	cases := []struct {
+		desc      string
+		cpt       *configMapCheckpoint
+		decodeErr string
+	}{
+		// empty data
+		{"empty data",
+			&configMapCheckpoint{&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "empty-data-sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+					UID:  "uid",
+				},
+				Data: map[string]string{}}},
+			""},
+		// two keys
+		{"two keys",
+			&configMapCheckpoint{&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
+					UID:  "uid",
+				},
+				Data: map[string]string{"one": "", "two": "2"}}},
+			""},
+		// missing uid
+		{"missing uid",
+			&configMapCheckpoint{&apiv1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
+					UID:  "",
+				},
+				Data: map[string]string{"one": "", "two": "2"}}},
+			"must have a UID"},
+	}
+	for _, c := range cases {
+		// we don't expect any errors from encoding
+		data, err := c.cpt.Encode()
+		if utiltest.SkipRest(t, c.desc, err, "") {
+			continue
+		}
+		after, err := DecodeCheckpoint(data)
+		if utiltest.SkipRest(t, c.desc, err, c.decodeErr) {
+			continue
+		}
+		if !apiequality.Semantic.DeepEqual(c.cpt.object(), after.object()) {
+			t.Errorf("case %q, expect round-trip result %s but got %s", c.desc, spew.Sdump(c.cpt), spew.Sdump(after))
+		}
+	}
+}

pkg/kubelet/kubeletconfig/checkpoint/download.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"fmt"
+
+	apiv1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/api"
+	utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+// RemoteConfigSource represents a remote config source object that can be downloaded as a Checkpoint
+type RemoteConfigSource interface {
+	// UID returns the UID of the remote config source object
+	UID() string
+	// Download downloads the remote config source object returns a Checkpoint backed by the object,
+	// or a sanitized failure reason and error if the download fails
+	Download(client clientset.Interface) (Checkpoint, string, error)
+	// Encode returns a []byte representation of the object behind the RemoteConfigSource
+	Encode() ([]byte, error)
+
+	// object returns the underlying source object. If you want to compare sources for equality, use EqualRemoteConfigSources,
+	// which compares the underlying source objects for semantic API equality.
+	object() interface{}
+}
+
+// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object, or returns
+// a sanitized failure reason and an error if the `source` is blatantly invalid.
+// You should only call this with a non-nil config source.
+func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource, string, error) {
+	// exactly one subfield of the config source must be non-nil, toady ConfigMapRef is the only reference
+	if source.ConfigMapRef == nil {
+		reason := "invalid NodeConfigSource, exactly one subfield must be non-nil, but all were nil"
+		return nil, reason, fmt.Errorf("%s, NodeConfigSource was: %#v", reason, source)
+	}
+
+	// validate the NodeConfigSource:
+
+	// at this point we know we're using the ConfigMapRef subfield
+	ref := source.ConfigMapRef
+
+	// name, namespace, and UID must all be non-empty for ConfigMapRef
+	if ref.Name == "" || ref.Namespace == "" || string(ref.UID) == "" {
+		reason := "invalid ObjectReference, all of UID, Name, and Namespace must be specified"
+		return nil, reason, fmt.Errorf("%s, ObjectReference was: %#v", reason, ref)
+	}
+
+	return &remoteConfigMap{source}, "", nil
+}
+
+// DecodeRemoteConfigSource is a helper for using the apimachinery to decode serialized RemoteConfigSources;
+// e.g. the objects stored in the .cur and .lkg files by checkpoint/store/fsstore.go
+func DecodeRemoteConfigSource(data []byte) (RemoteConfigSource, error) {
+	// decode the remote config source
+	obj, err := runtime.Decode(api.Codecs.UniversalDecoder(), data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode, error: %v", err)
+	}
+
+	// for now we assume we are trying to load an apiv1.NodeConfigSource,
+	// this may need to be extended if e.g. a new version of the api is born
+
+	// convert it to the external NodeConfigSource type, so we're consistently working with the external type outside of the on-disk representation
+	cs := &apiv1.NodeConfigSource{}
+	err = api.Scheme.Convert(obj, cs, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert decoded object into a v1 NodeConfigSource, error: %v", err)
+	}
+	source, _, err := NewRemoteConfigSource(cs)
+	return source, err
+}
+
+// EqualRemoteConfigSources is a helper for comparing remote config sources by
+// comparing the underlying API objects for semantic equality.
+func EqualRemoteConfigSources(a, b RemoteConfigSource) bool {
+	if a != nil && b != nil {
+		return apiequality.Semantic.DeepEqual(a.object(), b.object())
+	}
+	if a == nil && b == nil {
+		return true
+	}
+	return false
+}
+
+// remoteConfigMap implements RemoteConfigSource for v1/ConfigMap config sources
+type remoteConfigMap struct {
+	source *apiv1.NodeConfigSource
+}
+
+func (r *remoteConfigMap) UID() string {
+	return string(r.source.ConfigMapRef.UID)
+}
+
+func (r *remoteConfigMap) Download(client clientset.Interface) (Checkpoint, string, error) {
+	var reason string
+
+	uid := string(r.source.ConfigMapRef.UID)
+
+	utillog.Infof("attempting to download ConfigMap with UID %q", uid)
+
+	// get the ConfigMap via namespace/name, there doesn't seem to be a way to get it by UID
+	cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMapRef.Namespace).Get(r.source.ConfigMapRef.Name, metav1.GetOptions{})
+	if err != nil {
+		reason = fmt.Sprintf("could not download ConfigMap with name %q from namespace %q", r.source.ConfigMapRef.Name, r.source.ConfigMapRef.Namespace)
+		return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
+	}
+
+	// ensure that UID matches the UID on the reference, the ObjectReference must be unambiguous
+	if r.source.ConfigMapRef.UID != cm.UID {
+		reason = fmt.Sprintf("invalid ObjectReference, UID %q does not match UID of downloaded ConfigMap %q", r.source.ConfigMapRef.UID, cm.UID)
+		return nil, reason, fmt.Errorf(reason)
+	}
+
+	utillog.Infof("successfully downloaded ConfigMap with UID %q", uid)
+	return &configMapCheckpoint{cm}, "", nil
+}
+
+func (r *remoteConfigMap) Encode() ([]byte, error) {
+	encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
+	if err != nil {
+		return nil, err
+	}
+	data, err := runtime.Encode(encoder, r.source)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+func (r *remoteConfigMap) object() interface{} {
+	return r.source
+}

pkg/kubelet/kubeletconfig/checkpoint/download_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package checkpoint
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	apiv1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	fakeclient "k8s.io/client-go/kubernetes/fake"
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+func TestNewRemoteConfigSource(t *testing.T) {
+	cases := []struct {
+		desc   string
+		source *apiv1.NodeConfigSource
+		expect RemoteConfigSource
+		err    string
+	}{
+		// all NodeConfigSource subfields nil
+		{"all NodeConfigSource subfields nil",
+			&apiv1.NodeConfigSource{}, nil, "exactly one subfield must be non-nil"},
+		{"ConfigMapRef: empty name, namespace, and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty name and namespace
+		{"ConfigMapRef: empty name and namespace",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "uid"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty name and UID
+		{"ConfigMapRef: empty name and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty namespace and UID
+		{"ConfigMapRef: empty namespace and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty UID
+		{"ConfigMapRef: empty namespace and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty namespace
+		{"ConfigMapRef: empty namespace and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", UID: "uid"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: empty name
+		{"ConfigMapRef: empty namespace and UID",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace", UID: "uid"}}, nil, "invalid ObjectReference"},
+		// ConfigMapRef: valid reference
+		{"ConfigMapRef: valid reference",
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}},
+			&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}}, ""},
+	}
+
+	for _, c := range cases {
+		src, _, err := NewRemoteConfigSource(c.source)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		// underlying object should match the object passed in
+		if !apiequality.Semantic.DeepEqual(c.expect.object(), src.object()) {
+			t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(src))
+		}
+	}
+}
+
+func TestRemoteConfigMapUID(t *testing.T) {
+	cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
+	for _, uidIn := range cases {
+		cpt := &remoteConfigMap{
+			&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: types.UID(uidIn)}},
+		}
+		// UID method should return the correct value of the UID
+		uidOut := cpt.UID()
+		if uidIn != uidOut {
+			t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
+		}
+	}
+}
+
+func TestRemoteConfigMapDownload(t *testing.T) {
+	cm := &apiv1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "name",
+			Namespace: "namespace",
+			UID:       "uid",
+		}}
+	client := fakeclient.NewSimpleClientset(cm)
+
+	cases := []struct {
+		desc   string
+		source RemoteConfigSource
+		expect Checkpoint
+		err    string
+	}{
+
+		// object doesn't exist
+		{"object doesn't exist",
+			&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "bogus", Namespace: "namespace", UID: "bogus"}}},
+			nil, "could not download ConfigMap"},
+		// UID of downloaded object doesn't match UID of referent found via namespace/name
+		{"UID is incorrect for namespace/name",
+			&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "bogus"}}},
+			nil, "does not match UID"},
+		// successful download
+		{"object exists and reference is correct",
+			&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}},
+			&configMapCheckpoint{cm}, ""},
+	}
+
+	for _, c := range cases {
+		cpt, _, err := c.source.Download(client)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		// "downloaded" object should match the expected
+		if !apiequality.Semantic.DeepEqual(c.expect.object(), cpt.object()) {
+			t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
+		}
+	}
+}

pkg/kubelet/kubeletconfig/checkpoint/store/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "fsstore_test.go",
+        "store_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/test:go_default_library",
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fakestore.go",
+        "fsstore.go",
+        "store.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
+)
+
+// so far only implements Current(), LastKnownGood(), SetCurrent(), and SetLastKnownGood()
+type fakeStore struct {
+	current       checkpoint.RemoteConfigSource
+	lastKnownGood checkpoint.RemoteConfigSource
+}
+
+func (s *fakeStore) Initialize() error {
+	return fmt.Errorf("Initialize method not supported")
+}
+
+func (s *fakeStore) Exists(uid string) (bool, error) {
+	return false, fmt.Errorf("Exists method not supported")
+}
+
+func (s *fakeStore) Save(c checkpoint.Checkpoint) error {
+	return fmt.Errorf("Save method not supported")
+}
+
+func (s *fakeStore) Load(uid string) (checkpoint.Checkpoint, error) {
+	return nil, fmt.Errorf("Load method not supported")
+}
+
+func (s *fakeStore) CurrentModified() (time.Time, error) {
+	return time.Time{}, fmt.Errorf("CurrentModified method not supported")
+}
+
+func (s *fakeStore) Current() (checkpoint.RemoteConfigSource, error) {
+	return s.current, nil
+}
+
+func (s *fakeStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
+	return s.lastKnownGood, nil
+}
+
+func (s *fakeStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
+	s.current = source
+	return nil
+}
+
+func (s *fakeStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
+	return setCurrentUpdated(s, source)
+}
+
+func (s *fakeStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
+	s.lastKnownGood = source
+	return nil
+}
+
+func (s *fakeStore) Reset() (bool, error) {
+	return false, fmt.Errorf("Reset method not supported")
+}

pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
+	utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+const (
+	curFile = ".cur"
+	lkgFile = ".lkg"
+)
+
+// fsStore is for tracking checkpoints in the local filesystem, implements Store
+type fsStore struct {
+	// fs is the filesystem to use for storage operations; can be mocked for testing
+	fs utilfs.Filesystem
+	// checkpointsDir is the absolute path to the storage directory for fsStore
+	checkpointsDir string
+}
+
+// NewFsStore returns a Store that saves its data in `checkpointsDir`
+func NewFsStore(fs utilfs.Filesystem, checkpointsDir string) Store {
+	return &fsStore{
+		fs:             fs,
+		checkpointsDir: checkpointsDir,
+	}
+}
+
+func (s *fsStore) Initialize() error {
+	utillog.Infof("initializing config checkpoints directory %q", s.checkpointsDir)
+	if err := utilfiles.EnsureDir(s.fs, s.checkpointsDir); err != nil {
+		return err
+	}
+	if err := utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, curFile)); err != nil {
+		return err
+	}
+	if err := utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, lkgFile)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *fsStore) Exists(uid string) (bool, error) {
+	ok, err := utilfiles.FileExists(s.fs, filepath.Join(s.checkpointsDir, uid))
+	if err != nil {
+		return false, fmt.Errorf("failed to determine whether checkpoint %q exists, error: %v", uid, err)
+	}
+	return ok, nil
+}
+
+func (s *fsStore) Save(c checkpoint.Checkpoint) error {
+	// encode the checkpoint
+	data, err := c.Encode()
+	if err != nil {
+		return err
+	}
+	// save the file
+	if err := utilfiles.ReplaceFile(s.fs, filepath.Join(s.checkpointsDir, c.UID()), data); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *fsStore) Load(uid string) (checkpoint.Checkpoint, error) {
+	filePath := filepath.Join(s.checkpointsDir, uid)
+	utillog.Infof("loading configuration from %q", filePath)
+
+	// load the file
+	data, err := s.fs.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read checkpoint file %q, error: %v", filePath, err)
+	}
+
+	// decode it
+	c, err := checkpoint.DecodeCheckpoint(data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode checkpoint file %q, error: %v", filePath, err)
+	}
+	return c, nil
+}
+
+func (s *fsStore) CurrentModified() (time.Time, error) {
+	path := filepath.Join(s.checkpointsDir, curFile)
+	info, err := s.fs.Stat(path)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to stat %q while checking modification time, error: %v", path, err)
+	}
+	return info.ModTime(), nil
+}
+
+func (s *fsStore) Current() (checkpoint.RemoteConfigSource, error) {
+	return s.sourceFromFile(curFile)
+}
+
+func (s *fsStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
+	return s.sourceFromFile(lkgFile)
+}
+
+func (s *fsStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
+	return s.setSourceFile(curFile, source)
+}
+
+func (s *fsStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
+	return setCurrentUpdated(s, source)
+}
+
+func (s *fsStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
+	return s.setSourceFile(lkgFile, source)
+}
+
+func (s *fsStore) Reset() (bool, error) {
+	return reset(s)
+}
+
+// sourceFromFile returns the RemoteConfigSource stored in the file at `s.checkpointsDir/relPath`,
+// or nil if the file is empty
+func (s *fsStore) sourceFromFile(relPath string) (checkpoint.RemoteConfigSource, error) {
+	path := filepath.Join(s.checkpointsDir, relPath)
+	data, err := s.fs.ReadFile(path)
+	if err != nil {
+		return nil, err
+	} else if len(data) == 0 {
+		return nil, nil
+	}
+	return checkpoint.DecodeRemoteConfigSource(data)
+}
+
+// set source file replaces the file at `s.checkpointsDir/relPath` with a file containing `source`
+func (s *fsStore) setSourceFile(relPath string, source checkpoint.RemoteConfigSource) error {
+	path := filepath.Join(s.checkpointsDir, relPath)
+	// if nil, reset the file
+	if source == nil {
+		return utilfiles.ReplaceFile(s.fs, path, []byte{})
+	}
+	// encode the source and save it to the file
+	data, err := source.Encode()
+	if err != nil {
+		return err
+	}
+	return utilfiles.ReplaceFile(s.fs, path, data)
+}

pkg/kubelet/kubeletconfig/checkpoint/store/store.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
+)
+
+// Store saves checkpoints and information about which is the current and last-known-good checkpoint to a storage layer
+type Store interface {
+	// Initialize sets up the storage layer
+	Initialize() error
+	// Exists returns true if a checkpoint with `uid` exists in the store, false otherwise
+	Exists(uid string) (bool, error)
+	// Save saves the checkpoint to the storage layer
+	Save(c checkpoint.Checkpoint) error
+	// Load loads the checkpoint with UID `uid` from the storage layer, or returns an error if the checkpoint does not exist
+	Load(uid string) (checkpoint.Checkpoint, error)
+	// CurrentModified returns the last time that the current UID was set
+	CurrentModified() (time.Time, error)
+	// Current returns the source that points to the current checkpoint, or nil if no current checkpoint is set
+	Current() (checkpoint.RemoteConfigSource, error)
+	// LastKnownGood returns the source that points to the last-known-good checkpoint, or nil if no last-known-good checkpoint is set
+	LastKnownGood() (checkpoint.RemoteConfigSource, error)
+	// SetCurrent saves the source that points to the current checkpoint, set to nil to unset
+	SetCurrent(source checkpoint.RemoteConfigSource) error
+	// SetCurrentUpdated is similar to SetCurrent, but also returns whether the current checkpoint changed as a result
+	SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error)
+	// SetLastKnownGood saves the source that points to the last-known-good checkpoint, set to nil to unset
+	SetLastKnownGood(source checkpoint.RemoteConfigSource) error
+	// Reset unsets the current and last-known-good UIDs and returns whether the current UID was unset as a result of the reset
+	Reset() (bool, error)
+}
+
+// reset is a helper for implementing Reset, which can be implemented in terms of Store methods
+func reset(s Store) (bool, error) {
+	if err := s.SetLastKnownGood(nil); err != nil {
+		return false, fmt.Errorf("failed to reset last-known-good UID in checkpoint store, error: %v", err)
+	}
+	updated, err := s.SetCurrentUpdated(nil)
+	if err != nil {
+		return false, fmt.Errorf("failed to reset current UID in checkpoint store, error: %v", err)
+	}
+	return updated, nil
+}
+
+// setCurrentUpdated is a helper for implementing SetCurrentUpdated, which can be implemented in terms of Store methods
+func setCurrentUpdated(s Store, source checkpoint.RemoteConfigSource) (bool, error) {
+	cur, err := s.Current()
+	if err != nil {
+		return false, err
+	}
+
+	// if both are nil, no need to update
+	if cur == nil && source == nil {
+		return false, nil
+	}
+	// if UIDs match, no need to update
+	if (source != nil && cur != nil) && cur.UID() == source.UID() {
+		return false, nil
+	}
+	// update the source
+	if err := s.SetCurrent(source); err != nil {
+		return false, err
+	}
+	return true, nil
+}

pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
+)
+
+func TestReset(t *testing.T) {
+	source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	cases := []struct {
+		s       *fakeStore
+		updated bool
+	}{
+		{&fakeStore{current: nil, lastKnownGood: nil}, false},
+		{&fakeStore{current: source, lastKnownGood: nil}, true},
+		{&fakeStore{current: nil, lastKnownGood: source}, false},
+		{&fakeStore{current: source, lastKnownGood: source}, true},
+		{&fakeStore{current: source, lastKnownGood: otherSource}, true},
+		{&fakeStore{current: otherSource, lastKnownGood: source}, true},
+	}
+	for _, c := range cases {
+		updated, err := reset(c.s)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if c.s.current != nil || c.s.lastKnownGood != nil {
+			t.Errorf("case %q, expect nil for current and last-known-good checkpoints, but still have %q and %q, respectively",
+				spew.Sdump(c.s), c.s.current, c.s.lastKnownGood)
+		}
+		if c.updated != updated {
+			t.Errorf("case %q, expect reset to return %t, but got %t", spew.Sdump(c.s), c.updated, updated)
+		}
+	}
+}
+
+func TestSetCurrentUpdated(t *testing.T) {
+	source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	cases := []struct {
+		s          *fakeStore
+		newCurrent checkpoint.RemoteConfigSource
+		updated    bool
+	}{
+		{&fakeStore{current: nil}, nil, false},
+		{&fakeStore{current: nil}, source, true},
+		{&fakeStore{current: source}, source, false},
+		{&fakeStore{current: source}, nil, true},
+		{&fakeStore{current: source}, otherSource, true},
+	}
+	for _, c := range cases {
+		current := c.s.current
+		updated, err := setCurrentUpdated(c.s, c.newCurrent)
+		if err != nil {
+			t.Fatalf("case %q -> %q, unexpected error: %v", current, c.newCurrent, err)
+		}
+		if c.newCurrent != c.s.current {
+			t.Errorf("case %q -> %q, expect current UID to be %q, but got %q", current, c.newCurrent, c.newCurrent, c.s.current)
+		}
+		if c.updated != updated {
+			t.Errorf("case %q -> %q, expect setCurrentUpdated to return %t, but got %t", current, c.newCurrent, c.updated, updated)
+		}
+	}
+}

pkg/kubelet/kubeletconfig/configfiles/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["configfiles.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/configfiles/configfiles.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package configfiles
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+)
+
+// Loader loads configuration from a storage layer
+type Loader interface {
+	// Load loads and returns the KubeletConfiguration from the storage layer, or an error if a configuration could not be loaded
+	Load() (*componentconfig.KubeletConfiguration, error)
+}
+
+// fsLoader loads configuration from `configDir`
+type fsLoader struct {
+	// fs is the filesystem where the config files exist; can be mocked for testing
+	fs utilfs.Filesystem
+	// configDir is the absolute path to the directory containing the configuration files
+	configDir string
+}
+
+// NewFSLoader returns a Loader that loads a KubeletConfiguration from the files in `configDir`
+func NewFSLoader(fs utilfs.Filesystem, configDir string) Loader {
+	return &fsLoader{
+		fs:        fs,
+		configDir: configDir,
+	}
+}
+
+func (loader *fsLoader) Load() (*componentconfig.KubeletConfiguration, error) {
+	errfmt := fmt.Sprintf("failed to load Kubelet config files from %q, error: ", loader.configDir) + "%v"
+
+	// require the config be in a file called "kubelet"
+	path := filepath.Join(loader.configDir, "kubelet")
+	data, err := loader.fs.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read init config file %q, error: %v", path, err)
+	}
+
+	// no configuration is an error, some parameters are required
+	if len(data) == 0 {
+		return nil, fmt.Errorf(errfmt, fmt.Errorf("config file was empty, but some parameters are required"))
+	}
+
+	return utilcodec.DecodeKubeletConfiguration(data)
+}

pkg/kubelet/kubeletconfig/configsync.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeletconfig
+
+import (
+	"fmt"
+	"os"
+
+	apiv1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+// pokeConfiSourceWorker tells the worker thread that syncs config sources that work needs to be done
+func (cc *Controller) pokeConfigSourceWorker() {
+	select {
+	case cc.pendingConfigSource <- true:
+	default:
+	}
+}
+
+// syncConfigSource checks if work needs to be done to use a new configuration, and does that work if necessary
+func (cc *Controller) syncConfigSource(client clientset.Interface, nodeName string) {
+	select {
+	case <-cc.pendingConfigSource:
+	default:
+		// no work to be done, return
+		return
+	}
+
+	// if the sync fails, we want to retry
+	var syncerr error
+	defer func() {
+		if syncerr != nil {
+			utillog.Errorf(syncerr.Error())
+			cc.pokeConfigSourceWorker()
+		}
+	}()
+
+	node, err := latestNode(cc.informer.GetStore(), nodeName)
+	if err != nil {
+		reason := "unable to read Node from internal object cache"
+		cc.configOK.SetFailedSyncCondition(reason)
+		syncerr = fmt.Errorf("%s, error: %v", reason, err)
+		return
+	}
+
+	// check the Node and download any new config
+	if updated, reason, err := cc.doSyncConfigSource(client, node.Spec.ConfigSource); err != nil {
+		cc.configOK.SetFailedSyncCondition(reason)
+		syncerr = fmt.Errorf("%s, error: %v", reason, err)
+		return
+	} else if updated {
+		// TODO(mtaufen): Consider adding a "currently restarting" node condition for this case
+		utillog.Infof("config updated, Kubelet will restart to begin using new config")
+		os.Exit(0)
+	}
+
+	// If we get here:
+	// - there is no need to restart to update the current config
+	// - there was no error trying to sync configuration
+	// - if, previously, there was an error trying to sync configuration, we need to update to the correct condition
+	errfmt := `sync succeeded but unable to clear "failed to sync" message from ConfigOK, error: %v`
+
+	currentUID := ""
+	if currentSource, err := cc.checkpointStore.Current(); err != nil {
+		utillog.Errorf(errfmt, err)
+		return
+	} else if currentSource != nil {
+		currentUID = currentSource.UID()
+	}
+
+	lkgUID := ""
+	if lkgSource, err := cc.checkpointStore.LastKnownGood(); err != nil {
+		utillog.Errorf(errfmt, err)
+		return
+	} else if lkgSource != nil {
+		lkgUID = lkgSource.UID()
+	}
+
+	currentBadReason := ""
+	if entry, err := cc.badConfigTracker.Entry(currentUID); err != nil {
+		utillog.Errorf(errfmt, err)
+	} else if entry != nil {
+		currentBadReason = entry.Reason
+	}
+	cc.configOK.ClearFailedSyncCondition(currentUID, lkgUID, currentBadReason, cc.initConfig != nil)
+}
+
+// doSyncConfigSource checkpoints and sets the store's current config to the new config or resets config,
+// depending on the `source`, and returns whether the current config in the checkpoint store was updated as a result
+func (cc *Controller) doSyncConfigSource(client clientset.Interface, source *apiv1.NodeConfigSource) (bool, string, error) {
+	if source == nil {
+		utillog.Infof("Node.Spec.ConfigSource is empty, will reset current and last-known-good to defaults")
+		updated, reason, err := cc.resetConfig()
+		if err != nil {
+			return false, reason, err
+		}
+		return updated, "", nil
+	}
+
+	// if the NodeConfigSource is non-nil, download the config
+	utillog.Infof("Node.Spec.ConfigSource is non-empty, will checkpoint source and update config if necessary")
+	remote, reason, err := checkpoint.NewRemoteConfigSource(source)
+	if err != nil {
+		return false, reason, err
+	}
+	reason, err = cc.checkpointConfigSource(client, remote)
+	if err != nil {
+		return false, reason, err
+	}
+	updated, reason, err := cc.setCurrentConfig(remote)
+	if err != nil {
+		return false, reason, err
+	}
+	return updated, "", nil
+}
+
+// checkpointConfigSource downloads and checkpoints the object referred to by `source` if the checkpoint does not already exist,
+// if a failure occurs, returns a sanitized failure reason and an error
+func (cc *Controller) checkpointConfigSource(client clientset.Interface, source checkpoint.RemoteConfigSource) (string, error) {
+	uid := source.UID()
+
+	// if the checkpoint already exists, skip downloading
+	if ok, err := cc.checkpointStore.Exists(uid); err != nil {
+		reason := fmt.Sprintf("unable to determine whether object with UID %q was already checkpointed", uid)
+		return reason, fmt.Errorf("%s, error: %v", reason, err)
+	} else if ok {
+		utillog.Infof("checkpoint already exists for object with UID %q, skipping download", uid)
+		return "", nil
+	}
+
+	// download
+	checkpoint, reason, err := source.Download(client)
+	if err != nil {
+		return reason, fmt.Errorf("%s, error: %v", reason, err)
+	}
+
+	// save
+	err = cc.checkpointStore.Save(checkpoint)
+	if err != nil {
+		reason := fmt.Sprintf("failed to save checkpoint for object with UID %q", checkpoint.UID())
+		return reason, fmt.Errorf("%s, error: %v", reason, err)
+	}
+
+	return "", nil
+}
+
+// setCurrentConfig updates UID of the current checkpoint in the checkpoint store to `uid` and returns whether the
+// current UID changed as a result, or a sanitized failure reason and an error.
+func (cc *Controller) setCurrentConfig(source checkpoint.RemoteConfigSource) (bool, string, error) {
+	updated, err := cc.checkpointStore.SetCurrentUpdated(source)
+	if err != nil {
+		str := "default"
+		if source != nil {
+			str = fmt.Sprintf("object with UID %q", source.UID())
+		}
+		return false, fmt.Sprintf("failed to set current checkpoint to %s", str), err
+	}
+	return updated, "", nil
+}
+
+// resetConfig resets the current and last-known-good checkpoints in the checkpoint store to their default values and
+// returns whether the current checkpoint changed as a result, or a sanitized failure reason and an error.
+func (cc *Controller) resetConfig() (bool, string, error) {
+	updated, err := cc.checkpointStore.Reset()
+	if err != nil {
+		return false, "failed to reset to using local (default or init) config", err
+	}
+	return updated, "", nil
+}
+
+// latestNode returns the most recent Node with `nodeName` from `store`
+func latestNode(store cache.Store, nodeName string) (*apiv1.Node, error) {
+	obj, ok, err := store.GetByKey(nodeName)
+	if err != nil {
+		err := fmt.Errorf("failed to retrieve Node %q from informer's store, error: %v", nodeName, err)
+		utillog.Errorf(err.Error())
+		return nil, err
+	} else if !ok {
+		err := fmt.Errorf("Node %q does not exist in the informer's store, can't sync config source", nodeName)
+		utillog.Errorf(err.Error())
+		return nil, err
+	}
+	node, ok := obj.(*apiv1.Node)
+	if !ok {
+		err := fmt.Errorf("failed to cast object from informer's store to Node, can't sync config source for Node %q", nodeName)
+		utillog.Errorf(err.Error())
+		return nil, err
+	}
+	return node, nil
+}

pkg/kubelet/kubeletconfig/rollback.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeletconfig
+
+import (
+	"fmt"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/apis/componentconfig/validation"
+	"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+// badRollback makes an entry in the bad-config-tracking file for `uid` with `reason`, and returns the result of rolling back to the last-known-good config
+func (cc *Controller) badRollback(uid, reason, detail string) (*componentconfig.KubeletConfiguration, error) {
+	utillog.Errorf(fmt.Sprintf("%s, %s", reason, detail))
+	if err := cc.badConfigTracker.MarkBad(uid, reason); err != nil {
+		return nil, err
+	}
+	return cc.lkgRollback(reason)
+}
+
+// lkgRollback returns a valid last-known-good configuration, and updates the `cc.configOK` condition
+// regarding the `reason` for the rollback, or returns an error if a valid last-known-good could not be produced
+func (cc *Controller) lkgRollback(reason string) (*componentconfig.KubeletConfiguration, error) {
+	utillog.Infof("rolling back to last-known-good config")
+
+	lkgUID := ""
+	if lkgSource, err := cc.checkpointStore.LastKnownGood(); err != nil {
+		return nil, fmt.Errorf("unable to determine last-known-good config, error: %v", err)
+	} else if lkgSource != nil {
+		lkgUID = lkgSource.UID()
+	}
+
+	// if lkgUID indicates the default should be used, return initConfig or defaultConfig
+	if len(lkgUID) == 0 {
+		if cc.initConfig != nil {
+			cc.configOK.Set(status.LkgInitMessage, reason, apiv1.ConditionFalse)
+			return cc.initConfig, nil
+		}
+		cc.configOK.Set(status.LkgDefaultMessage, reason, apiv1.ConditionFalse)
+		return cc.defaultConfig, nil
+	}
+
+	// load
+	checkpoint, err := cc.checkpointStore.Load(lkgUID)
+	if err != nil {
+		return nil, fmt.Errorf("%s, error: %v", fmt.Sprintf(status.LkgFailLoadReasonFmt, lkgUID), err)
+	}
+
+	// parse
+	lkg, err := checkpoint.Parse()
+	if err != nil {
+		return nil, fmt.Errorf("%s, error: %v", fmt.Sprintf(status.LkgFailParseReasonFmt, lkgUID), err)
+	}
+
+	// validate
+	if err := validation.ValidateKubeletConfiguration(lkg); err != nil {
+		return nil, fmt.Errorf("%s, error: %v", fmt.Sprintf(status.LkgFailValidateReasonFmt, lkgUID), err)
+	}
+
+	cc.configOK.Set(fmt.Sprintf(status.LkgRemoteMessageFmt, lkgUID), reason, apiv1.ConditionFalse)
+	return lkg, nil
+}

pkg/kubelet/kubeletconfig/startups/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fstracker.go",
+        "startups.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/apis/componentconfig/validation:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "fstracker_test.go",
+        "startups_test.go",
+    ],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/kubelet/kubeletconfig/util/files:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/test:go_default_library",
+    ],
+)

pkg/kubelet/kubeletconfig/startups/fstracker.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package startups
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+const (
+	startupsFile = "startups.json"
+)
+
+// fsTracker tracks startups in the local filesystem
+type fsTracker struct {
+	// fs is the filesystem to use for storage operations; can be mocked for testing
+	fs utilfs.Filesystem
+	// trackingDir is the absolute path to the storage directory for fsTracker
+	trackingDir string
+}
+
+// NewFsTracker returns a Tracker that will store information in the `trackingDir`
+func NewFsTracker(fs utilfs.Filesystem, trackingDir string) Tracker {
+	return &fsTracker{
+		fs:          fs,
+		trackingDir: trackingDir,
+	}
+}
+
+func (tracker *fsTracker) Initialize() error {
+	utillog.Infof("initializing startups tracking directory %q", tracker.trackingDir)
+	if err := utilfiles.EnsureDir(tracker.fs, tracker.trackingDir); err != nil {
+		return err
+	}
+	if err := utilfiles.EnsureFile(tracker.fs, filepath.Join(tracker.trackingDir, startupsFile)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tracker *fsTracker) RecordStartup() error {
+	// load the file
+	ls, err := tracker.load()
+	if err != nil {
+		return err
+	}
+
+	ls = recordStartup(ls)
+
+	// save the file
+	err = tracker.save(ls)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tracker *fsTracker) StartupsSince(t time.Time) (int32, error) {
+	// load the startups-tracking file
+	ls, err := tracker.load()
+	if err != nil {
+		return 0, err
+	}
+	return startupsSince(ls, t)
+}
+
+// TODO(mtaufen): refactor into encode/decode like in badconfig.go
+
+// load loads the startups-tracking file from disk
+func (tracker *fsTracker) load() ([]string, error) {
+	path := filepath.Join(tracker.trackingDir, startupsFile)
+
+	// load the file
+	b, err := tracker.fs.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load startups-tracking file %q, error: %v", path, err)
+	}
+
+	// parse json into the slice
+	ls := []string{}
+
+	// if the file is empty, just return empty slice
+	if len(b) == 0 {
+		return ls, nil
+	}
+
+	// otherwise unmarshal the json
+	if err := json.Unmarshal(b, &ls); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal json from startups-tracking file %q, error: %v", path, err)
+	}
+	return ls, nil
+}
+
+// save replaces the contents of the startups-tracking file with `ls`
+func (tracker *fsTracker) save(ls []string) error {
+	// marshal the json
+	b, err := json.Marshal(ls)
+	if err != nil {
+		return err
+	}
+	// save the file
+	path := filepath.Join(tracker.trackingDir, startupsFile)
+	if err := utilfiles.ReplaceFile(tracker.fs, path, b); err != nil {
+		return err
+	}
+	return nil
+}

pkg/kubelet/kubeletconfig/startups/fstracker_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package startups
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+const testTrackingDir = "/test-tracking-dir"
+
+// TODO(mtaufen): this file reuses a lot of test code from startups_test.go, should consolidate
+
+func newInitializedFakeFsTracker() (*fsTracker, error) {
+	fs := utilfs.NewFakeFs()
+	tracker := NewFsTracker(fs, testTrackingDir)
+	if err := tracker.Initialize(); err != nil {
+		return nil, err
+	}
+	return tracker.(*fsTracker), nil
+}
+
+func TestFsTrackerInitialize(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("tracker.Initialize() failed with error: %v", err)
+	}
+
+	// check that testTrackingDir exists
+	_, err = tracker.fs.Stat(testTrackingDir)
+	if err != nil {
+		t.Fatalf("expect %q to exist, but stat failed with error: %v", testTrackingDir, err)
+	}
+
+	// check that testTrackingDir contains the startupsFile
+	path := filepath.Join(testTrackingDir, startupsFile)
+	_, err = tracker.fs.Stat(path)
+	if err != nil {
+		t.Fatalf("expect %q to exist, but stat failed with error: %v", path, err)
+	}
+}
+
+func TestFsTrackerRecordStartup(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	now := time.Now()
+
+	fullList := func() []string {
+		ls := []string{}
+		for i := maxStartups; i > 0; i-- {
+			// subtract decreasing amounts so timestamps increase but remain in the past
+			ls = append(ls, now.Add(-time.Duration(i)*time.Second).Format(time.RFC3339))
+		}
+		return ls
+	}()
+	cases := []struct {
+		desc       string
+		ls         []string
+		expectHead []string // what we expect the first length-1 elements to look like after recording a new timestamp
+		expectLen  int      // how long the list should be after recording
+	}{
+		// start empty
+		{
+			"start empty",
+			[]string{},
+			[]string{},
+			1,
+		},
+		// start non-empty
+		{
+			"start non-empty",
+			// subtract 1 so stamps are in the past
+			[]string{now.Add(-1 * time.Second).Format(time.RFC3339)},
+			[]string{now.Add(-1 * time.Second).Format(time.RFC3339)},
+			2,
+		},
+		// rotate list
+		{
+			"rotate list",
+			// make a slice with len == maxStartups, containing monotonically-increasing timestamps
+			fullList,
+			fullList[1:],
+			maxStartups,
+		},
+	}
+
+	for _, c := range cases {
+		// save the starting point, record a "startup" time, then load list from fs
+		if err := tracker.save(c.ls); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if err := tracker.RecordStartup(); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		ls, err := tracker.load()
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+
+		if c.expectLen != len(ls) {
+			t.Errorf("case %q, expected list %q to have length %d", c.desc, ls, c.expectLen)
+		}
+		if !reflect.DeepEqual(c.expectHead, ls[:len(ls)-1]) {
+			t.Errorf("case %q, expected elements 0 through n-1 of list %q to equal %q", c.desc, ls, c.expectHead)
+		}
+		// timestamps should be monotonically increasing (assuming system clock isn't jumping around at least)
+		if sorted, err := timestampsSorted(ls); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		} else if !sorted {
+			t.Errorf("case %q, expected monotonically increasing timestamps, but got %q", c.desc, ls)
+		}
+	}
+}
+
+func TestFsTrackerStartupsSince(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	now, err := time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	cases := []struct {
+		desc   string
+		ls     []string
+		expect int32
+		err    string
+	}{
+		// empty list
+		{"empty list", []string{}, 0, ""},
+		// no startups since
+		{
+			"no startups since",
+			[]string{"2014-01-02T15:04:05Z", "2015-01-02T15:04:05Z", "2016-01-02T15:04:05Z"},
+			0,
+			"",
+		},
+		// 2 startups since
+		{
+			"some startups since",
+			[]string{"2016-01-02T15:04:05Z", "2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z"},
+			2,
+			"",
+		},
+		// all startups since
+		{
+			"all startups since",
+			[]string{"2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z", "2020-01-02T15:04:05Z"},
+			3,
+			"",
+		},
+		// invalid timestamp
+		{"invalid timestamp", []string{"2018-01-02T15:04:05Z08:00"}, 0, "failed to parse"},
+	}
+
+	for _, c := range cases {
+		if err := tracker.save(c.ls); err != nil {
+			t.Fatalf("unexected error: %v", err)
+		}
+		num, err := tracker.StartupsSince(now)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		if num != c.expect {
+			t.Errorf("case %q, expect %d startups but got %d", c.desc, c.expect, num)
+		}
+	}
+}
+
+func TestFsTrackerLoad(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	nowstamp := time.Now().Format(time.RFC3339)
+	cases := []struct {
+		desc   string
+		data   []byte
+		expect []string
+		err    string
+	}{
+		// empty file
+		{"empty file", []byte(""), []string{}, ""},
+		// empty list
+		{"empty list", []byte("[]"), []string{}, ""},
+		// valid json
+		{"valid json", []byte(fmt.Sprintf(`["%s"]`, nowstamp)), []string{nowstamp}, ""},
+		// invalid json
+		{"invalid json", []byte(`*`), []string{}, "failed to unmarshal"},
+	}
+
+	for _, c := range cases {
+		// save a file containing the correct serialization
+		utilfiles.ReplaceFile(tracker.fs, filepath.Join(testTrackingDir, startupsFile), c.data)
+
+		// loading valid json should result in an object with the correct serialization
+		ls, err := tracker.load()
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		if !reflect.DeepEqual(c.expect, ls) {
+			// ls should equal expected decoded object
+			t.Errorf("case %q, expect %#v but got %#v", c.desc, c.expect, ls)
+		}
+	}
+
+}
+
+func TestFsTrackerSave(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	nowstamp := time.Now().Format(time.RFC3339)
+	cases := []struct {
+		desc   string
+		ls     []string
+		expect string
+		err    string
+	}{
+		// empty list
+		{"empty list", []string{}, "[]", ""},
+		// 1-entry list
+		{"valid json", []string{nowstamp}, fmt.Sprintf(`["%s"]`, nowstamp), ""},
+	}
+
+	for _, c := range cases {
+		if err := tracker.save(c.ls); utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+
+		data, err := tracker.fs.ReadFile(filepath.Join(testTrackingDir, startupsFile))
+		if err != nil {
+			t.Fatalf("failed to read startups file, error: %v", err)
+		}
+		json := string(data)
+
+		if json != c.expect {
+			t.Errorf("case %q, expect %q but got %q", c.desc, c.expect, json)
+		}
+	}
+}
+
+func TestFsTrackerRoundTrip(t *testing.T) {
+	tracker, err := newInitializedFakeFsTracker()
+	if err != nil {
+		t.Fatalf("failed to construct a tracker, error: %v", err)
+	}
+
+	nowstamp := time.Now().Format(time.RFC3339)
+	expect := []string{nowstamp}
+
+	// test that saving and loading an object results in the same value
+	err = tracker.save(expect)
+	if err != nil {
+		t.Fatalf("failed to save startups data, error: %v", err)
+	}
+	after, err := tracker.load()
+	if err != nil {
+		t.Fatalf("failed to load startups data, error: %v", err)
+	}
+	if !reflect.DeepEqual(expect, after) {
+		t.Errorf("expect round-tripping %#v to result in the same value, but got %#v", expect, after)
+	}
+}

pkg/kubelet/kubeletconfig/startups/startups.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package startups
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/kubernetes/pkg/apis/componentconfig/validation"
+)
+
+const (
+	// we allow one extra startup to account for the startup necessary to update configuration
+	maxStartups = validation.MaxCrashLoopThreshold + 1
+)
+
+// Tracker tracks Kubelet startups in a storage layer
+type Tracker interface {
+	// Initialize sets up the storage layer
+	Initialize() error
+	// RecordStartup records the current time as a Kubelet startup
+	RecordStartup() error
+	// StartupsSince returns the number of Kubelet startus recorded since `t`
+	StartupsSince(t time.Time) (int32, error)
+}
+
+func startupsSince(ls []string, start time.Time) (int32, error) {
+	// since the list is append-only we only need to count the number of timestamps since `t`
+	startups := int32(0)
+	for _, stamp := range ls {
+		t, err := time.Parse(time.RFC3339, stamp)
+		if err != nil {
+			return 0, fmt.Errorf("failed to parse timestamp while counting startups, error: %v", err)
+		}
+		if t.After(start) {
+			startups++
+		}
+	}
+	return startups, nil
+}
+
+func recordStartup(ls []string) []string {
+	// record current time
+	now := time.Now()
+	stamp := now.Format(time.RFC3339) // use RFC3339 time format
+	ls = append(ls, stamp)
+
+	// rotate the slice if necessary
+	if len(ls) > maxStartups {
+		ls = ls[1:]
+	}
+
+	// return the new slice
+	return ls
+}

pkg/kubelet/kubeletconfig/startups/startups_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package startups
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
+)
+
+func TestRecordStartup(t *testing.T) {
+	now := time.Now()
+
+	fullList := func() []string {
+		ls := []string{}
+		for i := maxStartups; i > 0; i-- {
+			// subtract decreasing amounts so timestamps increase but remain in the past
+			ls = append(ls, now.Add(-time.Duration(i)*time.Second).Format(time.RFC3339))
+		}
+		return ls
+	}()
+	cases := []struct {
+		desc       string
+		ls         []string
+		expectHead []string // what we expect the first length-1 elements to look like after recording a new timestamp
+		expectLen  int      // how long the list should be after recording
+	}{
+		// start empty
+		{
+			"start empty",
+			[]string{},
+			[]string{},
+			1,
+		},
+		// start non-empty
+		{
+			"start non-empty",
+			// subtract 1 so stamps are in the past
+			[]string{now.Add(-1 * time.Second).Format(time.RFC3339)},
+			[]string{now.Add(-1 * time.Second).Format(time.RFC3339)},
+			2,
+		},
+		// rotate list
+		{
+			"rotate list",
+			// make a slice with len == maxStartups, containing monotonically-increasing timestamps
+			fullList,
+			fullList[1:],
+			maxStartups,
+		},
+	}
+
+	for _, c := range cases {
+		ls := recordStartup(c.ls)
+		if c.expectLen != len(ls) {
+			t.Errorf("case %q, expected list %q to have length %d", c.desc, ls, c.expectLen)
+		}
+		if !reflect.DeepEqual(c.expectHead, ls[:len(ls)-1]) {
+			t.Errorf("case %q, expected elements 0 through n-1 of list %q to equal %q", c.desc, ls, c.expectHead)
+		}
+		// timestamps should be monotonically increasing (assuming system clock isn't jumping around at least)
+		if sorted, err := timestampsSorted(ls); err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		} else if !sorted {
+			t.Errorf("case %q, expected monotonically increasing timestamps, but got %q", c.desc, ls)
+		}
+	}
+}
+
+func TestStartupsSince(t *testing.T) {
+	now, err := time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	cases := []struct {
+		desc   string
+		ls     []string
+		expect int32
+		err    string
+	}{
+		// empty list
+		{"empty list", []string{}, 0, ""},
+		// no startups since
+		{
+			"no startups since",
+			[]string{"2014-01-02T15:04:05Z", "2015-01-02T15:04:05Z", "2016-01-02T15:04:05Z"},
+			0,
+			"",
+		},
+		// 2 startups since
+		{
+			"some startups since",
+			[]string{"2016-01-02T15:04:05Z", "2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z"},
+			2,
+			"",
+		},
+		// all startups since
+		{
+			"all startups since",
+			[]string{"2018-01-02T15:04:05Z", "2019-01-02T15:04:05Z", "2020-01-02T15:04:05Z"},
+			3,
+			"",
+		},
+		// invalid timestamp
+		{"invalid timestamp", []string{"2018-01-02T15:04:05Z08:00"}, 0, "failed to parse"},
+	}
+
+	for _, c := range cases {
+		num, err := startupsSince(c.ls, now)
+		if utiltest.SkipRest(t, c.desc, err, c.err) {
+			continue
+		}
+		if num != c.expect {
+			t.Errorf("case %q, expect %d startups but got %d", c.desc, c.expect, num)
+		}
+	}
+
+}
+
+// returns true if the timestamps are monotically increasing, false otherwise
+func timestampsSorted(ls []string) (bool, error) {
+	if len(ls) < 2 {
+		return true, nil
+	}
+	prev, err := time.Parse(time.RFC3339, ls[0])
+	if err != nil {
+		return false, err
+	}
+	for _, stamp := range ls[1:] {
+		cur, err := time.Parse(time.RFC3339, stamp)
+		if err != nil {
+			return false, err
+		}
+		if !cur.After(prev) {
+			return false, nil
+		}
+		prev = cur
+	}
+	return true, nil
+}

pkg/kubelet/kubeletconfig/status/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["status.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/equal:go_default_library",
+        "//pkg/kubelet/kubeletconfig/util/log:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/codec/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["codec.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/install:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/install:go_default_library",
+        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/codec/codec.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package codec
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/kubernetes/pkg/api"
+	// ensure the core apis are installed
+	_ "k8s.io/kubernetes/pkg/api/install"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	// ensure the componentconfig api group is installed
+	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install"
+	ccv1a1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+)
+
+// TODO(mtaufen): allow an encoder to be injected into checkpoint objects at creation time? (then we could ultimately instantiate only one encoder)
+
+// NewJSONEncoder generates a new runtime.Encoder that encodes objects to JSON
+func NewJSONEncoder(groupName string) (runtime.Encoder, error) {
+	// encode to json
+	mediaType := "application/json"
+	info, ok := runtime.SerializerInfoForMediaType(api.Codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return nil, fmt.Errorf("unsupported media type %q", mediaType)
+	}
+
+	versions := api.Registry.EnabledVersionsForGroup(groupName)
+	if len(versions) == 0 {
+		return nil, fmt.Errorf("no enabled versions for group %q", groupName)
+	}
+
+	// the "best" version supposedly comes first in the list returned from api.Registry.EnabledVersionsForGroup
+	return api.Codecs.EncoderForVersion(info.Serializer, versions[0]), nil
+}
+
+// DecodeKubeletConfiguration decodes an encoded (v1alpha1) KubeletConfiguration object to the internal type
+func DecodeKubeletConfiguration(data []byte) (*componentconfig.KubeletConfiguration, error) {
+	// TODO(mtaufen): when KubeletConfiguration moves out of componentconfig, will the UniversalDecoder still work?
+	// decode the object, note we use the external version scheme to decode, because users provide the external version
+	obj, err := runtime.Decode(api.Codecs.UniversalDecoder(ccv1a1.SchemeGroupVersion), data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode, error: %v", err)
+	}
+
+	externalKC, ok := obj.(*ccv1a1.KubeletConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("failed to cast object to KubeletConfiguration, object: %#v", obj)
+	}
+
+	// TODO(mtaufen): confirm whether api.Codecs.UniversalDecoder runs the defaulting, which would make this redundant
+	// run the defaulter on the decoded configuration before converting to internal type
+	api.Scheme.Default(externalKC)
+
+	// convert to internal type
+	internalKC := &componentconfig.KubeletConfiguration{}
+	err = api.Scheme.Convert(externalKC, internalKC, nil)
+	if err != nil {
+		return nil, err
+	}
+	return internalKC, nil
+}

pkg/kubelet/kubeletconfig/util/equal/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["equal.go"],
+    tags = ["automanaged"],
+    deps = ["//vendor/k8s.io/api/core/v1:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/equal/equal.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package equal
+
+import apiv1 "k8s.io/api/core/v1"
+
+// ConfigSourceEq returns true if the two config sources are semantically equivalent in the context of dynamic config
+func ConfigSourceEq(a, b *apiv1.NodeConfigSource) bool {
+	if a == b {
+		return true
+	} else if a == nil || b == nil {
+		// not equal, and one is nil
+		return false
+	}
+	// check equality of config source subifelds
+	if a.ConfigMapRef != b.ConfigMapRef {
+		return ObjectRefEq(a.ConfigMapRef, b.ConfigMapRef)
+	}
+	// all internal subfields of the config soruce are equal
+	return true
+}
+
+// ObjectRefEq returns true if the two object references are semantically equivalent in the context of dynamic config
+func ObjectRefEq(a, b *apiv1.ObjectReference) bool {
+	if a == b {
+		return true
+	} else if a == nil || b == nil {
+		// not equal, and one is nil
+		return false
+	}
+	return a.UID == b.UID && a.Namespace == b.Namespace && a.Name == b.Name
+}
+
+// ConfigOKEq returns true if the two conditions are semantically equivalent in the context of dynamic config
+func ConfigOKEq(a, b *apiv1.NodeCondition) bool {
+	return a.Message == b.Message && a.Reason == b.Reason && a.Status == b.Status
+}

pkg/kubelet/kubeletconfig/util/files/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["files.go"],
+    tags = ["automanaged"],
+    deps = ["//pkg/kubelet/kubeletconfig/util/filesystem:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/files/files.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package files
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	utilfs "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/filesystem"
+)
+
+const defaultPerm = 0666
+
+// FileExists returns true if a regular file exists at `path`, false if `path` does not exist, otherwise an error
+func FileExists(fs utilfs.Filesystem, path string) (bool, error) {
+	if info, err := fs.Stat(path); err == nil {
+		if info.Mode().IsRegular() {
+			return true, nil
+		}
+		return false, fmt.Errorf("expected regular file at %q, but mode is %q", path, info.Mode().String())
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else {
+		return false, err
+	}
+}
+
+// EnsureFile ensures that a regular file exists at `path`, and if it must create the file any
+// necessary parent directories will also be created and the new file will be empty.
+func EnsureFile(fs utilfs.Filesystem, path string) error {
+	// if file exists, don't change it, but do report any unexpected errors
+	if ok, err := FileExists(fs, path); ok || err != nil {
+		return err
+	} // Assert: file does not exist
+
+	// create any necessary parents
+	err := fs.MkdirAll(filepath.Dir(path), defaultPerm)
+	if err != nil {
+		return err
+	}
+
+	// create the file
+	file, err := fs.Create(path)
+	if err != nil {
+		return err
+	}
+	// close the file, since we don't intend to use it yet
+	return file.Close()
+}
+
+// ReplaceFile replaces the contents of the file at `path` with `data` by writing to a tmp file in the same
+// dir as `path` and renaming the tmp file over `path`. The file does not have to exist to use ReplaceFile.
+func ReplaceFile(fs utilfs.Filesystem, path string, data []byte) error {
+	dir := filepath.Dir(path)
+	prefix := filepath.Base(path)
+
+	// create the tmp file
+	tmpFile, err := fs.TempFile(dir, prefix)
+	if err != nil {
+		return err
+	}
+
+	// Name() will be an absolute path when using utilfs.DefaultFS, because ioutil.TempFile passes
+	// an absolute path to os.Open, and we ensure similar behavior in utilfs.FakeFS for testing.
+	tmpPath := tmpFile.Name()
+
+	// write data
+	if _, err := tmpFile.Write(data); err != nil {
+		return err
+	}
+	if err := tmpFile.Close(); err != nil {
+		return err
+	}
+
+	// rename over existing file
+	if err := fs.Rename(tmpPath, path); err != nil {
+		return err
+	}
+	return nil
+}
+
+// DirExists returns true if a directory exists at `path`, false if `path` does not exist, otherwise an error
+func DirExists(fs utilfs.Filesystem, path string) (bool, error) {
+	if info, err := fs.Stat(path); err == nil {
+		if info.IsDir() {
+			return true, nil
+		}
+		return false, fmt.Errorf("expected dir at %q, but mode is is %q", path, info.Mode().String())
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else {
+		return false, err
+	}
+}
+
+// EnsureDir ensures that a directory exists at `path`, and if it must create the directory any
+// necessary parent directories will also be created and the new directory will be empty.
+func EnsureDir(fs utilfs.Filesystem, path string) error {
+	// if dir exists, don't change it, but do report any unexpected errors
+	if ok, err := DirExists(fs, path); ok || err != nil {
+		return err
+	} // Assert: dir does not exist
+
+	// create the dir
+	if err := fs.MkdirAll(path, defaultPerm); err != nil {
+		return err
+	}
+	return nil
+}

pkg/kubelet/kubeletconfig/util/filesystem/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "defaultfs.go",
+        "fakefs.go",
+        "filesystem.go",
+    ],
+    tags = ["automanaged"],
+    deps = ["//vendor/github.com/spf13/afero:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/filesystem/defaultfs.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+// DefaultFs implements Filesystem using same-named functions from "os" and "io/ioutil"
+type DefaultFs struct{}
+
+// Stat via os.Stat
+func (DefaultFs) Stat(name string) (os.FileInfo, error) {
+	return os.Stat(name)
+}
+
+// Create via os.Create
+func (DefaultFs) Create(name string) (File, error) {
+	file, err := os.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &defaultFile{file}, nil
+}
+
+// Rename via os.Rename
+func (DefaultFs) Rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}
+
+// MkdirAll via os.MkdirAll
+func (DefaultFs) MkdirAll(path string, perm os.FileMode) error {
+	return os.MkdirAll(path, perm)
+}
+
+// Chtimes via os.Chtimes
+func (DefaultFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return os.Chtimes(name, atime, mtime)
+}
+
+// ReadFile via os.ReadFile
+func (DefaultFs) ReadFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}
+
+// TempFile via os.TempFile
+func (DefaultFs) TempFile(dir, prefix string) (File, error) {
+	file, err := ioutil.TempFile(dir, prefix)
+	if err != nil {
+		return nil, err
+	}
+	return &defaultFile{file}, nil
+}
+
+// defaultFile implements File using same-named functions from "os"
+type defaultFile struct {
+	file *os.File
+}
+
+// Name via os.File.Name
+func (file *defaultFile) Name() string {
+	return file.file.Name()
+}
+
+// Write via os.File.Write
+func (file *defaultFile) Write(b []byte) (n int, err error) {
+	return file.file.Write(b)
+}
+
+// Close via os.File.Close
+func (file *defaultFile) Close() error {
+	return file.file.Close()
+}

pkg/kubelet/kubeletconfig/util/filesystem/fakefs.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filesystem
+
+import (
+	"os"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+// fakeFs is implemented in terms of afero
+type fakeFs struct {
+	a afero.Afero
+}
+
+// NewFakeFs returns a fake Filesystem that exists in-memory, useful for unit tests
+func NewFakeFs() Filesystem {
+	return &fakeFs{a: afero.Afero{Fs: afero.NewMemMapFs()}}
+}
+
+// Stat via afero.Fs.Stat
+func (fs *fakeFs) Stat(name string) (os.FileInfo, error) {
+	return fs.a.Fs.Stat(name)
+}
+
+// Create via afero.Fs.Create
+func (fs *fakeFs) Create(name string) (File, error) {
+	file, err := fs.a.Fs.Create(name)
+	if err != nil {
+		return nil, err
+	}
+	return &fakeFile{file}, nil
+}
+
+// Rename via afero.Fs.Rename
+func (fs *fakeFs) Rename(oldpath, newpath string) error {
+	return fs.a.Fs.Rename(oldpath, newpath)
+}
+
+// MkdirAll via afero.Fs.MkdirAll
+func (fs *fakeFs) MkdirAll(path string, perm os.FileMode) error {
+	return fs.a.Fs.MkdirAll(path, perm)
+}
+
+// Chtimes via afero.Fs.Chtimes
+func (fs *fakeFs) Chtimes(name string, atime time.Time, mtime time.Time) error {
+	return fs.a.Fs.Chtimes(name, atime, mtime)
+}
+
+// ReadFile via afero.Fs.ReadFile
+func (fs *fakeFs) ReadFile(filename string) ([]byte, error) {
+	return fs.a.ReadFile(filename)
+}
+
+// TempFile via afero.Fs.TempFile
+func (fs *fakeFs) TempFile(dir, prefix string) (File, error) {
+	file, err := fs.a.TempFile(dir, prefix)
+	if err != nil {
+		return nil, err
+	}
+	return &fakeFile{file}, nil
+}
+
+// fakeFile implements File; for use with fakeFs
+type fakeFile struct {
+	file afero.File
+}
+
+// Name via afero.File.Name
+func (file *fakeFile) Name() string {
+	return file.file.Name()
+}
+
+// Write via afero.File.Write
+func (file *fakeFile) Write(b []byte) (n int, err error) {
+	return file.file.Write(b)
+}
+
+// Close via afero.File.Close
+func (file *fakeFile) Close() error {
+	return file.file.Close()
+}

pkg/kubelet/kubeletconfig/util/filesystem/filesystem.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filesystem
+
+import (
+	"os"
+	"time"
+)
+
+// Filesystem is an interface that we can use to mock various filesystem operations
+type Filesystem interface {
+	// from "os"
+	Stat(name string) (os.FileInfo, error)
+	Create(name string) (File, error)
+	Rename(oldpath, newpath string) error
+	MkdirAll(path string, perm os.FileMode) error
+	Chtimes(name string, atime time.Time, mtime time.Time) error
+
+	// from "io/ioutil"
+	ReadFile(filename string) ([]byte, error)
+	TempFile(dir, prefix string) (File, error)
+}
+
+// File is an interface that we can use to mock various filesystem operations typically
+// accessed through the File object from the "os" package
+type File interface {
+	// for now, the only os.File methods used are those below, add more as necessary
+	Name() string
+	Write(b []byte) (n int, err error)
+	Close() error
+}

pkg/kubelet/kubeletconfig/util/log/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["log.go"],
+    tags = ["automanaged"],
+    deps = ["//vendor/github.com/golang/glog:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/log/log.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package log
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+)
+
+const logFmt = "kubelet config controller: %s"
+
+// Errorf shim that inserts "kubelet config controller" at the beginning of the log message,
+// while still reporting the call site of the logging function.
+func Errorf(format string, args ...interface{}) {
+	var s string
+	if len(args) > 0 {
+		s = fmt.Sprintf(format, args...)
+	} else {
+		s = format
+	}
+	glog.ErrorDepth(1, fmt.Sprintf(logFmt, s))
+}
+
+// Infof shim that inserts "kubelet config controller" at the beginning of the log message,
+// while still reporting the call site of the logging function.
+func Infof(format string, args ...interface{}) {
+	var s string
+	if len(args) > 0 {
+		s = fmt.Sprintf(format, args...)
+	} else {
+		s = format
+	}
+	glog.InfoDepth(1, fmt.Sprintf(logFmt, s))
+}

pkg/kubelet/kubeletconfig/util/panic/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["panic.go"],
+    tags = ["automanaged"],
+    deps = ["//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/panic/panic.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package panic
+
+import utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+// HandlePanic returns a function that wraps `fn` with the utilruntime.PanicHandlers, and continues
+// to bubble the panic after the PanicHandlers are called
+func HandlePanic(fn func()) func() {
+	return func() {
+		defer func() {
+			if r := recover(); r != nil {
+				for _, fn := range utilruntime.PanicHandlers {
+					fn(r)
+				}
+				panic(r)
+			}
+		}()
+		// call the function
+		fn()
+	}
+}

pkg/kubelet/kubeletconfig/util/test/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["test.go"],
+    tags = ["automanaged"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/kubeletconfig/util/test/test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package test
+
+import (
+	"strings"
+	"testing"
+)
+
+// SkipRest returns true if there was a non-nil error or if we expected an error that didn't happen,
+// and logs the appropriate error on the test object.
+// The return value indicates whether we should skip the rest of the test case due to the error result.
+func SkipRest(t *testing.T, desc string, err error, contains string) bool {
+	if err != nil {
+		if len(contains) == 0 {
+			t.Errorf("case %q, expect nil error but got %q", desc, err.Error())
+		} else if !strings.Contains(err.Error(), contains) {
+			t.Errorf("case %q, expect error to contain %q but got %q", desc, contains, err.Error())
+		}
+		return true
+	} else if len(contains) > 0 {
+		t.Errorf("case %q, expect error to contain %q but got nil error", desc, contains)
+		return true
+	}
+	return false
+}

pkg/kubelet/kubeletconfig/watch.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeletconfig
+
+import (
+	"math/rand"
+	"time"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	kuberuntime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	utilequal "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/equal"
+	utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
+)
+
+// newSharedNodeInformer returns a shared informer that uses `client` to watch the Node with
+// `nodeName` for changes and respond with `addFunc`, `updateFunc`, and `deleteFunc`.
+func newSharedNodeInformer(client clientset.Interface, nodeName string,
+	addFunc func(newObj interface{}),
+	updateFunc func(oldObj interface{}, newObj interface{}),
+	deleteFunc func(deletedObj interface{})) cache.SharedInformer {
+	// select nodes by name
+	fieldselector := fields.OneTermEqualSelector("metadata.name", nodeName)
+
+	// add some randomness to resync period, which can help avoid controllers falling into lock-step
+	minResyncPeriod := 15 * time.Minute
+	factor := rand.Float64() + 1
+	resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor)
+
+	lw := &cache.ListWatch{
+		ListFunc: func(options metav1.ListOptions) (kuberuntime.Object, error) {
+			return client.Core().Nodes().List(metav1.ListOptions{
+				FieldSelector: fieldselector.String(),
+			})
+		},
+		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+			return client.Core().Nodes().Watch(metav1.ListOptions{
+				FieldSelector:   fieldselector.String(),
+				ResourceVersion: options.ResourceVersion,
+			})
+		},
+	}
+
+	handler := cache.ResourceEventHandlerFuncs{
+		AddFunc:    addFunc,
+		UpdateFunc: updateFunc,
+		DeleteFunc: deleteFunc,
+	}
+
+	informer := cache.NewSharedInformer(lw, &apiv1.Node{}, resyncPeriod)
+	informer.AddEventHandler(handler)
+
+	return informer
+}
+
+// onAddNodeEvent calls onUpdateNodeEvent with the new object and a nil old object
+func (cc *Controller) onAddNodeEvent(newObj interface{}) {
+	cc.onUpdateNodeEvent(nil, newObj)
+}
+
+// onUpdateNodeEvent checks whether the configSource changed between oldObj and newObj, and pokes the
+// configuration sync worker if there was a change
+func (cc *Controller) onUpdateNodeEvent(oldObj interface{}, newObj interface{}) {
+	newNode, ok := newObj.(*apiv1.Node)
+	if !ok {
+		utillog.Errorf("failed to cast new object to Node, couldn't handle event")
+		return
+	}
+	if oldObj == nil {
+		// Node was just added, need to sync
+		cc.pokeConfigSourceWorker()
+		return
+	}
+	oldNode, ok := oldObj.(*apiv1.Node)
+	if !ok {
+		utillog.Errorf("failed to cast old object to Node, couldn't handle event")
+		return
+	}
+	if !utilequal.ConfigSourceEq(oldNode.Spec.ConfigSource, newNode.Spec.ConfigSource) {
+		cc.pokeConfigSourceWorker()
+	}
+}
+
+// onDeleteNodeEvent logs a message if the Node was deleted and may log errors
+// if an unexpected DeletedFinalStateUnknown was received.
+// We allow the sync-loop to continue, because it is possible that the Kubelet detected
+// a Node with unexpected externalID and is attempting to delete and re-create the Node
+// (see pkg/kubelet/kubelet_node_status.go), or that someone accidently deleted the Node
+// (the Kubelet will re-create it).
+func (cc *Controller) onDeleteNodeEvent(deletedObj interface{}) {
+	node, ok := deletedObj.(*apiv1.Node)
+	if !ok {
+		tombstone, ok := deletedObj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			utillog.Errorf("couldn't cast deleted object to DeletedFinalStateUnknown, object: %+v", deletedObj)
+			return
+		}
+		node, ok = tombstone.Obj.(*apiv1.Node)
+		if !ok {
+			utillog.Errorf("received DeletedFinalStateUnknown object but it did not contain a Node, object: %+v", deletedObj)
+			return
+		}
+		utillog.Infof("Node was deleted (DeletedFinalStateUnknown), sync-loop will continue because the Kubelet might recreate the Node, node: %+v", node)
+		return
+	}
+	utillog.Infof("Node was deleted, sync-loop will continue because the Kubelet might recreate the Node, node: %+v", node)
+}

pkg/kubemark/hollow_kubelet.go

@@ -110,6 +110,7 @@ func GetHollowKubeletConfig(
 
 	// Flags struct
 	f := &options.KubeletFlags{
+		RootDirectory:    testRootDir,
 		HostnameOverride: nodeName,
 		// Use the default runtime options.
 		ContainerRuntimeOptions: *options.NewContainerRuntimeOptions(),
@@ -123,7 +124,6 @@ func GetHollowKubeletConfig(
 	c := &componentconfig.KubeletConfiguration{}
 	api.Scheme.Convert(tmp, c, nil)
 
-	c.RootDirectory = testRootDir
 	c.ManifestURL = ""
 	c.Address = "0.0.0.0" /* bind address */
 	c.Port = int32(kubeletPort)

plugin/pkg/admission/noderestriction/BUILD

@@ -20,6 +20,7 @@ go_library(
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/clientset_generated/internalclientset/typed/core/internalversion:go_default_library",
         "//pkg/kubeapiserver/admission:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",

plugin/pkg/admission/noderestriction/admission.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/admission"
@@ -242,18 +243,44 @@ func (c *nodePlugin) admitPodEviction(nodeName string, a admission.Attributes) e
 
 func (c *nodePlugin) admitNode(nodeName string, a admission.Attributes) error {
 	requestedName := a.GetName()
-
-	// On create, get name from new object if unset in admission
-	if len(requestedName) == 0 && a.GetOperation() == admission.Create {
+	if a.GetOperation() == admission.Create {
 		node, ok := a.GetObject().(*api.Node)
 		if !ok {
 			return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
 		}
-		requestedName = node.Name
-	}
 
+		// Don't allow a node to create its Node API object with the config source set.
+		// We scope node access to things listed in the Node.Spec, so allowing this would allow a view escalation.
+		if node.Spec.ConfigSource != nil {
+			return admission.NewForbidden(a, fmt.Errorf("cannot create with non-nil configSource"))
+		}
+
+		// On create, get name from new object if unset in admission
+		if len(requestedName) == 0 {
+			requestedName = node.Name
+		}
+	}
 	if requestedName != nodeName {
 		return admission.NewForbidden(a, fmt.Errorf("node %q cannot modify node %q", nodeName, requestedName))
 	}
+
+	if a.GetOperation() == admission.Update {
+		node, ok := a.GetObject().(*api.Node)
+		if !ok {
+			return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
+		}
+		oldNode, ok := a.GetOldObject().(*api.Node)
+		if !ok {
+			return admission.NewForbidden(a, fmt.Errorf("unexpected type %T", a.GetObject()))
+		}
+
+		// Don't allow a node to update the config source on its Node API object.
+		// We scope node access to things listed in the Node.Spec, so allowing this would allow a view escalation.
+		// We only do the check if the new node's configSource is non-nil; old kubelets might drop the field during a status update.
+		if node.Spec.ConfigSource != nil && !apiequality.Semantic.DeepEqual(node.Spec.ConfigSource, oldNode.Spec.ConfigSource) {
+			return admission.NewForbidden(a, fmt.Errorf("cannot update configSource to a new non-nil configSource"))
+		}
+	}
+
 	return nil
 }