Skip to content

K
k3s
Commit fcc1f8e7 authored by Michael Taufen's avatar Michael Taufen
Browse files
Options

Move to a structured status for dynamic Kubelet config

Updates dynamic Kubelet config to use a structured status, rather than a node condition. This makes the status machine-readable, and thus more useful for config orchestration. Fixes: #56896
parent 7e75a09d
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 560 additions and 152 deletions
api/openapi-spec/swagger.json
View file @ fcc1f8e7
......@@ -77278,6 +77278,27 @@
}
}
},
"io.k8s.api.core.v1.NodeConfigStatus": {
"description": "NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.",
"properties": {
"active": {
"description": "Active reports the checkpointed config the node is actively using. Active will represent either the current version of the Assigned config, or the current LastKnownGood config, depending on whether attempting to use the Assigned config results in an error.",
"$ref": "#/definitions/io.k8s.api.core.v1.NodeConfigSource"
},
"assigned": {
"description": "Assigned reports the checkpointed config the node will try to use. When Node.Spec.ConfigSource is updated, the node checkpoints the associated config payload to local disk, along with a record indicating intended config. The node refers to this record to choose its config checkpoint, and reports this record in Assigned. Assigned only updates in the status after the record has been checkpointed to disk. When the Kubelet is restarted, it tries to make the Assigned config the Active config by loading and validating the checkpointed payload identified by Assigned.",
"$ref": "#/definitions/io.k8s.api.core.v1.NodeConfigSource"
},
"error": {
"description": "Error describes any problems reconciling the Spec.ConfigSource to the Active config. Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting to load or validate the Assigned config, etc. Errors may occur at different points while syncing config. Earlier errors (e.g. download or checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error by fixing the config assigned in Spec.ConfigSource. You can find additional information for debugging by searching the error message in the Kubelet log. Error is a human-readable description of the error state; machines can check whether or not Error is empty, but should not rely on the stability of the Error text across Kubelet versions.",
"type": "string"
},
"lastKnownGood": {
"description": "LastKnownGood reports the checkpointed config the node will fall back to when it encounters an error attempting to use the Assigned config. The Assigned config becomes the LastKnownGood config when the node determines that the Assigned config is stable and correct. This is currently implemented as a 10-minute soak period starting when the local record of Assigned config is updated. If the Assigned config is Active at the end of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil, because the local default config is always assumed good. You should not make assumptions about the node's method of determining config stability and correctness, as this may change or become configurable in the future.",
"$ref": "#/definitions/io.k8s.api.core.v1.NodeConfigSource"
}
}
},
"io.k8s.api.core.v1.NodeDaemonEndpoints": {
"description": "NodeDaemonEndpoints lists ports opened by daemons running on the Node.",
"properties": {
......@@ -77446,6 +77467,10 @@
"x-kubernetes-patch-merge-key": "type",
"x-kubernetes-patch-strategy": "merge"
},
"config": {
"description": "Status of the config assigned to the node via the dynamic Kubelet config feature.",
"$ref": "#/definitions/io.k8s.api.core.v1.NodeConfigStatus"
},
"daemonEndpoints": {
"description": "Endpoints of daemons running on the Node.",
"$ref": "#/definitions/io.k8s.api.core.v1.NodeDaemonEndpoints"
api/swagger-spec/v1.json
View file @ fcc1f8e7
......@@ -18813,6 +18813,10 @@
"$ref": "v1.AttachedVolume"
},
"description": "List of volumes that are attached to the node."
},
"config": {
"$ref": "v1.NodeConfigStatus",
"description": "Status of the config assigned to the node via the dynamic Kubelet config feature."
}
}
},
......@@ -18993,6 +18997,28 @@
}
}
},
"v1.NodeConfigStatus": {
"id": "v1.NodeConfigStatus",
"description": "NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.",
"properties": {
"assigned": {
"$ref": "v1.NodeConfigSource",
"description": "Assigned reports the checkpointed config the node will try to use. When Node.Spec.ConfigSource is updated, the node checkpoints the associated config payload to local disk, along with a record indicating intended config. The node refers to this record to choose its config checkpoint, and reports this record in Assigned. Assigned only updates in the status after the record has been checkpointed to disk. When the Kubelet is restarted, it tries to make the Assigned config the Active config by loading and validating the checkpointed payload identified by Assigned."
},
"active": {
"$ref": "v1.NodeConfigSource",
"description": "Active reports the checkpointed config the node is actively using. Active will represent either the current version of the Assigned config, or the current LastKnownGood config, depending on whether attempting to use the Assigned config results in an error."
},
"lastKnownGood": {
"$ref": "v1.NodeConfigSource",
"description": "LastKnownGood reports the checkpointed config the node will fall back to when it encounters an error attempting to use the Assigned config. The Assigned config becomes the LastKnownGood config when the node determines that the Assigned config is stable and correct. This is currently implemented as a 10-minute soak period starting when the local record of Assigned config is updated. If the Assigned config is Active at the end of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil, because the local default config is always assumed good. You should not make assumptions about the node's method of determining config stability and correctness, as this may change or become configurable in the future."
},
"error": {
"type": "string",
"description": "Error describes any problems reconciling the Spec.ConfigSource to the Active config. Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting to load or validate the Assigned config, etc. Errors may occur at different points while syncing config. Earlier errors (e.g. download or checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error by fixing the config assigned in Spec.ConfigSource. You can find additional information for debugging by searching the error message in the Kubelet log. Error is a human-readable description of the error state; machines can check whether or not Error is empty, but should not rely on the stability of the Error text across Kubelet versions."
}
}
},
"v1.PersistentVolumeClaimList": {
"id": "v1.PersistentVolumeClaimList",
"description": "PersistentVolumeClaimList is a list of PersistentVolumeClaim items.",
......
cmd/kubelet/app/server.go
View file @ fcc1f8e7
......@@ -198,25 +198,43 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API
}
}
// TODO(#63305): always validate the combination of the local config file and flags, this is the fallback
// when the dynamic config controller tells us to use local config (this can be fixed alongside other validation fixes).
// use dynamic kubelet config, if enabled
var kubeletConfigController *dynamickubeletconfig.Controller
if dynamicConfigDir := kubeletFlags.DynamicConfigDir.Value(); len(dynamicConfigDir) > 0 {
kubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(kubeletConfig, dynamicConfigDir)
var dynamicKubeletConfig *kubeletconfiginternal.KubeletConfiguration
dynamicKubeletConfig, kubeletConfigController, err = BootstrapKubeletConfigController(dynamicConfigDir)
if err != nil {
glog.Fatal(err)
}
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil {
glog.Fatal(err)
}
// update feature gates based on new config
if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
glog.Fatal(err)
// If we should just use our existing, local config, the controller will return a nil config
if dynamicKubeletConfig != nil {
kubeletConfig = dynamicKubeletConfig
// We must enforce flag precedence by re-parsing the command line into the new object.
// This is necessary to preserve backwards-compatibility across binary upgrades.
// See issue #56171 for more details.
if err := kubeletConfigFlagPrecedence(kubeletConfig, args); err != nil {
glog.Fatal(err)
}
// update feature gates based on new config
if err := utilfeature.DefaultFeatureGate.SetFromMap(kubeletConfig.FeatureGates); err != nil {
glog.Fatal(err)
}
}
}
// TODO(#63305): need to reconcile that validation performed inside the dynamic config controller
// will happen against currently set feature gates, rather than future adjustments from combination of files
// and flags. There's a potential scenario where a valid config (because it sets new gates) is considered
// invalid against current gates (at least until --feature-gates flag is removed).
// We should validate against the combination of current feature gates, overrides from feature gates in the file,
// and overrides from feature gates set via flags, rather than currently set feature gates.
// Once the --feature-gates flag is removed, we should strictly validate against the combination of current
// feature gates and feature gates in the file (always need to validate against the combo, because feature-gates
// can layer between the file and dynamic config right now - though maybe we should change this).
// construct a KubeletServer from kubeletFlags and kubeletConfig
kubeletServer := &options.KubeletServer{
KubeletFlags: *kubeletFlags,
......@@ -1089,8 +1107,7 @@ func parseResourceList(m map[string]string) (v1.ResourceList, error) {
}
// BootstrapKubeletConfigController constructs and bootstrap a configuration controller
func BootstrapKubeletConfigController(defaultConfig *kubeletconfiginternal.KubeletConfiguration,
dynamicConfigDir string) (*kubeletconfiginternal.KubeletConfiguration, *dynamickubeletconfig.Controller, error) {
func BootstrapKubeletConfigController(dynamicConfigDir string) (*kubeletconfiginternal.KubeletConfiguration, *dynamickubeletconfig.Controller, error) {
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
return nil, nil, fmt.Errorf("failed to bootstrap Kubelet config controller, you must enable the DynamicKubeletConfig feature gate")
}
......@@ -1104,7 +1121,7 @@ func BootstrapKubeletConfigController(defaultConfig *kubeletconfiginternal.Kubel
return nil, nil, fmt.Errorf("failed to get absolute path for --dynamic-config-dir=%s", dynamicConfigDir)
}
// get the latest KubeletConfiguration checkpoint from disk, or return the default config if no valid checkpoints exist
c := dynamickubeletconfig.NewController(defaultConfig, dir)
c := dynamickubeletconfig.NewController(dir)
kc, err := c.Bootstrap()
if err != nil {
return nil, nil, fmt.Errorf("failed to determine a valid configuration, error: %v", err)
......
pkg/apis/core/types.go
View file @ fcc1f8e7
......@@ -3348,6 +3348,53 @@ type NodeSystemInfo struct {
Architecture string
}
// NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.
type NodeConfigStatus struct {
// Assigned reports the checkpointed config the node will try to use.
// When Node.Spec.ConfigSource is updated, the node checkpoints the associated
// config payload to local disk, along with a record indicating intended
// config. The node refers to this record to choose its config checkpoint, and
// reports this record in Assigned. Assigned only updates in the status after
// the record has been checkpointed to disk. When the Kubelet is restarted,
// it tries to make the Assigned config the Active config by loading and
// validating the checkpointed payload identified by Assigned.
// +optional
Assigned *NodeConfigSource
// Active reports the checkpointed config the node is actively using.
// Active will represent either the current version of the Assigned config,
// or the current LastKnownGood config, depending on whether attempting to use the
// Assigned config results in an error.
// +optional
Active *NodeConfigSource
// LastKnownGood reports the checkpointed config the node will fall back to
// when it encounters an error attempting to use the Assigned config.
// The Assigned config becomes the LastKnownGood config when the node determines
// that the Assigned config is stable and correct.
// This is currently implemented as a 10-minute soak period starting when the local
// record of Assigned config is updated. If the Assigned config is Active at the end
// of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is
// reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil,
// because the local default config is always assumed good.
// You should not make assumptions about the node's method of determining config stability
// and correctness, as this may change or become configurable in the future.
// +optional
LastKnownGood *NodeConfigSource
// Error describes any problems reconciling the Spec.ConfigSource to the Active config.
// Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned
// record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting
// to load or validate the Assigned config, etc.
// Errors may occur at different points while syncing config. Earlier errors (e.g. download or
// checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across
// Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in
// a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error
// by fixing the config assigned in Spec.ConfigSource.
// You can find additional information for debugging by searching the error message in the Kubelet log.
// Error is a human-readable description of the error state; machines can check whether or not Error
// is empty, but should not rely on the stability of the Error text across Kubelet versions.
// +optional
Error string
}
// NodeStatus is information about the current status of a node.
type NodeStatus struct {
// Capacity represents the total resources of a node.
......@@ -3380,6 +3427,9 @@ type NodeStatus struct {
// List of volumes that are attached to the node.
// +optional
VolumesAttached []AttachedVolume
// Status of the config assigned to the node via the dynamic Kubelet config feature.
// +optional
Config *NodeConfigStatus
}
type UniqueVolumeName string
......@@ -3464,8 +3514,6 @@ const (
NodeDiskPressure NodeConditionType = "DiskPressure"
// NodeNetworkUnavailable means that network for the node is not correctly configured.
NodeNetworkUnavailable NodeConditionType = "NetworkUnavailable"
// NodeKubeletConfigOk indicates whether the kubelet is correctly configured
NodeKubeletConfigOk NodeConditionType = "KubeletConfigOk"
)
type NodeCondition struct {
......
pkg/apis/core/v1/zz_generated.conversion.go
View file @ fcc1f8e7
......@@ -210,6 +210,8 @@ func RegisterConversions(scheme *runtime.Scheme) error {
Convert_core_NodeCondition_To_v1_NodeCondition,
Convert_v1_NodeConfigSource_To_core_NodeConfigSource,
Convert_core_NodeConfigSource_To_v1_NodeConfigSource,
Convert_v1_NodeConfigStatus_To_core_NodeConfigStatus,
Convert_core_NodeConfigStatus_To_v1_NodeConfigStatus,
Convert_v1_NodeDaemonEndpoints_To_core_NodeDaemonEndpoints,
Convert_core_NodeDaemonEndpoints_To_v1_NodeDaemonEndpoints,
Convert_v1_NodeList_To_core_NodeList,
......@@ -2635,6 +2637,32 @@ func Convert_core_NodeConfigSource_To_v1_NodeConfigSource(in *core.NodeConfigSou
return autoConvert_core_NodeConfigSource_To_v1_NodeConfigSource(in, out, s)
}
func autoConvert_v1_NodeConfigStatus_To_core_NodeConfigStatus(in *v1.NodeConfigStatus, out *core.NodeConfigStatus, s conversion.Scope) error {
out.Assigned = (*core.NodeConfigSource)(unsafe.Pointer(in.Assigned))
out.Active = (*core.NodeConfigSource)(unsafe.Pointer(in.Active))
out.LastKnownGood = (*core.NodeConfigSource)(unsafe.Pointer(in.LastKnownGood))
out.Error = in.Error
return nil
}
// Convert_v1_NodeConfigStatus_To_core_NodeConfigStatus is an autogenerated conversion function.
func Convert_v1_NodeConfigStatus_To_core_NodeConfigStatus(in *v1.NodeConfigStatus, out *core.NodeConfigStatus, s conversion.Scope) error {
return autoConvert_v1_NodeConfigStatus_To_core_NodeConfigStatus(in, out, s)
}
func autoConvert_core_NodeConfigStatus_To_v1_NodeConfigStatus(in *core.NodeConfigStatus, out *v1.NodeConfigStatus, s conversion.Scope) error {
out.Assigned = (*v1.NodeConfigSource)(unsafe.Pointer(in.Assigned))
out.Active = (*v1.NodeConfigSource)(unsafe.Pointer(in.Active))
out.LastKnownGood = (*v1.NodeConfigSource)(unsafe.Pointer(in.LastKnownGood))
out.Error = in.Error
return nil
}
// Convert_core_NodeConfigStatus_To_v1_NodeConfigStatus is an autogenerated conversion function.
func Convert_core_NodeConfigStatus_To_v1_NodeConfigStatus(in *core.NodeConfigStatus, out *v1.NodeConfigStatus, s conversion.Scope) error {
return autoConvert_core_NodeConfigStatus_To_v1_NodeConfigStatus(in, out, s)
}
func autoConvert_v1_NodeDaemonEndpoints_To_core_NodeDaemonEndpoints(in *v1.NodeDaemonEndpoints, out *core.NodeDaemonEndpoints, s conversion.Scope) error {
if err := Convert_v1_DaemonEndpoint_To_core_DaemonEndpoint(&in.KubeletEndpoint, &out.KubeletEndpoint, s); err != nil {
return err
......@@ -2832,6 +2860,7 @@ func autoConvert_v1_NodeStatus_To_core_NodeStatus(in *v1.NodeStatus, out *core.N
out.Images = *(*[]core.ContainerImage)(unsafe.Pointer(&in.Images))
out.VolumesInUse = *(*[]core.UniqueVolumeName)(unsafe.Pointer(&in.VolumesInUse))
out.VolumesAttached = *(*[]core.AttachedVolume)(unsafe.Pointer(&in.VolumesAttached))
out.Config = (*core.NodeConfigStatus)(unsafe.Pointer(in.Config))
return nil
}
......@@ -2855,6 +2884,7 @@ func autoConvert_core_NodeStatus_To_v1_NodeStatus(in *core.NodeStatus, out *v1.N
out.Images = *(*[]v1.ContainerImage)(unsafe.Pointer(&in.Images))
out.VolumesInUse = *(*[]v1.UniqueVolumeName)(unsafe.Pointer(&in.VolumesInUse))
out.VolumesAttached = *(*[]v1.AttachedVolume)(unsafe.Pointer(&in.VolumesAttached))
out.Config = (*v1.NodeConfigStatus)(unsafe.Pointer(in.Config))
return nil
}
......
pkg/apis/core/validation/validation.go
View file @ fcc1f8e7
......@@ -4010,9 +4010,14 @@ func ValidateNode(node *core.Node) field.ErrorList {
// That said, if specified, we need to ensure they are valid.
allErrs = append(allErrs, ValidateNodeResources(node)...)
// Only allow Node.Spec.ConfigSource to be set if the DynamicKubeletConfig feature gate is enabled
if node.Spec.ConfigSource != nil && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "configSource"), "configSource may only be set if the DynamicKubeletConfig feature gate is enabled)"))
// Only allow Spec.ConfigSource and Status.Config to be set if the DynamicKubeletConfig feature gate is enabled
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
if node.Spec.ConfigSource != nil {
allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "configSource"), "configSource may only be set if the DynamicKubeletConfig feature gate is enabled)"))
}
if node.Status.Config != nil {
allErrs = append(allErrs, field.Forbidden(field.NewPath("status", "config"), "config may only be set if the DynamicKubeletConfig feature gate is enabled)"))
}
}
if len(node.Spec.PodCIDR) != 0 {
......@@ -4097,6 +4102,18 @@ func ValidateNodeUpdate(node, oldNode *core.Node) field.ErrorList {
}
}
// Allow and validate updates to Node.Spec.ConfigSource and Node.Status.Config if DynamicKubeletConfig feature gate is enabled
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
if node.Spec.ConfigSource != nil {
allErrs = append(allErrs, validateNodeConfigSourceSpec(node.Spec.ConfigSource, field.NewPath("spec", "configSource"))...)
}
oldNode.Spec.ConfigSource = node.Spec.ConfigSource
if node.Status.Config != nil {
allErrs = append(allErrs, validateNodeConfigStatus(node.Status.Config, field.NewPath("status", "config"))...)
}
oldNode.Status.Config = node.Status.Config
}
// TODO: move reset function to its own location
// Ignore metadata changes now that they have been tested
oldNode.ObjectMeta = node.ObjectMeta
......@@ -4113,14 +4130,6 @@ func ValidateNodeUpdate(node, oldNode *core.Node) field.ErrorList {
}
oldNode.Spec.Taints = node.Spec.Taints
// Allow updates to Node.Spec.ConfigSource if DynamicKubeletConfig feature gate is enabled
if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
if node.Spec.ConfigSource != nil {
allErrs = append(allErrs, validateNodeConfigSourceSpec(node.Spec.ConfigSource, field.NewPath("spec", "configSource"))...)
}
oldNode.Spec.ConfigSource = node.Spec.ConfigSource
}
// We made allowed changes to oldNode, and now we compare oldNode to node. Any remaining differences indicate changes to protected fields.
// TODO: Add a 'real' error type for this error and provide print actual diffs.
if !apiequality.Semantic.DeepEqual(oldNode, node) {
......@@ -4162,12 +4171,56 @@ func validateConfigMapNodeConfigSourceSpec(source *core.ConfigMapNodeConfigSourc
return append(allErrs, validateConfigMapNodeConfigSource(source, fldPath)...)
}
// validation specififc to Node.Status.Config
func validateNodeConfigStatus(status *core.NodeConfigStatus, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if status.Assigned != nil {
allErrs = append(allErrs, validateNodeConfigSourceStatus(status.Assigned, fldPath.Child("assigned"))...)
}
if status.Active != nil {
allErrs = append(allErrs, validateNodeConfigSourceStatus(status.Active, fldPath.Child("active"))...)
}
if status.LastKnownGood != nil {
allErrs = append(allErrs, validateNodeConfigSourceStatus(status.LastKnownGood, fldPath.Child("lastKnownGood"))...)
}
return allErrs
}
// validation specific to Node.Status.Config.(Active|Assigned|LastKnownGood)
func validateNodeConfigSourceStatus(source *core.NodeConfigSource, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
count := int(0)
if source.ConfigMap != nil {
count++
allErrs = append(allErrs, validateConfigMapNodeConfigSourceStatus(source.ConfigMap, fldPath.Child("configMap"))...)
}
// add more subfields here in the future as they are added to NodeConfigSource
// exactly one reference subfield must be non-nil
if count != 1 {
allErrs = append(allErrs, field.Invalid(fldPath, source, "exactly one reference subfield must be non-nil"))
}
return allErrs
}
// validation specific to Node.Status.Config.(Active|Assigned|LastKnownGood).ConfigMap
func validateConfigMapNodeConfigSourceStatus(source *core.ConfigMapNodeConfigSource, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
if string(source.UID) == "" {
allErrs = append(allErrs, field.Required(fldPath.Child("uid"), "uid must be set in status"))
}
// TODO(#63221): require ResourceVersion in status when we start respecting ConfigMap mutations (the Kubelet isn't tracking it internally until
// that PR, which makes it difficult to report for now).
return append(allErrs, validateConfigMapNodeConfigSource(source, fldPath)...)
}
// common validation
func validateConfigMapNodeConfigSource(source *core.ConfigMapNodeConfigSource, fldPath *field.Path) field.ErrorList {
allErrs := field.ErrorList{}
// validate target configmap namespace
if source.Namespace == "" {
allErrs = append(allErrs, field.Required(fldPath.Child("namespace"), "namespace must be set in spec"))
allErrs = append(allErrs, field.Required(fldPath.Child("namespace"), "namespace must be set"))
} else {
for _, msg := range ValidateNameFunc(ValidateNamespaceName)(source.Namespace, false) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("namespace"), source.Namespace, msg))
......@@ -4175,7 +4228,7 @@ func validateConfigMapNodeConfigSource(source *core.ConfigMapNodeConfigSource, f
}
// validate target configmap name
if source.Name == "" {
allErrs = append(allErrs, field.Required(fldPath.Child("name"), "name must be set in spec"))
allErrs = append(allErrs, field.Required(fldPath.Child("name"), "name must be set"))
} else {
for _, msg := range ValidateNameFunc(ValidateConfigMapName)(source.Name, false) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), source.Name, msg))
......@@ -4183,7 +4236,7 @@ func validateConfigMapNodeConfigSource(source *core.ConfigMapNodeConfigSource, f
}
// validate kubeletConfigKey against rules for configMap key names
if source.KubeletConfigKey == "" {
allErrs = append(allErrs, field.Required(fldPath.Child("kubeletConfigKey"), "kubeletConfigKey must be set in spec"))
allErrs = append(allErrs, field.Required(fldPath.Child("kubeletConfigKey"), "kubeletConfigKey must be set"))
} else {
for _, msg := range validation.IsConfigMapKey(source.KubeletConfigKey) {
allErrs = append(allErrs, field.Invalid(fldPath.Child("kubeletConfigKey"), source.KubeletConfigKey, msg))
......
pkg/apis/core/zz_generated.deepcopy.go
View file @ fcc1f8e7
......@@ -2403,6 +2403,49 @@ func (in *NodeConfigSource) DeepCopy() *NodeConfigSource {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeConfigStatus) DeepCopyInto(out *NodeConfigStatus) {
*out = *in
if in.Assigned != nil {
in, out := &in.Assigned, &out.Assigned
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
if in.Active != nil {
in, out := &in.Active, &out.Active
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
if in.LastKnownGood != nil {
in, out := &in.LastKnownGood, &out.LastKnownGood
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeConfigStatus.
func (in *NodeConfigStatus) DeepCopy() *NodeConfigStatus {
if in == nil {
return nil
}
out := new(NodeConfigStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeDaemonEndpoints) DeepCopyInto(out *NodeDaemonEndpoints) {
*out = *in
out.KubeletEndpoint = in.KubeletEndpoint
......@@ -2654,6 +2697,15 @@ func (in *NodeStatus) DeepCopyInto(out *NodeStatus) {
*out = make([]AttachedVolume, len(*in))
copy(*out, *in)
}
if in.Config != nil {
in, out := &in.Config, &out.Config
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigStatus)
(*in).DeepCopyInto(*out)
}
}
return
}
......
pkg/kubelet/kubeletconfig/checkpoint/download.go
View file @ fcc1f8e7
......@@ -55,13 +55,14 @@ type RemoteConfigSource interface {
// Download downloads the remote config source object returns a Payload backed by the object,
// or a sanitized failure reason and error if the download fails
Download(client clientset.Interface) (Payload, string, error)
// Encode returns a []byte representation of the object behind the RemoteConfigSource
// Encode returns a []byte representation of the NodeConfigSource behind the RemoteConfigSource
Encode() ([]byte, error)
// object returns the underlying source object.
// If you want to compare sources for equality, use EqualRemoteConfigSources,
// which compares the underlying source objects for semantic API equality.
object() interface{}
// NodeConfigSource returns a copy of the underlying apiv1.NodeConfigSource object.
// All RemoteConfigSources are expected to be backed by a NodeConfigSource,
// though the convenience methods on the interface will target the source
// type that was detected in a call to NewRemoteConfigSource.
NodeConfigSource() *apiv1.NodeConfigSource
}
// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object
......@@ -72,9 +73,10 @@ func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource,
// nil here, so that if a new API server allows a new config source type, old clients can send
// an error message rather than crashing due to a nil pointer dereference.
// exactly one reference subfield of the config source must be non-nil, today ConfigMap is the only reference subfield
// Exactly one reference subfield of the config source must be non-nil.
// Currently ConfigMap is the only reference subfield.
if source.ConfigMap == nil {
return nil, status.FailSyncReasonAllNilSubfields, fmt.Errorf("%s, NodeConfigSource was: %#v", status.FailSyncReasonAllNilSubfields, source)
return nil, status.AllNilSubfieldsError, fmt.Errorf("%s, NodeConfigSource was: %#v", status.AllNilSubfieldsError, source)
}
return &remoteConfigMap{source}, "", nil
}
......@@ -109,7 +111,7 @@ func DecodeRemoteConfigSource(data []byte) (RemoteConfigSource, error) {
// comparing the underlying API objects for semantic equality.
func EqualRemoteConfigSources(a, b RemoteConfigSource) bool {
if a != nil && b != nil {
return apiequality.Semantic.DeepEqual(a.object(), b.object())
return apiequality.Semantic.DeepEqual(a.NodeConfigSource(), b.NodeConfigSource())
}
return a == b
}
......@@ -145,13 +147,12 @@ func (r *remoteConfigMap) Download(client clientset.Interface) (Payload, string,
// get the ConfigMap via namespace/name, there doesn't seem to be a way to get it by UID
cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Get(r.source.ConfigMap.Name, metav1.GetOptions{})
if err != nil {
reason = fmt.Sprintf(status.FailSyncReasonDownloadFmt, r.APIPath())
return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
return nil, status.DownloadError, fmt.Errorf("%s, error: %v", status.DownloadError, err)
}
// ensure that UID matches the UID on the source
if r.source.ConfigMap.UID != cm.UID {
reason = fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, r.source.ConfigMap.UID, r.APIPath(), cm.UID)
reason = fmt.Sprintf(status.UIDMismatchErrorFmt, r.source.ConfigMap.UID, r.APIPath(), cm.UID)
return nil, reason, fmt.Errorf(reason)
}
......@@ -178,6 +179,6 @@ func (r *remoteConfigMap) Encode() ([]byte, error) {
return data, nil
}
func (r *remoteConfigMap) object() interface{} {
return r.source
func (r *remoteConfigMap) NodeConfigSource() *apiv1.NodeConfigSource {
return r.source.DeepCopy()
}
pkg/kubelet/kubeletconfig/checkpoint/download_test.go
View file @ fcc1f8e7
......@@ -70,7 +70,7 @@ func TestNewRemoteConfigSource(t *testing.T) {
return
}
// underlying object should match the object passed in
if !apiequality.Semantic.DeepEqual(c.expect.object(), source.object()) {
if !apiequality.Semantic.DeepEqual(c.expect.NodeConfigSource(), source.NodeConfigSource()) {
t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(source))
}
})
......
pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go
View file @ fcc1f8e7
......@@ -24,9 +24,9 @@ import (
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
)
// so far only implements Current(), LastKnownGood(), SetCurrent(), and SetLastKnownGood()
// so far only implements Assigned(), LastKnownGood(), SetAssigned(), and SetLastKnownGood()
type fakeStore struct {
current checkpoint.RemoteConfigSource
assigned checkpoint.RemoteConfigSource
lastKnownGood checkpoint.RemoteConfigSource
}
......@@ -48,20 +48,20 @@ func (s *fakeStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.K
return nil, fmt.Errorf("Load method not supported")
}
func (s *fakeStore) CurrentModified() (time.Time, error) {
return time.Time{}, fmt.Errorf("CurrentModified method not supported")
func (s *fakeStore) AssignedModified() (time.Time, error) {
return time.Time{}, fmt.Errorf("AssignedModified method not supported")
}
func (s *fakeStore) Current() (checkpoint.RemoteConfigSource, error) {
return s.current, nil
func (s *fakeStore) Assigned() (checkpoint.RemoteConfigSource, error) {
return s.assigned, nil
}
func (s *fakeStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
return s.lastKnownGood, nil
}
func (s *fakeStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
s.current = source
func (s *fakeStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
s.assigned = source
return nil
}
......
pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go
View file @ fcc1f8e7
......@@ -31,7 +31,7 @@ import (
const (
metaDir = "meta"
currentFile = "current"
assignedFile = "assigned"
lastKnownGoodFile = "last-known-good"
checkpointsDir = "checkpoints"
......@@ -61,11 +61,11 @@ func (s *fsStore) Initialize() error {
if err := utilfiles.EnsureDir(s.fs, s.dir); err != nil {
return err
}
// ensure metadata directory and reference files (tracks current and lkg configs)
// ensure metadata directory and reference files (tracks assigned and lkg configs)
if err := utilfiles.EnsureDir(s.fs, filepath.Join(s.dir, metaDir)); err != nil {
return err
}
if err := utilfiles.EnsureFile(s.fs, s.metaPath(currentFile)); err != nil {
if err := utilfiles.EnsureFile(s.fs, s.metaPath(assignedFile)); err != nil {
return err
}
if err := utilfiles.EnsureFile(s.fs, s.metaPath(lastKnownGoodFile)); err != nil {
......@@ -111,8 +111,8 @@ func (s *fsStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.Kub
return kc, nil
}
func (s *fsStore) CurrentModified() (time.Time, error) {
path := s.metaPath(currentFile)
func (s *fsStore) AssignedModified() (time.Time, error) {
path := s.metaPath(assignedFile)
info, err := s.fs.Stat(path)
if err != nil {
return time.Time{}, fmt.Errorf("failed to stat %q while checking modification time, error: %v", path, err)
......@@ -120,16 +120,16 @@ func (s *fsStore) CurrentModified() (time.Time, error) {
return info.ModTime(), nil
}
func (s *fsStore) Current() (checkpoint.RemoteConfigSource, error) {
return readRemoteConfigSource(s.fs, s.metaPath(currentFile))
func (s *fsStore) Assigned() (checkpoint.RemoteConfigSource, error) {
return readRemoteConfigSource(s.fs, s.metaPath(assignedFile))
}
func (s *fsStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
return readRemoteConfigSource(s.fs, s.metaPath(lastKnownGoodFile))
}
func (s *fsStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
return writeRemoteConfigSource(s.fs, s.metaPath(currentFile), source)
func (s *fsStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
return writeRemoteConfigSource(s.fs, s.metaPath(assignedFile), source)
}
func (s *fsStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
......
pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go
View file @ fcc1f8e7
......@@ -87,9 +87,9 @@ func TestFsStoreInitialize(t *testing.T) {
t.Fatalf("expect %q to exist, but stat failed with error: %v", store.checkpointPath(""), err)
}
// check that currentFile exists
if _, err := store.fs.Stat(store.metaPath(currentFile)); err != nil {
t.Fatalf("expect %q to exist, but stat failed with error: %v", store.metaPath(currentFile), err)
// check that assignedFile exists
if _, err := store.fs.Stat(store.metaPath(assignedFile)); err != nil {
t.Fatalf("expect %q to exist, but stat failed with error: %v", store.metaPath(assignedFile), err)
}
// check that lastKnownGoodFile exists
......@@ -270,34 +270,34 @@ func TestFsStoreLoad(t *testing.T) {
}
}
func TestFsStoreCurrentModified(t *testing.T) {
func TestFsStoreAssignedModified(t *testing.T) {
store, err := newInitializedFakeFsStore()
if err != nil {
t.Fatalf("error constructing store: %v", err)
}
// create an empty current file, this is good enough for testing
saveTestSourceFile(t, store, currentFile, nil)
// create an empty assigned file, this is good enough for testing
saveTestSourceFile(t, store, assignedFile, nil)
// set the timestamps to the current time, so we can compare to result of store.CurrentModified
// set the timestamps to the current time, so we can compare to result of store.AssignedModified
now := time.Now()
err = store.fs.Chtimes(store.metaPath(currentFile), now, now)
err = store.fs.Chtimes(store.metaPath(assignedFile), now, now)
if err != nil {
t.Fatalf("could not change timestamps, error: %v", err)
}
// for now we hope that the system won't truncate the time to a less precise unit,
// if this test fails on certain systems that may be the reason.
modTime, err := store.CurrentModified()
modTime, err := store.AssignedModified()
if err != nil {
t.Fatalf("unable to determine modification time of current config source, error: %v", err)
t.Fatalf("unable to determine modification time of assigned config source, error: %v", err)
}
if !now.Equal(modTime) {
t.Errorf("expect %q but got %q", now.Format(time.RFC3339), modTime.Format(time.RFC3339))
}
}
func TestFsStoreCurrent(t *testing.T) {
func TestFsStoreAssigned(t *testing.T) {
store, err := newInitializedFakeFsStore()
if err != nil {
t.Fatalf("error constructing store: %v", err)
......@@ -325,10 +325,10 @@ func TestFsStoreCurrent(t *testing.T) {
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
// save the last known good source
saveTestSourceFile(t, store, currentFile, c.expect)
saveTestSourceFile(t, store, assignedFile, c.expect)
// load last-known-good and compare to expected result
source, err := store.Current()
source, err := store.Assigned()
utiltest.ExpectError(t, err, c.err)
if err != nil {
return
......@@ -383,7 +383,7 @@ func TestFsStoreLastKnownGood(t *testing.T) {
}
}
func TestFsStoreSetCurrent(t *testing.T) {
func TestFsStoreSetAssigned(t *testing.T) {
store, err := newInitializedFakeFsStore()
if err != nil {
t.Fatalf("error constructing store: %v", err)
......@@ -409,15 +409,15 @@ source:
t.Fatalf("unexpected error: %v", err)
}
// save the current source
if err := store.SetCurrent(source); err != nil {
// save the assigned source
if err := store.SetAssigned(source); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// check that the source saved as we would expect
data := readTestSourceFile(t, store, currentFile)
data := readTestSourceFile(t, store, assignedFile)
if expect != string(data) {
t.Errorf("expect current source file to contain %q, but got %q", expect, string(data))
t.Errorf("expect assigned source file to contain %q, but got %q", expect, string(data))
}
}
......@@ -485,7 +485,7 @@ func TestFsStoreReset(t *testing.T) {
}
cases := []struct {
desc string
current checkpoint.RemoteConfigSource
assigned checkpoint.RemoteConfigSource
lastKnownGood checkpoint.RemoteConfigSource
updated bool
}{
......@@ -499,7 +499,7 @@ func TestFsStoreReset(t *testing.T) {
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
// manually save the sources to their respective files
saveTestSourceFile(t, store, currentFile, c.current)
saveTestSourceFile(t, store, assignedFile, c.assigned)
saveTestSourceFile(t, store, lastKnownGoodFile, c.lastKnownGood)
// reset
......@@ -509,15 +509,15 @@ func TestFsStoreReset(t *testing.T) {
}
// make sure the files were emptied
if size := testSourceFileSize(t, store, currentFile); size > 0 {
t.Errorf("case %q, expect source file %q to be empty but got %d bytes", c.desc, currentFile, size)
if size := testSourceFileSize(t, store, assignedFile); size > 0 {
t.Errorf("case %q, expect source file %q to be empty but got %d bytes", c.desc, assignedFile, size)
}
if size := testSourceFileSize(t, store, lastKnownGoodFile); size > 0 {
t.Errorf("case %q, expect source file %q to be empty but got %d bytes", c.desc, lastKnownGoodFile, size)
}
// make sure Current() and LastKnownGood() both return nil
current, err := store.Current()
// make sure Assigned() and LastKnownGood() both return nil
assigned, err := store.Assigned()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
......@@ -525,9 +525,9 @@ func TestFsStoreReset(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if current != nil || lastKnownGood != nil {
t.Errorf("case %q, expect nil for current and last-known-good checkpoints, but still have %q and %q, respectively",
c.desc, current, lastKnownGood)
if assigned != nil || lastKnownGood != nil {
t.Errorf("case %q, expect nil for assigned and last-known-good checkpoints, but still have %q and %q, respectively",
c.desc, assigned, lastKnownGood)
}
if c.updated != updated {
t.Errorf("case %q, expect reset to return %t, but got %t", c.desc, c.updated, updated)
......
pkg/kubelet/kubeletconfig/checkpoint/store/store.go
View file @ fcc1f8e7
......@@ -24,7 +24,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
)
// Store saves checkpoints and information about which is the current and last-known-good checkpoint to a storage layer
// Store saves checkpoints and information about which is the assigned and last-known-good checkpoint to a storage layer
type Store interface {
// Initialize sets up the storage layer
Initialize() error
......@@ -38,32 +38,32 @@ type Store interface {
// Load loads the KubeletConfiguration from the checkpoint referenced by `source`.
Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error)
// CurrentModified returns the last time that the current UID was set
CurrentModified() (time.Time, error)
// Current returns the source that points to the current checkpoint, or nil if no current checkpoint is set
Current() (checkpoint.RemoteConfigSource, error)
// AssignedModified returns the last time that the assigned checkpoint was set
AssignedModified() (time.Time, error)
// Assigned returns the source that points to the checkpoint currently assigned to the Kubelet, or nil if no assigned checkpoint is set
Assigned() (checkpoint.RemoteConfigSource, error)
// LastKnownGood returns the source that points to the last-known-good checkpoint, or nil if no last-known-good checkpoint is set
LastKnownGood() (checkpoint.RemoteConfigSource, error)
// SetCurrent saves the source that points to the current checkpoint, set to nil to unset
SetCurrent(source checkpoint.RemoteConfigSource) error
// SetAssigned saves the source that points to the assigned checkpoint, set to nil to unset
SetAssigned(source checkpoint.RemoteConfigSource) error
// SetLastKnownGood saves the source that points to the last-known-good checkpoint, set to nil to unset
SetLastKnownGood(source checkpoint.RemoteConfigSource) error
// Reset unsets the current and last-known-good UIDs and returns whether the current UID was unset as a result of the reset
// Reset unsets the assigned and last-known-good checkpoints and returns whether the assigned checkpoint was unset as a result of the reset
Reset() (bool, error)
}
// reset is a helper for implementing Reset, which can be implemented in terms of Store methods
func reset(s Store) (bool, error) {
current, err := s.Current()
assigned, err := s.Assigned()
if err != nil {
return false, err
}
if err := s.SetLastKnownGood(nil); err != nil {
return false, fmt.Errorf("failed to reset last-known-good UID in checkpoint store, error: %v", err)
}
if err := s.SetCurrent(nil); err != nil {
return false, fmt.Errorf("failed to reset current UID in checkpoint store, error: %v", err)
if err := s.SetAssigned(nil); err != nil {
return false, fmt.Errorf("failed to reset assigned UID in checkpoint store, error: %v", err)
}
return current != nil, nil
return assigned != nil, nil
}
pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go
View file @ fcc1f8e7
......@@ -48,21 +48,21 @@ func TestReset(t *testing.T) {
s *fakeStore
updated bool
}{
{&fakeStore{current: nil, lastKnownGood: nil}, false},
{&fakeStore{current: source, lastKnownGood: nil}, true},
{&fakeStore{current: nil, lastKnownGood: source}, false},
{&fakeStore{current: source, lastKnownGood: source}, true},
{&fakeStore{current: source, lastKnownGood: otherSource}, true},
{&fakeStore{current: otherSource, lastKnownGood: source}, true},
{&fakeStore{assigned: nil, lastKnownGood: nil}, false},
{&fakeStore{assigned: source, lastKnownGood: nil}, true},
{&fakeStore{assigned: nil, lastKnownGood: source}, false},
{&fakeStore{assigned: source, lastKnownGood: source}, true},
{&fakeStore{assigned: source, lastKnownGood: otherSource}, true},
{&fakeStore{assigned: otherSource, lastKnownGood: source}, true},
}
for _, c := range cases {
updated, err := reset(c.s)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if c.s.current != nil || c.s.lastKnownGood != nil {
t.Errorf("case %q, expect nil for current and last-known-good checkpoints, but still have %q and %q, respectively",
spew.Sdump(c.s), c.s.current, c.s.lastKnownGood)
if c.s.assigned != nil || c.s.lastKnownGood != nil {
t.Errorf("case %q, expect nil for assigned and last-known-good checkpoints, but still have %q and %q, respectively",
spew.Sdump(c.s), c.s.assigned, c.s.lastKnownGood)
}
if c.updated != updated {
t.Errorf("case %q, expect reset to return %t, but got %t", spew.Sdump(c.s), c.updated, updated)
......
pkg/kubelet/kubeletconfig/configsync.go
View file @ fcc1f8e7
......@@ -71,14 +71,14 @@ func (cc *Controller) syncConfigSource(client clientset.Interface, eventClient v
node, err := latestNode(cc.informer.GetStore(), nodeName)
if err != nil {
cc.configOk.SetFailSyncCondition(status.FailSyncReasonInformer)
syncerr = fmt.Errorf("%s, error: %v", status.FailSyncReasonInformer, err)
cc.configStatus.SetErrorOverride(fmt.Sprintf(status.SyncErrorFmt, status.InternalError))
syncerr = fmt.Errorf("%s, error: %v", status.InternalError, err)
return
}
// check the Node and download any new config
if updated, cur, reason, err := cc.doSyncConfigSource(client, node.Spec.ConfigSource); err != nil {
cc.configOk.SetFailSyncCondition(reason)
cc.configStatus.SetErrorOverride(fmt.Sprintf(status.SyncErrorFmt, reason))
syncerr = fmt.Errorf("%s, error: %v", reason, err)
return
} else if updated {
......@@ -99,8 +99,8 @@ func (cc *Controller) syncConfigSource(client clientset.Interface, eventClient v
// If we get here:
// - there is no need to restart to update the current config
// - there was no error trying to sync configuration
// - if, previously, there was an error trying to sync configuration, we need to clear that error from the condition
cc.configOk.ClearFailSyncCondition()
// - if, previously, there was an error trying to sync configuration, we need to clear that error from the status
cc.configStatus.SetErrorOverride("")
}
// doSyncConfigSource checkpoints and sets the store's current config to the new config or resets config,
......@@ -125,7 +125,7 @@ func (cc *Controller) doSyncConfigSource(client clientset.Interface, source *api
if err != nil {
return false, nil, reason, err
}
updated, reason, err := cc.setCurrentConfig(remote)
updated, reason, err := cc.setAssignedConfig(remote)
if err != nil {
return false, nil, reason, err
}
......@@ -137,9 +137,9 @@ func (cc *Controller) doSyncConfigSource(client clientset.Interface, source *api
func (cc *Controller) checkpointConfigSource(client clientset.Interface, source checkpoint.RemoteConfigSource) (string, error) {
// if the checkpoint already exists, skip downloading
if ok, err := cc.checkpointStore.Exists(source); err != nil {
reason := fmt.Sprintf(status.FailSyncReasonCheckpointExistenceFmt, source.APIPath(), source.UID())
return reason, fmt.Errorf("%s, error: %v", reason, err)
return status.InternalError, fmt.Errorf("%s, error: %v", status.InternalError, err)
} else if ok {
// TODO(mtaufen): update this to include ResourceVersion in #63221
utillog.Infof("checkpoint already exists for object %s with UID %s, skipping download", source.APIPath(), source.UID())
return "", nil
}
......@@ -153,28 +153,21 @@ func (cc *Controller) checkpointConfigSource(client clientset.Interface, source
// save
err = cc.checkpointStore.Save(payload)
if err != nil {
reason := fmt.Sprintf(status.FailSyncReasonSaveCheckpointFmt, source.APIPath(), payload.UID())
return reason, fmt.Errorf("%s, error: %v", reason, err)
return status.InternalError, fmt.Errorf("%s, error: %v", status.InternalError, err)
}
return "", nil
}
// setCurrentConfig the current checkpoint config in the store
// returns whether the current config changed as a result, or a sanitized failure reason and an error.
func (cc *Controller) setCurrentConfig(source checkpoint.RemoteConfigSource) (bool, string, error) {
failReason := func(s checkpoint.RemoteConfigSource) string {
if source == nil {
return status.FailSyncReasonSetCurrentLocal
}
return fmt.Sprintf(status.FailSyncReasonSetCurrentUIDFmt, source.APIPath(), source.UID())
}
current, err := cc.checkpointStore.Current()
// setAssignedConfig updates the assigned checkpoint config in the store.
// Returns whether the current config changed as a result, or a sanitized failure reason and an error.
func (cc *Controller) setAssignedConfig(source checkpoint.RemoteConfigSource) (bool, string, error) {
current, err := cc.checkpointStore.Assigned()
if err != nil {
return false, failReason(source), err
return false, status.InternalError, err
}
if err := cc.checkpointStore.SetCurrent(source); err != nil {
return false, failReason(source), err
if err := cc.checkpointStore.SetAssigned(source); err != nil {
return false, status.InternalError, err
}
return !checkpoint.EqualRemoteConfigSources(current, source), "", nil
}
......@@ -184,7 +177,7 @@ func (cc *Controller) setCurrentConfig(source checkpoint.RemoteConfigSource) (bo
func (cc *Controller) resetConfig() (bool, string, error) {
updated, err := cc.checkpointStore.Reset()
if err != nil {
return false, status.FailSyncReasonReset, err
return false, status.InternalError, err
}
return updated, "", nil
}
......
pkg/kubelet/kubeletconfig/status/BUILD
View file @ fcc1f8e7
......@@ -10,14 +10,11 @@ go_library(
srcs = ["status.go"],
importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status",
deps = [
"//pkg/api/legacyscheme:go_default_library",
"//pkg/apis/core:go_default_library",
"//pkg/kubelet/kubeletconfig/util/equal:go_default_library",
"//pkg/kubelet/kubeletconfig/util/log:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
......
pkg/registry/core/node/storage/storage.go
View file @ fcc1f8e7
......@@ -54,7 +54,7 @@ type REST struct {
proxyTransport http.RoundTripper
}
// StatusREST implements the REST endpoint for changing the status of a pod.
// StatusREST implements the REST endpoint for changing the status of a node.
type StatusREST struct {
store *genericregistry.Store
}
......
pkg/registry/core/node/strategy.go
View file @ fcc1f8e7
......@@ -127,14 +127,23 @@ type nodeStatusStrategy struct {
var StatusStrategy = nodeStatusStrategy{Strategy}
func (nodeStatusStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {
_ = obj.(*api.Node)
node := obj.(*api.Node)
// Nodes allow *all* fields, including status, to be set on create.
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
node.Status.Config = nil
}
}
func (nodeStatusStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {
newNode := obj.(*api.Node)
oldNode := old.(*api.Node)
newNode.Spec = oldNode.Spec
if !utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) {
newNode.Status.Config = nil
oldNode.Status.Config = nil
}
}
func (nodeStatusStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList {
......
staging/src/k8s.io/api/core/v1/generated.pb.go
View file @ fcc1f8e7
This source diff could not be displayed because it is too large. You can view the blob instead.
staging/src/k8s.io/api/core/v1/generated.proto
View file @ fcc1f8e7
......@@ -1847,6 +1847,56 @@ message NodeConfigSource {
optional ConfigMapNodeConfigSource configMap = 2;
}
// NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.
message NodeConfigStatus {
// Assigned reports the checkpointed config the node will try to use.
// When Node.Spec.ConfigSource is updated, the node checkpoints the associated
// config payload to local disk, along with a record indicating intended
// config. The node refers to this record to choose its config checkpoint, and
// reports this record in Assigned. Assigned only updates in the status after
// the record has been checkpointed to disk. When the Kubelet is restarted,
// it tries to make the Assigned config the Active config by loading and
// validating the checkpointed payload identified by Assigned.
// +optional
optional NodeConfigSource assigned = 1;
// Active reports the checkpointed config the node is actively using.
// Active will represent either the current version of the Assigned config,
// or the current LastKnownGood config, depending on whether attempting to use the
// Assigned config results in an error.
// +optional
optional NodeConfigSource active = 2;
// LastKnownGood reports the checkpointed config the node will fall back to
// when it encounters an error attempting to use the Assigned config.
// The Assigned config becomes the LastKnownGood config when the node determines
// that the Assigned config is stable and correct.
// This is currently implemented as a 10-minute soak period starting when the local
// record of Assigned config is updated. If the Assigned config is Active at the end
// of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is
// reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil,
// because the local default config is always assumed good.
// You should not make assumptions about the node's method of determining config stability
// and correctness, as this may change or become configurable in the future.
// +optional
optional NodeConfigSource lastKnownGood = 3;
// Error describes any problems reconciling the Spec.ConfigSource to the Active config.
// Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned
// record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting
// to load or validate the Assigned config, etc.
// Errors may occur at different points while syncing config. Earlier errors (e.g. download or
// checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across
// Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in
// a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error
// by fixing the config assigned in Spec.ConfigSource.
// You can find additional information for debugging by searching the error message in the Kubelet log.
// Error is a human-readable description of the error state; machines can check whether or not Error
// is empty, but should not rely on the stability of the Error text across Kubelet versions.
// +optional
optional string error = 4;
}
// NodeDaemonEndpoints lists ports opened by daemons running on the Node.
message NodeDaemonEndpoints {
// Endpoint on which Kubelet is listening.
......@@ -2001,6 +2051,10 @@ message NodeStatus {
// List of volumes that are attached to the node.
// +optional
repeated AttachedVolume volumesAttached = 10;
// Status of the config assigned to the node via the dynamic Kubelet config feature.
// +optional
optional NodeConfigStatus config = 11;
}
// NodeSystemInfo is a set of ids/uuids to uniquely identify the node.
......
staging/src/k8s.io/api/core/v1/types.go
View file @ fcc1f8e7
......@@ -3732,6 +3732,53 @@ type NodeSystemInfo struct {
Architecture string `json:"architecture" protobuf:"bytes,10,opt,name=architecture"`
}
// NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.
type NodeConfigStatus struct {
// Assigned reports the checkpointed config the node will try to use.
// When Node.Spec.ConfigSource is updated, the node checkpoints the associated
// config payload to local disk, along with a record indicating intended
// config. The node refers to this record to choose its config checkpoint, and
// reports this record in Assigned. Assigned only updates in the status after
// the record has been checkpointed to disk. When the Kubelet is restarted,
// it tries to make the Assigned config the Active config by loading and
// validating the checkpointed payload identified by Assigned.
// +optional
Assigned *NodeConfigSource `json:"assigned,omitempty" protobuf:"bytes,1,opt,name=assigned"`
// Active reports the checkpointed config the node is actively using.
// Active will represent either the current version of the Assigned config,
// or the current LastKnownGood config, depending on whether attempting to use the
// Assigned config results in an error.
// +optional
Active *NodeConfigSource `json:"active,omitempty" protobuf:"bytes,2,opt,name=active"`
// LastKnownGood reports the checkpointed config the node will fall back to
// when it encounters an error attempting to use the Assigned config.
// The Assigned config becomes the LastKnownGood config when the node determines
// that the Assigned config is stable and correct.
// This is currently implemented as a 10-minute soak period starting when the local
// record of Assigned config is updated. If the Assigned config is Active at the end
// of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is
// reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil,
// because the local default config is always assumed good.
// You should not make assumptions about the node's method of determining config stability
// and correctness, as this may change or become configurable in the future.
// +optional
LastKnownGood *NodeConfigSource `json:"lastKnownGood,omitempty" protobuf:"bytes,3,opt,name=lastKnownGood"`
// Error describes any problems reconciling the Spec.ConfigSource to the Active config.
// Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned
// record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting
// to load or validate the Assigned config, etc.
// Errors may occur at different points while syncing config. Earlier errors (e.g. download or
// checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across
// Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in
// a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error
// by fixing the config assigned in Spec.ConfigSource.
// You can find additional information for debugging by searching the error message in the Kubelet log.
// Error is a human-readable description of the error state; machines can check whether or not Error
// is empty, but should not rely on the stability of the Error text across Kubelet versions.
// +optional
Error string `json:"error,omitempty" protobuf:"bytes,4,opt,name=error"`
}
// NodeStatus is information about the current status of a node.
type NodeStatus struct {
// Capacity represents the total resources of a node.
......@@ -3776,6 +3823,9 @@ type NodeStatus struct {
// List of volumes that are attached to the node.
// +optional
VolumesAttached []AttachedVolume `json:"volumesAttached,omitempty" protobuf:"bytes,10,rep,name=volumesAttached"`
// Status of the config assigned to the node via the dynamic Kubelet config feature.
// +optional
Config *NodeConfigStatus `json:"config,omitempty" protobuf:"bytes,11,opt,name=config"`
}
type UniqueVolumeName string
......@@ -3863,8 +3913,6 @@ const (
NodePIDPressure NodeConditionType = "PIDPressure"
// NodeNetworkUnavailable means that network for the node is not correctly configured.
NodeNetworkUnavailable NodeConditionType = "NetworkUnavailable"
// NodeKubeletConfigOk indicates whether the kubelet is correctly configured
NodeKubeletConfigOk NodeConditionType = "KubeletConfigOk"
)
// NodeCondition contains condition information for a node.
......
staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go
View file @ fcc1f8e7
......@@ -990,6 +990,18 @@ func (NodeConfigSource) SwaggerDoc() map[string]string {
return map_NodeConfigSource
}
var map_NodeConfigStatus = map[string]string{
"": "NodeConfigStatus describes the status of the config assigned by Node.Spec.ConfigSource.",
"assigned": "Assigned reports the checkpointed config the node will try to use. When Node.Spec.ConfigSource is updated, the node checkpoints the associated config payload to local disk, along with a record indicating intended config. The node refers to this record to choose its config checkpoint, and reports this record in Assigned. Assigned only updates in the status after the record has been checkpointed to disk. When the Kubelet is restarted, it tries to make the Assigned config the Active config by loading and validating the checkpointed payload identified by Assigned.",
"active": "Active reports the checkpointed config the node is actively using. Active will represent either the current version of the Assigned config, or the current LastKnownGood config, depending on whether attempting to use the Assigned config results in an error.",
"lastKnownGood": "LastKnownGood reports the checkpointed config the node will fall back to when it encounters an error attempting to use the Assigned config. The Assigned config becomes the LastKnownGood config when the node determines that the Assigned config is stable and correct. This is currently implemented as a 10-minute soak period starting when the local record of Assigned config is updated. If the Assigned config is Active at the end of this period, it becomes the LastKnownGood. Note that if Spec.ConfigSource is reset to nil (use local defaults), the LastKnownGood is also immediately reset to nil, because the local default config is always assumed good. You should not make assumptions about the node's method of determining config stability and correctness, as this may change or become configurable in the future.",
"error": "Error describes any problems reconciling the Spec.ConfigSource to the Active config. Errors may occur, for example, attempting to checkpoint Spec.ConfigSource to the local Assigned record, attempting to checkpoint the payload associated with Spec.ConfigSource, attempting to load or validate the Assigned config, etc. Errors may occur at different points while syncing config. Earlier errors (e.g. download or checkpointing errors) will not result in a rollback to LastKnownGood, and may resolve across Kubelet retries. Later errors (e.g. loading or validating a checkpointed config) will result in a rollback to LastKnownGood. In the latter case, it is usually possible to resolve the error by fixing the config assigned in Spec.ConfigSource. You can find additional information for debugging by searching the error message in the Kubelet log. Error is a human-readable description of the error state; machines can check whether or not Error is empty, but should not rely on the stability of the Error text across Kubelet versions.",
}
func (NodeConfigStatus) SwaggerDoc() map[string]string {
return map_NodeConfigStatus
}
var map_NodeDaemonEndpoints = map[string]string{
"": "NodeDaemonEndpoints lists ports opened by daemons running on the Node.",
"kubeletEndpoint": "Endpoint on which Kubelet is listening.",
......@@ -1083,6 +1095,7 @@ var map_NodeStatus = map[string]string{
"images": "List of container images on this node",
"volumesInUse": "List of attachable volumes in use (mounted) by the node.",
"volumesAttached": "List of volumes that are attached to the node.",
"config": "Status of the config assigned to the node via the dynamic Kubelet config feature.",
}
func (NodeStatus) SwaggerDoc() map[string]string {
......
staging/src/k8s.io/api/core/v1/zz_generated.deepcopy.go
View file @ fcc1f8e7
......@@ -2399,6 +2399,49 @@ func (in *NodeConfigSource) DeepCopy() *NodeConfigSource {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeConfigStatus) DeepCopyInto(out *NodeConfigStatus) {
*out = *in
if in.Assigned != nil {
in, out := &in.Assigned, &out.Assigned
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
if in.Active != nil {
in, out := &in.Active, &out.Active
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
if in.LastKnownGood != nil {
in, out := &in.LastKnownGood, &out.LastKnownGood
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigSource)
(*in).DeepCopyInto(*out)
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeConfigStatus.
func (in *NodeConfigStatus) DeepCopy() *NodeConfigStatus {
if in == nil {
return nil
}
out := new(NodeConfigStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeDaemonEndpoints) DeepCopyInto(out *NodeDaemonEndpoints) {
*out = *in
out.KubeletEndpoint = in.KubeletEndpoint
......@@ -2650,6 +2693,15 @@ func (in *NodeStatus) DeepCopyInto(out *NodeStatus) {
*out = make([]AttachedVolume, len(*in))
copy(*out, *in)
}
if in.Config != nil {
in, out := &in.Config, &out.Config
if *in == nil {
*out = nil
} else {
*out = new(NodeConfigStatus)
(*in).DeepCopyInto(*out)
}
}
return
}
......
test/e2e_node/BUILD
View file @ fcc1f8e7
......@@ -147,6 +147,7 @@ go_test(
"//vendor/github.com/onsi/gomega/gstruct:go_default_library",
"//vendor/github.com/onsi/gomega/types:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
......
test/e2e_node/util.go
View file @ fcc1f8e7
......@@ -225,17 +225,6 @@ func setNodeConfigSource(f *framework.Framework, source *apiv1.NodeConfigSource)
return nil
}
// getKubeletConfigOkCondition returns the first NodeCondition in `cs` with Type == apiv1.NodeKubeletConfigOk,
// or if no such condition exists, returns nil.
func getKubeletConfigOkCondition(cs []apiv1.NodeCondition) *apiv1.NodeCondition {
for i := range cs {
if cs[i].Type == apiv1.NodeKubeletConfigOk {
return &cs[i]
}
}
return nil
}
// Causes the test to fail, or returns a status 200 response from the /configz endpoint
func pollConfigz(timeout time.Duration, pollInterval time.Duration) *http.Response {
endpoint := fmt.Sprintf("http://127.0.0.1:8080/api/v1/nodes/%s/proxy/configz", framework.TestContext.NodeName)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment