Skip to content

K
k3s
Commit 3dc7c64c authored by Aaron Levy's avatar Aaron Levy Committed by Aaron Levy
Browse files
Options

kubelet: report NodeReady last in status list

Addresses a version skew issue where the last condition status is always evaluated as the NodeReady status. As a workaround force the NodeReady condition to be the last in the list of node conditions. ref: https://github.com/kubernetes/kubernetes/issues/16961
parent 46eca25d
Hide whitespace changes
Inline Side-by-side
Showing with 104 additions and 80 deletions
pkg/kubelet/kubelet.go
View file @ 3dc7c64c
...@@ -2773,52 +2773,6 @@ func (kl *Kubelet) setNodeStatus(node *api.Node) error { ...@@ -2773,52 +2773,6 @@ func (kl *Kubelet) setNodeStatus(node *api.Node) error {
node.Status.DaemonEndpoints = *kl.daemonEndpoints node.Status.DaemonEndpoints = *kl.daemonEndpoints
currentTime := unversioned.Now() currentTime := unversioned.Now()
var newNodeReadyCondition api.NodeCondition
var oldNodeReadyConditionStatus api.ConditionStatus
if rs := kl.runtimeState.errors(); len(rs) == 0 {
newNodeReadyCondition = api.NodeCondition{
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: "kubelet is posting ready status",
LastHeartbeatTime: currentTime,
}
} else {
newNodeReadyCondition = api.NodeCondition{
Type: api.NodeReady,
Status: api.ConditionFalse,
Reason: "KubeletNotReady",
Message: strings.Join(rs, ","),
LastHeartbeatTime: currentTime,
}
}
updated := false
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == api.NodeReady {
oldNodeReadyConditionStatus = node.Status.Conditions[i].Status
if oldNodeReadyConditionStatus == newNodeReadyCondition.Status {
newNodeReadyCondition.LastTransitionTime = node.Status.Conditions[i].LastTransitionTime
} else {
newNodeReadyCondition.LastTransitionTime = currentTime
}
node.Status.Conditions[i] = newNodeReadyCondition
updated = true
break
}
}
if !updated {
newNodeReadyCondition.LastTransitionTime = currentTime
node.Status.Conditions = append(node.Status.Conditions, newNodeReadyCondition)
}
if !updated || oldNodeReadyConditionStatus != newNodeReadyCondition.Status {
if newNodeReadyCondition.Status == api.ConditionTrue {
kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeReady)
} else {
kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeNotReady)
}
}
var nodeOODCondition *api.NodeCondition var nodeOODCondition *api.NodeCondition
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update. // Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
...@@ -2873,6 +2827,55 @@ func (kl *Kubelet) setNodeStatus(node *api.Node) error { ...@@ -2873,6 +2827,55 @@ func (kl *Kubelet) setNodeStatus(node *api.Node) error {
node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition) node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
} }
// NOTE(aaronlevy): NodeReady condition needs to be the last in the list of node conditions.
// This is due to an issue with version skewed kubelet and master components.
// ref: https://github.com/kubernetes/kubernetes/issues/16961
var newNodeReadyCondition api.NodeCondition
var oldNodeReadyConditionStatus api.ConditionStatus
if rs := kl.runtimeState.errors(); len(rs) == 0 {
newNodeReadyCondition = api.NodeCondition{
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: "kubelet is posting ready status",
LastHeartbeatTime: currentTime,
}
} else {
newNodeReadyCondition = api.NodeCondition{
Type: api.NodeReady,
Status: api.ConditionFalse,
Reason: "KubeletNotReady",
Message: strings.Join(rs, ","),
LastHeartbeatTime: currentTime,
}
}
updated := false
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == api.NodeReady {
oldNodeReadyConditionStatus = node.Status.Conditions[i].Status
if oldNodeReadyConditionStatus == newNodeReadyCondition.Status {
newNodeReadyCondition.LastTransitionTime = node.Status.Conditions[i].LastTransitionTime
} else {
newNodeReadyCondition.LastTransitionTime = currentTime
}
node.Status.Conditions[i] = newNodeReadyCondition
updated = true
break
}
}
if !updated {
newNodeReadyCondition.LastTransitionTime = currentTime
node.Status.Conditions = append(node.Status.Conditions, newNodeReadyCondition)
}
if !updated || oldNodeReadyConditionStatus != newNodeReadyCondition.Status {
if newNodeReadyCondition.Status == api.ConditionTrue {
kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeReady)
} else {
kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeNotReady)
}
}
if oldNodeUnschedulable != node.Spec.Unschedulable { if oldNodeUnschedulable != node.Spec.Unschedulable {
if node.Spec.Unschedulable { if node.Spec.Unschedulable {
kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeNotSchedulable) kl.recordNodeStatusEvent(api.EventTypeNormal, kubecontainer.NodeNotSchedulable)
......
pkg/kubelet/kubelet_test.go
View file @ 3dc7c64c
...@@ -2575,14 +2575,6 @@ func TestUpdateNewNodeStatus(t *testing.T) { ...@@ -2575,14 +2575,6 @@ func TestUpdateNewNodeStatus(t *testing.T) {
Status: api.NodeStatus{ Status: api.NodeStatus{
Conditions: []api.NodeCondition{ Conditions: []api.NodeCondition{
{ {
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{},
},
{
Type: api.NodeOutOfDisk, Type: api.NodeOutOfDisk,
Status: api.ConditionFalse, Status: api.ConditionFalse,
Reason: "KubeletHasSufficientDisk", Reason: "KubeletHasSufficientDisk",
...@@ -2590,6 +2582,14 @@ func TestUpdateNewNodeStatus(t *testing.T) { ...@@ -2590,6 +2582,14 @@ func TestUpdateNewNodeStatus(t *testing.T) {
LastHeartbeatTime: unversioned.Time{}, LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{}, LastTransitionTime: unversioned.Time{},
}, },
{
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{},
},
}, },
NodeInfo: api.NodeSystemInfo{ NodeInfo: api.NodeSystemInfo{
MachineID: "123", MachineID: "123",
...@@ -2639,6 +2639,11 @@ func TestUpdateNewNodeStatus(t *testing.T) { ...@@ -2639,6 +2639,11 @@ func TestUpdateNewNodeStatus(t *testing.T) {
updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{} updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{}
} }
// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
if updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type != api.NodeReady {
t.Errorf("unexpected node condition order. NodeReady should be last.")
}
if !reflect.DeepEqual(expectedNode, updatedNode) { if !reflect.DeepEqual(expectedNode, updatedNode) {
t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode)) t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode))
} }
...@@ -2691,14 +2696,6 @@ func testDockerRuntimeVersion(t *testing.T) { ...@@ -2691,14 +2696,6 @@ func testDockerRuntimeVersion(t *testing.T) {
Status: api.NodeStatus{ Status: api.NodeStatus{
Conditions: []api.NodeCondition{ Conditions: []api.NodeCondition{
{ {
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{},
},
{
Type: api.NodeOutOfDisk, Type: api.NodeOutOfDisk,
Status: api.ConditionFalse, Status: api.ConditionFalse,
Reason: "KubeletHasSufficientDisk", Reason: "KubeletHasSufficientDisk",
...@@ -2706,6 +2703,14 @@ func testDockerRuntimeVersion(t *testing.T) { ...@@ -2706,6 +2703,14 @@ func testDockerRuntimeVersion(t *testing.T) {
LastHeartbeatTime: unversioned.Time{}, LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{}, LastTransitionTime: unversioned.Time{},
}, },
{
Type: api.NodeReady,
Status: api.ConditionTrue,
Reason: "KubeletReady",
Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{},
},
}, },
NodeInfo: api.NodeSystemInfo{ NodeInfo: api.NodeSystemInfo{
MachineID: "123", MachineID: "123",
...@@ -2754,6 +2759,12 @@ func testDockerRuntimeVersion(t *testing.T) { ...@@ -2754,6 +2759,12 @@ func testDockerRuntimeVersion(t *testing.T) {
updatedNode.Status.Conditions[i].LastHeartbeatTime = unversioned.Time{} updatedNode.Status.Conditions[i].LastHeartbeatTime = unversioned.Time{}
updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{} updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{}
} }
// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
if updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type != api.NodeReady {
t.Errorf("unexpected node condition order. NodeReady should be last.")
}
if !reflect.DeepEqual(expectedNode, updatedNode) { if !reflect.DeepEqual(expectedNode, updatedNode) {
t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode)) t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode))
} }
...@@ -2793,18 +2804,18 @@ func TestUpdateExistingNodeStatus(t *testing.T) { ...@@ -2793,18 +2804,18 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
Status: api.NodeStatus{ Status: api.NodeStatus{
Conditions: []api.NodeCondition{ Conditions: []api.NodeCondition{
{ {
Type: api.NodeReady, Type: api.NodeOutOfDisk,
Status: api.ConditionTrue, Status: api.ConditionTrue,
Reason: "KubeletReady", Reason: "KubeletOutOfDisk",
Message: fmt.Sprintf("kubelet is posting ready status"), Message: "out of disk space",
LastHeartbeatTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastHeartbeatTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
}, },
{ {
Type: api.NodeOutOfDisk, Type: api.NodeReady,
Status: api.ConditionTrue, Status: api.ConditionTrue,
Reason: "KubeletOutOfDisk", Reason: "KubeletReady",
Message: "out of disk space", Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastHeartbeatTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
}, },
...@@ -2855,18 +2866,18 @@ func TestUpdateExistingNodeStatus(t *testing.T) { ...@@ -2855,18 +2866,18 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
Status: api.NodeStatus{ Status: api.NodeStatus{
Conditions: []api.NodeCondition{ Conditions: []api.NodeCondition{
{ {
Type: api.NodeReady, Type: api.NodeOutOfDisk,
Status: api.ConditionTrue, Status: api.ConditionTrue,
Reason: "KubeletReady", Reason: "KubeletOutOfDisk",
Message: fmt.Sprintf("kubelet is posting ready status"), Message: "out of disk space",
LastHeartbeatTime: unversioned.Time{}, // placeholder LastHeartbeatTime: unversioned.Time{}, // placeholder
LastTransitionTime: unversioned.Time{}, // placeholder LastTransitionTime: unversioned.Time{}, // placeholder
}, },
{ {
Type: api.NodeOutOfDisk, Type: api.NodeReady,
Status: api.ConditionTrue, Status: api.ConditionTrue,
Reason: "KubeletOutOfDisk", Reason: "KubeletReady",
Message: "out of disk space", Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: unversioned.Time{}, // placeholder LastHeartbeatTime: unversioned.Time{}, // placeholder
LastTransitionTime: unversioned.Time{}, // placeholder LastTransitionTime: unversioned.Time{}, // placeholder
}, },
...@@ -2921,6 +2932,11 @@ func TestUpdateExistingNodeStatus(t *testing.T) { ...@@ -2921,6 +2932,11 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{} updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{}
} }
// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
if updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type != api.NodeReady {
t.Errorf("unexpected node condition order. NodeReady should be last.")
}
if !reflect.DeepEqual(expectedNode, updatedNode) { if !reflect.DeepEqual(expectedNode, updatedNode) {
t.Errorf("expected \n%v\n, got \n%v", expectedNode, updatedNode) t.Errorf("expected \n%v\n, got \n%v", expectedNode, updatedNode)
} }
...@@ -2977,18 +2993,18 @@ func TestUpdateNodeStatusWithoutContainerRuntime(t *testing.T) { ...@@ -2977,18 +2993,18 @@ func TestUpdateNodeStatusWithoutContainerRuntime(t *testing.T) {
Status: api.NodeStatus{ Status: api.NodeStatus{
Conditions: []api.NodeCondition{ Conditions: []api.NodeCondition{
{ {
Type: api.NodeReady, Type: api.NodeOutOfDisk,
Status: api.ConditionFalse, Status: api.ConditionFalse,
Reason: "KubeletNotReady", Reason: "KubeletHasSufficientDisk",
Message: fmt.Sprintf("container runtime is down"), Message: "kubelet has sufficient disk space available",
LastHeartbeatTime: unversioned.Time{}, LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{}, LastTransitionTime: unversioned.Time{},
}, },
{ {
Type: api.NodeOutOfDisk, Type: api.NodeReady,
Status: api.ConditionFalse, Status: api.ConditionFalse,
Reason: "KubeletHasSufficientDisk", Reason: "KubeletNotReady",
Message: "kubelet has sufficient disk space available", Message: fmt.Sprintf("container runtime is down"),
LastHeartbeatTime: unversioned.Time{}, LastHeartbeatTime: unversioned.Time{},
LastTransitionTime: unversioned.Time{}, LastTransitionTime: unversioned.Time{},
}, },
...@@ -3042,6 +3058,11 @@ func TestUpdateNodeStatusWithoutContainerRuntime(t *testing.T) { ...@@ -3042,6 +3058,11 @@ func TestUpdateNodeStatusWithoutContainerRuntime(t *testing.T) {
updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{} updatedNode.Status.Conditions[i].LastTransitionTime = unversioned.Time{}
} }
// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
if updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type != api.NodeReady {
t.Errorf("unexpected node condition order. NodeReady should be last.")
}
if !reflect.DeepEqual(expectedNode, updatedNode) { if !reflect.DeepEqual(expectedNode, updatedNode) {
t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode)) t.Errorf("unexpected objects: %s", util.ObjectDiff(expectedNode, updatedNode))
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment