Set reason and message on Pod during nodecontroller eviction

pkg/controller/node/controller_utils.go

@@ -21,6 +21,7 @@ import (
 	"strings"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/errors"
 	"k8s.io/kubernetes/pkg/client/cache"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/client/record"
@@ -28,6 +29,8 @@ import (
 	"k8s.io/kubernetes/pkg/fields"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
 	"k8s.io/kubernetes/pkg/types"
+	utilerrors "k8s.io/kubernetes/pkg/util/errors"
+	"k8s.io/kubernetes/pkg/util/node"
 	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
 	"k8s.io/kubernetes/pkg/version"
 
@@ -46,6 +49,8 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n
 	selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
 	options := api.ListOptions{FieldSelector: selector}
 	pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(options)
+	var updateErrList []error
+
 	if err != nil {
 		return remaining, err
 	}
@@ -59,6 +64,15 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n
 		if pod.Spec.NodeName != nodeName {
 			continue
 		}
+
+		// Set reason and message in the pod object.
+		if _, err = setPodTerminationReason(kubeClient, &pod, nodeName); err != nil {
+			if errors.IsConflict(err) {
+				updateErrList = append(updateErrList,
+					fmt.Errorf("update status failed for pod %q: %v", format.Pod(&pod), err))
+				continue
+			}
+		}
 		// if the pod has already been marked for deletion, we still return true that there are remaining pods.
 		if pod.DeletionGracePeriodSeconds != nil {
 			remaining = true
@@ -77,9 +91,31 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n
 		}
 		remaining = true
 	}
+
+	if len(updateErrList) > 0 {
+		return false, utilerrors.NewAggregate(updateErrList)
+	}
 	return remaining, nil
 }
 
+// setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver,
+// and returns an error if it encounters one.
+func setPodTerminationReason(kubeClient clientset.Interface, pod *api.Pod, nodeName string) (*api.Pod, error) {
+	if pod.Status.Reason == node.NodeUnreachablePodReason {
+		return pod, nil
+	}
+
+	pod.Status.Reason = node.NodeUnreachablePodReason
+	pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name)
+
+	var updatedPod *api.Pod
+	var err error
+	if updatedPod, err = kubeClient.Core().Pods(pod.Namespace).UpdateStatus(pod); err != nil {
+		return nil, err
+	}
+	return updatedPod, nil
+}
+
 func forcefullyDeletePod(c clientset.Interface, pod *api.Pod) error {
 	var zero int64
 	err := c.Core().Pods(pod.Namespace).Delete(pod.Name, &api.DeleteOptions{GracePeriodSeconds: &zero})

pkg/kubectl/resource_printer.go

@@ -30,8 +30,6 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/ghodss/yaml"
-	"github.com/golang/glog"
 	"k8s.io/kubernetes/federation/apis/federation"
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/events"
@@ -49,7 +47,11 @@ import (
 	"k8s.io/kubernetes/pkg/runtime"
 	utilerrors "k8s.io/kubernetes/pkg/util/errors"
 	"k8s.io/kubernetes/pkg/util/jsonpath"
+	"k8s.io/kubernetes/pkg/util/node"
 	"k8s.io/kubernetes/pkg/util/sets"
+
+	"github.com/ghodss/yaml"
+	"github.com/golang/glog"
 )
 
 const (
@@ -731,7 +733,10 @@ func printPodBase(pod *api.Pod, w io.Writer, options PrintOptions) error {
 			}
 		}
 	}
-	if pod.DeletionTimestamp != nil {
+
+	if pod.DeletionTimestamp != nil && pod.Status.Reason == node.NodeUnreachablePodReason {
+		reason = "Unknown"
+	} else if pod.DeletionTimestamp != nil {
 		reason = "Terminating"
 	}
 

pkg/util/node/node.go

@@ -31,6 +31,13 @@ import (
 	"k8s.io/kubernetes/pkg/types"
 )
 
+const (
+	// The reason and message set on a pod when its state cannot be confirmed as kubelet is unresponsive
+	// on the node it is (was) running.
+	NodeUnreachablePodReason  = "NodeLost"
+	NodeUnreachablePodMessage = "Node %v which was running pod %v is unresponsive"
+)
+
 func GetHostname(hostnameOverride string) string {
 	var hostname string = hostnameOverride
 	if hostname == "" {