Merge pull request #21274 from derekwaynecarr/kubelet_eviction

pkg/api/types.go

@@ -1889,6 +1889,8 @@ const (
 	// NodeOutOfDisk means the kubelet will not accept new pods due to insufficient free disk
 	// space on the node.
 	NodeOutOfDisk NodeConditionType = "OutOfDisk"
+	// NodeMemoryPressure means the kubelet is under pressure due to insufficient available memory.
+	NodeMemoryPressure NodeConditionType = "MemoryPressure"
 )
 
 type NodeCondition struct {

pkg/api/v1/types.go

@@ -2273,6 +2273,8 @@ const (
 	// NodeOutOfDisk means the kubelet will not accept new pods due to insufficient free disk
 	// space on the node.
 	NodeOutOfDisk NodeConditionType = "OutOfDisk"
+	// NodeMemoryPressure means the kubelet is under pressure due to insufficient available memory.
+	NodeMemoryPressure NodeConditionType = "MemoryPressure"
 )
 
 // NodeCondition contains condition infromation for a node.

pkg/kubelet/eviction/manager.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eviction
+
+import (
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/record"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	qosutil "k8s.io/kubernetes/pkg/kubelet/qos/util"
+	"k8s.io/kubernetes/pkg/kubelet/server/stats"
+	"k8s.io/kubernetes/pkg/kubelet/util/format"
+	"k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/util/wait"
+)
+
+// managerImpl implements NodeStabilityManager
+type managerImpl struct {
+	//  used to track time
+	clock util.Clock
+	// config is how the manager is configured
+	config Config
+	// the function to invoke to kill a pod
+	killPodFunc KillPodFunc
+	// protects access to internal state
+	sync.RWMutex
+	// node conditions are the set of conditions present
+	nodeConditions []api.NodeConditionType
+	// captures when a node condition was last observed based on a threshold being met
+	nodeConditionsLastObservedAt nodeConditionsObservedAt
+	// nodeRef is a reference to the node
+	nodeRef *api.ObjectReference
+	// used to record events about the node
+	recorder record.EventRecorder
+	// used to measure usage stats on system
+	summaryProvider stats.SummaryProvider
+	// records when a threshold was first observed
+	thresholdsFirstObservedAt thresholdsObservedAt
+}
+
+// ensure it implements the required interface
+var _ Manager = &managerImpl{}
+
+// NewManager returns a configured Manager and an associated admission handler to enforce eviction configuration.
+func NewManager(
+	summaryProvider stats.SummaryProvider,
+	config Config,
+	killPodFunc KillPodFunc,
+	recorder record.EventRecorder,
+	nodeRef *api.ObjectReference,
+	clock util.Clock) (Manager, lifecycle.PodAdmitHandler, error) {
+	manager := &managerImpl{
+		clock:           clock,
+		killPodFunc:     killPodFunc,
+		config:          config,
+		recorder:        recorder,
+		summaryProvider: summaryProvider,
+		nodeRef:         nodeRef,
+		nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
+		thresholdsFirstObservedAt:    thresholdsObservedAt{},
+	}
+	return manager, manager, nil
+}
+
+// Admit rejects a pod if its not safe to admit for node stability.
+func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAdmitResult {
+	m.RLock()
+	defer m.RUnlock()
+	if len(m.nodeConditions) == 0 {
+		return lifecycle.PodAdmitResult{Admit: true}
+	}
+	notBestEffort := qosutil.BestEffort != qosutil.GetPodQos(attrs.Pod)
+	if notBestEffort {
+		return lifecycle.PodAdmitResult{Admit: true}
+	}
+	glog.Warningf("Failed to admit pod %v - %s", format.Pod(attrs.Pod), "node has conditions: %v", m.nodeConditions)
+	// we reject all best effort pods until we are stable.
+	return lifecycle.PodAdmitResult{
+		Admit:   false,
+		Reason:  reason,
+		Message: message,
+	}
+}
+
+// Start starts the control loop to observe and response to low compute resources.
+func (m *managerImpl) Start(podFunc ActivePodsFunc, monitoringInterval time.Duration) {
+	go wait.Until(func() { m.synchronize(podFunc) }, monitoringInterval, wait.NeverStop)
+}
+
+// IsUnderMemoryPressure returns true if the node is under memory pressure.
+func (m *managerImpl) IsUnderMemoryPressure() bool {
+	m.RLock()
+	defer m.RUnlock()
+	return hasNodeCondition(m.nodeConditions, api.NodeMemoryPressure)
+}
+
+// synchronize is the main control loop that enforces eviction thresholds.
+func (m *managerImpl) synchronize(podFunc ActivePodsFunc) {
+	// if we have nothing to do, just return
+	thresholds := m.config.Thresholds
+	if len(thresholds) == 0 {
+		return
+	}
+
+	// make observations and get a function to derive pod usage stats relative to those observations.
+	observations, statsFunc, err := makeSignalObservations(m.summaryProvider)
+	if err != nil {
+		glog.Errorf("eviction manager: unexpected err: %v", err)
+		return
+	}
+
+	// find the list of thresholds that are met independent of grace period
+	now := m.clock.Now()
+
+	// determine the set of thresholds met independent of grace period
+	thresholds = thresholdsMet(thresholds, observations)
+
+	// track when a threshold was first observed
+	thresholdsFirstObservedAt := thresholdsFirstObservedAt(thresholds, m.thresholdsFirstObservedAt, now)
+
+	// the set of node conditions that are triggered by currently observed thresholds
+	nodeConditions := nodeConditions(thresholds)
+
+	// track when a node condition was last observed
+	nodeConditionsLastObservedAt := nodeConditionsLastObservedAt(nodeConditions, m.nodeConditionsLastObservedAt, now)
+
+	// node conditions report true if it has been observed within the transition period window
+	nodeConditions = nodeConditionsObservedSince(nodeConditionsLastObservedAt, m.config.PressureTransitionPeriod, now)
+
+	// determine the set of thresholds we need to drive eviction behavior (i.e. all grace periods are met)
+	thresholds = thresholdsMetGracePeriod(thresholdsFirstObservedAt, now)
+
+	// update internal state
+	m.Lock()
+	m.nodeConditions = nodeConditions
+	m.thresholdsFirstObservedAt = thresholdsFirstObservedAt
+	m.nodeConditionsLastObservedAt = nodeConditionsLastObservedAt
+	m.Unlock()
+
+	// determine the set of resources under starvation
+	starvedResources := reclaimResources(thresholds)
+	if len(starvedResources) == 0 {
+		glog.Infof("eviction manager: no resources are starved")
+		return
+	}
+
+	// rank the resources to reclaim by eviction priority
+	sort.Sort(byEvictionPriority(starvedResources))
+	resourceToReclaim := starvedResources[0]
+	glog.Warningf("eviction manager: attempting to reclaim %v", resourceToReclaim)
+
+	// record an event about the resources we are now attempting to reclaim via eviction
+	m.recorder.Eventf(m.nodeRef, api.EventTypeWarning, "EvictionThresholdMet", "Attempting to reclaim %s", resourceToReclaim)
+
+	// rank the pods for eviction
+	rank, ok := resourceToRankFunc[resourceToReclaim]
+	if !ok {
+		glog.Errorf("eviction manager: no ranking function for resource %s", resourceToReclaim)
+		return
+	}
+
+	// the only candidates viable for eviction are those pods that had anything running.
+	activePods := podFunc()
+	if len(activePods) == 0 {
+		glog.Errorf("eviction manager: eviction thresholds have been met, but no pods are active to evict")
+		return
+	}
+
+	// rank the running pods for eviction for the specified resource
+	rank(activePods, statsFunc)
+
+	glog.Infof("eviction manager: pods ranked for eviction: %s", format.Pods(activePods))
+
+	// we kill at most a single pod during each eviction interval
+	for i := range activePods {
+		pod := activePods[i]
+		status := api.PodStatus{
+			Phase:   api.PodFailed,
+			Message: message,
+			Reason:  reason,
+		}
+		// record that we are evicting the pod
+		m.recorder.Eventf(pod, api.EventTypeWarning, reason, message)
+		// TODO this needs to be based on soft or hard eviction threshold being met, soft eviction will allow a configured value.
+		gracePeriodOverride := int64(0)
+		// this is a blocking call and should only return when the pod and its containers are killed.
+		err := m.killPodFunc(pod, status, &gracePeriodOverride)
+		if err != nil {
+			glog.Infof("eviction manager: pod %s failed to evict %v", format.Pod(pod), err)
+			continue
+		}
+		// success, so we return until the next housekeeping interval
+		glog.Infof("eviction manager: pod %s evicted successfully", format.Pod(pod))
+		return
+	}
+	glog.Infof("eviction manager: unable to evict any pods from the node")
+}

pkg/kubelet/eviction/manager_test.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package eviction
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/client/record"
+	statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
+	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
+	"k8s.io/kubernetes/pkg/types"
+	"k8s.io/kubernetes/pkg/util"
+)
+
+// mockPodKiller is used to testing which pod is killed
+type mockPodKiller struct {
+	pod                 *api.Pod
+	status              api.PodStatus
+	gracePeriodOverride *int64
+}
+
+// killPodNow records the pod that was killed
+func (m *mockPodKiller) killPodNow(pod *api.Pod, status api.PodStatus, gracePeriodOverride *int64) error {
+	m.pod = pod
+	m.status = status
+	m.gracePeriodOverride = gracePeriodOverride
+	return nil
+}
+
+// TestMemoryPressure
+func TestMemoryPressure(t *testing.T) {
+	podMaker := func(name string, requests api.ResourceList, limits api.ResourceList, memoryWorkingSet string) (*api.Pod, statsapi.PodStats) {
+		pod := newPod(name, []api.Container{
+			newContainer(name, requests, api.ResourceList{}),
+		})
+		podStats := newPodMemoryStats(pod, resource.MustParse(memoryWorkingSet))
+		return pod, podStats
+	}
+	summaryStatsMaker := func(nodeAvailableBytes string, podStats map[*api.Pod]statsapi.PodStats) *statsapi.Summary {
+		val := resource.MustParse(nodeAvailableBytes)
+		availableBytes := uint64(val.Value())
+		result := &statsapi.Summary{
+			Node: statsapi.NodeStats{
+				Memory: &statsapi.MemoryStats{
+					AvailableBytes: &availableBytes,
+				},
+			},
+			Pods: []statsapi.PodStats{},
+		}
+		for _, podStat := range podStats {
+			result.Pods = append(result.Pods, podStat)
+		}
+		return result
+	}
+	podsToMake := []struct {
+		name             string
+		requests         api.ResourceList
+		limits           api.ResourceList
+		memoryWorkingSet string
+	}{
+		{name: "best-effort-high", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "500Mi"},
+		{name: "best-effort-low", requests: newResourceList("", ""), limits: newResourceList("", ""), memoryWorkingSet: "300Mi"},
+		{name: "burstable-high", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "800Mi"},
+		{name: "burstable-low", requests: newResourceList("100m", "100Mi"), limits: newResourceList("200m", "1Gi"), memoryWorkingSet: "300Mi"},
+		{name: "guaranteed-high", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "800Mi"},
+		{name: "guaranteed-low", requests: newResourceList("100m", "1Gi"), limits: newResourceList("100m", "1Gi"), memoryWorkingSet: "200Mi"},
+	}
+	pods := []*api.Pod{}
+	podStats := map[*api.Pod]statsapi.PodStats{}
+	for _, podToMake := range podsToMake {
+		pod, podStat := podMaker(podToMake.name, podToMake.requests, podToMake.limits, podToMake.memoryWorkingSet)
+		pods = append(pods, pod)
+		podStats[pod] = podStat
+	}
+	activePodsFunc := func() []*api.Pod {
+		return pods
+	}
+
+	fakeClock := util.NewFakeClock(time.Now())
+	podKiller := &mockPodKiller{}
+	nodeRef := &api.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""}
+
+	config := Config{
+		PressureTransitionPeriod: time.Minute * 5,
+		Thresholds: []Threshold{
+			{
+				Signal:   SignalMemoryAvailable,
+				Operator: OpLessThan,
+				Value:    resource.MustParse("1Gi"),
+			},
+		},
+	}
+	summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("2Gi", podStats)}
+	manager := &managerImpl{
+		clock:           fakeClock,
+		killPodFunc:     podKiller.killPodNow,
+		config:          config,
+		recorder:        &record.FakeRecorder{},
+		summaryProvider: summaryProvider,
+		nodeRef:         nodeRef,
+		nodeConditionsLastObservedAt: nodeConditionsObservedAt{},
+		thresholdsFirstObservedAt:    thresholdsObservedAt{},
+	}
+
+	// create a best effort pod to test admission
+	bestEffortPodToAdmit, _ := podMaker("best-admit", newResourceList("", ""), newResourceList("", ""), "0Gi")
+	burstablePodToAdmit, _ := podMaker("burst-admit", newResourceList("100m", "100Mi"), newResourceList("200m", "200Mi"), "0Gi")
+
+	// synchronize
+	manager.synchronize(activePodsFunc)
+
+	// we should not have memory pressure
+	if manager.IsUnderMemoryPressure() {
+		t.Errorf("Manager should not report memory pressure")
+	}
+
+	// try to admit our pods (they should succeed)
+	expected := []bool{true, true}
+	for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
+		if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
+			t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
+		}
+	}
+
+	// induce memory pressure!
+	fakeClock.Step(1 * time.Minute)
+	summaryProvider.result = summaryStatsMaker("500Mi", podStats)
+	manager.synchronize(activePodsFunc)
+
+	// we should have memory pressure
+	if !manager.IsUnderMemoryPressure() {
+		t.Errorf("Manager should report memory pressure")
+	}
+
+	// check the right pod was killed
+	if podKiller.pod != pods[0] {
+		t.Errorf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod, pods[0])
+	}
+
+	// the best-effort pod should not admit, burstable should
+	expected = []bool{false, true}
+	for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
+		if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
+			t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
+		}
+	}
+
+	// reduce memory pressure
+	fakeClock.Step(1 * time.Minute)
+	summaryProvider.result = summaryStatsMaker("2Gi", podStats)
+	podKiller.pod = nil // reset state
+	manager.synchronize(activePodsFunc)
+
+	// we should have memory pressure (because transition period not yet met)
+	if !manager.IsUnderMemoryPressure() {
+		t.Errorf("Manager should report memory pressure")
+	}
+
+	// no pod should have been killed
+	if podKiller.pod != nil {
+		t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod)
+	}
+
+	// the best-effort pod should not admit, burstable should
+	expected = []bool{false, true}
+	for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
+		if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
+			t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
+		}
+	}
+
+	// move the clock past transition period to ensure that we stop reporting pressure
+	fakeClock.Step(5 * time.Minute)
+	summaryProvider.result = summaryStatsMaker("2Gi", podStats)
+	podKiller.pod = nil // reset state
+	manager.synchronize(activePodsFunc)
+
+	// we should not have memory pressure (because transition period met)
+	if manager.IsUnderMemoryPressure() {
+		t.Errorf("Manager should not report memory pressure")
+	}
+
+	// no pod should have been killed
+	if podKiller.pod != nil {
+		t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod)
+	}
+
+	// all pods should admit now
+	expected = []bool{true, true}
+	for i, pod := range []*api.Pod{bestEffortPodToAdmit, burstablePodToAdmit} {
+		if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit {
+			t.Errorf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit)
+		}
+	}
+}

pkg/kubelet/eviction/types.go

@@ -21,6 +21,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/resource"
+	statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
 )
 
 // Signal defines a signal that can trigger eviction of pods on a node.
@@ -59,6 +60,15 @@ type Threshold struct {
 	GracePeriod time.Duration
 }
 
+// Manager evaluates when an eviction threshold for node stability has been met on the node.
+type Manager interface {
+	// Start starts the control loop to monitor eviction thresholds at specified interval.
+	Start(podFunc ActivePodsFunc, monitoringInterval time.Duration)
+
+	// IsUnderMemoryPressure returns true if the node is under memory pressure.
+	IsUnderMemoryPressure() bool
+}
+
 // KillPodFunc kills a pod.
 // The pod status is updated, and then it is killed with the specified grace period.
 // This function must block until either the pod is killed or an error is encountered.
@@ -67,3 +77,21 @@ type Threshold struct {
 // status - the desired status to associate with the pod (i.e. why its killed)
 // gracePeriodOverride - the grace period override to use instead of what is on the pod spec
 type KillPodFunc func(pod *api.Pod, status api.PodStatus, gracePeriodOverride *int64) error
+
+// ActivePodsFunc returns pods bound to the kubelet that are active (i.e. non-terminal state)
+type ActivePodsFunc func() []*api.Pod
+
+// statsFunc returns the usage stats if known for an input pod.
+type statsFunc func(pod *api.Pod) (statsapi.PodStats, bool)
+
+// rankFunc sorts the pods in eviction order
+type rankFunc func(pods []*api.Pod, stats statsFunc)
+
+// signalObservations maps a signal to an observed quantity
+type signalObservations map[Signal]resource.Quantity
+
+// thresholdsObservedAt maps a threshold to a time that it was observed
+type thresholdsObservedAt map[Threshold]time.Time
+
+// nodeConditionsObservedAt maps a node condition to a time that it was observed
+type nodeConditionsObservedAt map[api.NodeConditionType]time.Time

pkg/kubelet/kubelet.go

@@ -118,6 +118,10 @@ const (
 	// Period for performing global cleanup tasks.
 	housekeepingPeriod = time.Second * 2
 
+	// Period for performing eviction monitoring.
+	// TODO ensure this is in sync with internal cadvisor housekeeping.
+	evictionMonitoringPeriod = time.Second * 10
+
 	// The path in containers' filesystems where the hosts file is mounted.
 	etcHostsPath = "/etc/hosts"
 
@@ -496,6 +500,14 @@ func NewMainKubelet(
 	klet.podKillingCh = make(chan *kubecontainer.PodPair, podKillingChannelCapacity)
 	klet.setNodeStatusFuncs = klet.defaultNodeStatusFuncs()
 
+	// setup eviction manager
+	evictionManager, evictionAdmitHandler, err := eviction.NewManager(klet.resourceAnalyzer, evictionConfig, killPodNow(klet.podWorkers), recorder, nodeRef, klet.clock)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize eviction manager: %v", err)
+	}
+	klet.evictionManager = evictionManager
+	klet.AddPodAdmitHandler(evictionAdmitHandler)
+
 	// apply functional Option's
 	for _, opt := range kubeOptions {
 		opt(klet)
@@ -568,6 +580,9 @@ type Kubelet struct {
 	// this Kubelet services.
 	podManager kubepod.Manager
 
+	// Needed to observe and respond to situations that could impact node stability
+	evictionManager eviction.Manager
+
 	// Needed to report events for containers belonging to deleted/modified pods.
 	// Tracks references for reporting events
 	containerRefManager *kubecontainer.RefManager
@@ -961,12 +976,20 @@ func (kl *Kubelet) Run(updates <-chan kubetypes.PodUpdate) {
 	// Start component sync loops.
 	kl.statusManager.Start()
 	kl.probeManager.Start()
+	kl.evictionManager.Start(kl.getActivePods, evictionMonitoringPeriod)
 
 	// Start the pod lifecycle event generator.
 	kl.pleg.Start()
 	kl.syncLoop(updates, kl)
 }
 
+// getActivePods returns non-terminal pods
+func (kl *Kubelet) getActivePods() []*api.Pod {
+	allPods := kl.podManager.GetPods()
+	activePods := kl.filterOutTerminatedPods(allPods)
+	return activePods
+}
+
 // initialNodeStatus determines the initial node status, incorporating node
 // labels and information from the cloud provider.
 func (kl *Kubelet) initialNodeStatus() (*api.Node, error) {
@@ -3152,6 +3175,64 @@ func (kl *Kubelet) setNodeReadyCondition(node *api.Node) {
 	}
 }
 
+// setNodeMemoryPressureCondition for the node.
+// TODO: this needs to move somewhere centralized...
+func (kl *Kubelet) setNodeMemoryPressureCondition(node *api.Node) {
+	currentTime := unversioned.NewTime(kl.clock.Now())
+	var condition *api.NodeCondition
+
+	// Check if NodeMemoryPressure condition already exists and if it does, just pick it up for update.
+	for i := range node.Status.Conditions {
+		if node.Status.Conditions[i].Type == api.NodeMemoryPressure {
+			condition = &node.Status.Conditions[i]
+		}
+	}
+
+	newCondition := false
+	// If the NodeMemoryPressure condition doesn't exist, create one
+	if condition == nil {
+		condition = &api.NodeCondition{
+			Type:   api.NodeMemoryPressure,
+			Status: api.ConditionUnknown,
+		}
+		// cannot be appended to node.Status.Conditions here because it gets
+		// copied to the slice. So if we append to the slice here none of the
+		// updates we make below are reflected in the slice.
+		newCondition = true
+	}
+
+	// Update the heartbeat time
+	condition.LastHeartbeatTime = currentTime
+
+	// Note: The conditions below take care of the case when a new NodeMemoryPressure condition is
+	// created and as well as the case when the condition already exists. When a new condition
+	// is created its status is set to api.ConditionUnknown which matches either
+	// condition.Status != api.ConditionTrue or
+	// condition.Status != api.ConditionFalse in the conditions below depending on whether
+	// the kubelet is under memory pressure or not.
+	if kl.evictionManager.IsUnderMemoryPressure() {
+		if condition.Status != api.ConditionTrue {
+			condition.Status = api.ConditionTrue
+			condition.Reason = "KubeletHasInsufficientMemory"
+			condition.Message = "kubelet has insufficient memory available"
+			condition.LastTransitionTime = currentTime
+			kl.recordNodeStatusEvent(api.EventTypeNormal, "NodeHasInsufficientMemory")
+		}
+	} else {
+		if condition.Status != api.ConditionFalse {
+			condition.Status = api.ConditionFalse
+			condition.Reason = "KubeletHasSufficientMemory"
+			condition.Message = "kubelet has sufficient memory available"
+			condition.LastTransitionTime = currentTime
+			kl.recordNodeStatusEvent(api.EventTypeNormal, "NodeHasSufficientMemory")
+		}
+	}
+
+	if newCondition {
+		node.Status.Conditions = append(node.Status.Conditions, *condition)
+	}
+}
+
 // Set OODcondition for the node.
 func (kl *Kubelet) setNodeOODCondition(node *api.Node) {
 	currentTime := unversioned.NewTime(kl.clock.Now())
@@ -3257,6 +3338,7 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*api.Node) error {
 		kl.setNodeAddress,
 		withoutError(kl.setNodeStatusInfo),
 		withoutError(kl.setNodeOODCondition),
+		withoutError(kl.setNodeMemoryPressureCondition),
 		withoutError(kl.setNodeReadyCondition),
 		withoutError(kl.recordNodeSchedulableEvent),
 	}

pkg/kubelet/kubelet_test.go

@@ -47,6 +47,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/config"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	"k8s.io/kubernetes/pkg/kubelet/lifecycle"
 	"k8s.io/kubernetes/pkg/kubelet/network"
 	nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
@@ -55,6 +56,7 @@ import (
 	podtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
 	proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
 	probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
+	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	"k8s.io/kubernetes/pkg/kubelet/status"
 	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
 	"k8s.io/kubernetes/pkg/kubelet/util/queue"
@@ -202,6 +204,24 @@ func newTestKubelet(t *testing.T) *TestKubelet {
 	kubelet.pleg = pleg.NewGenericPLEG(fakeRuntime, 100, time.Hour, nil, util.RealClock{})
 	kubelet.clock = fakeClock
 	kubelet.setNodeStatusFuncs = kubelet.defaultNodeStatusFuncs()
+
+	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
+	volumeStatsAggPeriod := time.Second * 10
+	kubelet.resourceAnalyzer = stats.NewResourceAnalyzer(kubelet, volumeStatsAggPeriod, kubelet.containerRuntime)
+	nodeRef := &api.ObjectReference{
+		Kind:      "Node",
+		Name:      kubelet.nodeName,
+		UID:       types.UID(kubelet.nodeName),
+		Namespace: "",
+	}
+	// setup eviction manager
+	evictionManager, evictionAdmitHandler, err := eviction.NewManager(kubelet.resourceAnalyzer, eviction.Config{}, killPodNow(kubelet.podWorkers), fakeRecorder, nodeRef, kubelet.clock)
+	if err != nil {
+		t.Fatalf("failed to initialize eviction manager: %v", err)
+	}
+	kubelet.evictionManager = evictionManager
+	kubelet.AddPodAdmitHandler(evictionAdmitHandler)
+
 	return &TestKubelet{kubelet, fakeRuntime, mockCadvisor, fakeKubeClient, fakeMirrorClient, fakeClock, nil}
 }
 
@@ -2370,6 +2390,14 @@ func TestUpdateNewNodeStatus(t *testing.T) {
 					LastTransitionTime: unversioned.Time{},
 				},
 				{
+					Type:               api.NodeMemoryPressure,
+					Status:             api.ConditionFalse,
+					Reason:             "KubeletHasSufficientMemory",
+					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
+					LastHeartbeatTime:  unversioned.Time{},
+					LastTransitionTime: unversioned.Time{},
+				},
+				{
 					Type:               api.NodeReady,
 					Status:             api.ConditionTrue,
 					Reason:             "KubeletReady",
@@ -2549,6 +2577,14 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
 						LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
 					},
 					{
+						Type:               api.NodeMemoryPressure,
+						Status:             api.ConditionFalse,
+						Reason:             "KubeletHasSufficientMemory",
+						Message:            fmt.Sprintf("kubelet has sufficient memory available"),
+						LastHeartbeatTime:  unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+						LastTransitionTime: unversioned.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
+					},
+					{
 						Type:               api.NodeReady,
 						Status:             api.ConditionTrue,
 						Reason:             "KubeletReady",
@@ -2605,6 +2641,14 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
 					LastTransitionTime: unversioned.Time{}, // placeholder
 				},
 				{
+					Type:               api.NodeMemoryPressure,
+					Status:             api.ConditionFalse,
+					Reason:             "KubeletHasSufficientMemory",
+					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
+					LastHeartbeatTime:  unversioned.Time{},
+					LastTransitionTime: unversioned.Time{},
+				},
+				{
 					Type:               api.NodeReady,
 					Status:             api.ConditionTrue,
 					Reason:             "KubeletReady",
@@ -2886,6 +2930,14 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
 					LastHeartbeatTime:  unversioned.Time{},
 					LastTransitionTime: unversioned.Time{},
 				},
+				{
+					Type:               api.NodeMemoryPressure,
+					Status:             api.ConditionFalse,
+					Reason:             "KubeletHasSufficientMemory",
+					Message:            fmt.Sprintf("kubelet has sufficient memory available"),
+					LastHeartbeatTime:  unversioned.Time{},
+					LastTransitionTime: unversioned.Time{},
+				},
 				{}, //placeholder
 			},
 			NodeInfo: api.NodeSystemInfo{
@@ -2958,10 +3010,11 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
 		}
 
 		// Version skew workaround. See: https://github.com/kubernetes/kubernetes/issues/16961
-		if updatedNode.Status.Conditions[len(updatedNode.Status.Conditions)-1].Type != api.NodeReady {
+		lastIndex := len(updatedNode.Status.Conditions) - 1
+		if updatedNode.Status.Conditions[lastIndex].Type != api.NodeReady {
 			t.Errorf("unexpected node condition order. NodeReady should be last.")
 		}
-		expectedNode.Status.Conditions[1] = api.NodeCondition{
+		expectedNode.Status.Conditions[lastIndex] = api.NodeCondition{
 			Type:               api.NodeReady,
 			Status:             status,
 			Reason:             reason,

pkg/kubelet/runonce_test.go

@@ -31,11 +31,14 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/cm"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/eviction"
 	"k8s.io/kubernetes/pkg/kubelet/network"
 	nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
 	podtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
+	"k8s.io/kubernetes/pkg/kubelet/server/stats"
 	"k8s.io/kubernetes/pkg/kubelet/status"
+	"k8s.io/kubernetes/pkg/types"
 	"k8s.io/kubernetes/pkg/util"
 	utiltesting "k8s.io/kubernetes/pkg/util/testing"
 )
@@ -76,10 +79,30 @@ func TestRunOnce(t *testing.T) {
 		reasonCache:         NewReasonCache(),
 		clock:               util.RealClock{},
 		kubeClient:          &fake.Clientset{},
+		hostname:            testKubeletHostname,
+		nodeName:            testKubeletHostname,
 	}
 	kb.containerManager = cm.NewStubContainerManager()
 
 	kb.networkPlugin, _ = network.InitNetworkPlugin([]network.NetworkPlugin{}, "", nettest.NewFakeHost(nil), componentconfig.HairpinNone)
+	// TODO: Factor out "StatsProvider" from Kubelet so we don't have a cyclic dependency
+	volumeStatsAggPeriod := time.Second * 10
+	kb.resourceAnalyzer = stats.NewResourceAnalyzer(kb, volumeStatsAggPeriod, kb.containerRuntime)
+	nodeRef := &api.ObjectReference{
+		Kind:      "Node",
+		Name:      kb.nodeName,
+		UID:       types.UID(kb.nodeName),
+		Namespace: "",
+	}
+	fakeKillPodFunc := func(pod *api.Pod, podStatus api.PodStatus, gracePeriodOverride *int64) error {
+		return nil
+	}
+	evictionManager, evictionAdmitHandler, err := eviction.NewManager(kb.resourceAnalyzer, eviction.Config{}, fakeKillPodFunc, kb.recorder, nodeRef, kb.clock)
+	if err != nil {
+		t.Fatalf("failed to initialize eviction manager: %v", err)
+	}
+	kb.evictionManager = evictionManager
+	kb.AddPodAdmitHandler(evictionAdmitHandler)
 	if err := kb.setupDataDirs(); err != nil {
 		t.Errorf("Failed to init data dirs: %v", err)
 	}