Merge pull request #42204 from dashpole/allocatable_eviction

cmd/kubelet/app/server.go

@@ -520,7 +520,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 		var hardEvictionThresholds []evictionapi.Threshold
 		// If the user requested to ignore eviction thresholds, then do not set valid values for hardEvictionThresholds here.
 		if !s.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
-			hardEvictionThresholds, err = eviction.ParseThresholdConfig(s.EvictionHard, "", "", "")
+			hardEvictionThresholds, err = eviction.ParseThresholdConfig([]string{}, s.EvictionHard, "", "", "")
 			if err != nil {
 				return err
 			}

pkg/kubelet/eviction/BUILD

@@ -33,6 +33,7 @@ go_test(
         "//pkg/api:go_default_library",
         "//pkg/api/v1:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
+        "//pkg/kubelet/cm:go_default_library",
         "//pkg/kubelet/eviction/api:go_default_library",
         "//pkg/kubelet/lifecycle:go_default_library",
         "//pkg/kubelet/types:go_default_library",

pkg/kubelet/eviction/api/types.go

@@ -36,6 +36,8 @@ const (
 	SignalImageFsAvailable Signal = "imagefs.available"
 	// SignalImageFsInodesFree is amount of inodes available on filesystem that container runtime uses for storing images and container writeable layers.
 	SignalImageFsInodesFree Signal = "imagefs.inodesFree"
+	// SignalAllocatableMemoryAvailable is amount of memory available for pod allocation (i.e. allocatable - workingSet (of pods), in bytes.
+	SignalAllocatableMemoryAvailable Signal = "allocatableMemory.available"
 )
 
 // ThresholdOperator is the operator used to express a Threshold.

pkg/kubelet/eviction/eviction_manager.go

@@ -134,9 +134,9 @@ func (m *managerImpl) Admit(attrs *lifecycle.PodAdmitAttributes) lifecycle.PodAd
 }
 
 // Start starts the control loop to observe and response to low compute resources.
-func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration) {
+func (m *managerImpl) Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider, monitoringInterval time.Duration) {
 	// start the eviction manager monitoring
-	go wait.Until(func() { m.synchronize(diskInfoProvider, podFunc) }, monitoringInterval, wait.NeverStop)
+	go wait.Until(func() { m.synchronize(diskInfoProvider, podFunc, nodeProvider) }, monitoringInterval, wait.NeverStop)
 }
 
 // IsUnderMemoryPressure returns true if the node is under memory pressure.
@@ -187,7 +187,7 @@ func startMemoryThresholdNotifier(thresholds []evictionapi.Threshold, observatio
 }
 
 // synchronize is the main control loop that enforces eviction thresholds.
-func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc) {
+func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider) {
 	// if we have nothing to do, just return
 	thresholds := m.config.Thresholds
 	if len(thresholds) == 0 {
@@ -209,7 +209,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 	}
 
 	// make observations and get a function to derive pod usage stats relative to those observations.
-	observations, statsFunc, err := makeSignalObservations(m.summaryProvider)
+	observations, statsFunc, err := makeSignalObservations(m.summaryProvider, nodeProvider)
 	if err != nil {
 		glog.Errorf("eviction manager: unexpected err: %v", err)
 		return
@@ -224,7 +224,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		err = startMemoryThresholdNotifier(m.config.Thresholds, observations, false, func(desc string) {
 			glog.Infof("soft memory eviction threshold crossed at %s", desc)
 			// TODO wait grace period for soft memory limit
-			m.synchronize(diskInfoProvider, podFunc)
+			m.synchronize(diskInfoProvider, podFunc, nodeProvider)
 		})
 		if err != nil {
 			glog.Warningf("eviction manager: failed to create hard memory threshold notifier: %v", err)
@@ -232,7 +232,7 @@ func (m *managerImpl) synchronize(diskInfoProvider DiskInfoProvider, podFunc Act
 		// start hard memory notification
 		err = startMemoryThresholdNotifier(m.config.Thresholds, observations, true, func(desc string) {
 			glog.Infof("hard memory eviction threshold crossed at %s", desc)
-			m.synchronize(diskInfoProvider, podFunc)
+			m.synchronize(diskInfoProvider, podFunc, nodeProvider)
 		})
 		if err != nil {
 			glog.Warningf("eviction manager: failed to create soft memory threshold notifier: %v", err)

pkg/kubelet/eviction/helpers.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
+	"k8s.io/kubernetes/pkg/kubelet/cm"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	"k8s.io/kubernetes/pkg/kubelet/qos"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -68,6 +69,7 @@ func init() {
 	// map eviction signals to node conditions
 	signalToNodeCondition = map[evictionapi.Signal]v1.NodeConditionType{}
 	signalToNodeCondition[evictionapi.SignalMemoryAvailable] = v1.NodeMemoryPressure
+	signalToNodeCondition[evictionapi.SignalAllocatableMemoryAvailable] = v1.NodeMemoryPressure
 	signalToNodeCondition[evictionapi.SignalImageFsAvailable] = v1.NodeDiskPressure
 	signalToNodeCondition[evictionapi.SignalNodeFsAvailable] = v1.NodeDiskPressure
 	signalToNodeCondition[evictionapi.SignalImageFsInodesFree] = v1.NodeDiskPressure
@@ -76,6 +78,7 @@ func init() {
 	// map signals to resources (and vice-versa)
 	signalToResource = map[evictionapi.Signal]v1.ResourceName{}
 	signalToResource[evictionapi.SignalMemoryAvailable] = v1.ResourceMemory
+	signalToResource[evictionapi.SignalAllocatableMemoryAvailable] = v1.ResourceMemory
 	signalToResource[evictionapi.SignalImageFsAvailable] = resourceImageFs
 	signalToResource[evictionapi.SignalImageFsInodesFree] = resourceImageFsInodes
 	signalToResource[evictionapi.SignalNodeFsAvailable] = resourceNodeFs
@@ -93,8 +96,10 @@ func validSignal(signal evictionapi.Signal) bool {
 }
 
 // ParseThresholdConfig parses the flags for thresholds.
-func ParseThresholdConfig(evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim string) ([]evictionapi.Threshold, error) {
+func ParseThresholdConfig(allocatableConfig []string, evictionHard, evictionSoft, evictionSoftGracePeriod, evictionMinimumReclaim string) ([]evictionapi.Threshold, error) {
 	results := []evictionapi.Threshold{}
+	allocatableThresholds := getAllocatableThreshold(allocatableConfig)
+	results = append(results, allocatableThresholds...)
 
 	hardThresholds, err := parseThresholdStatements(evictionHard)
 	if err != nil {
@@ -214,6 +219,27 @@ func parseThresholdStatement(statement string) (evictionapi.Threshold, error) {
 	}, nil
 }
 
+// getAllocatableThreshold returns the thresholds applicable for the allocatable configuration
+func getAllocatableThreshold(allocatableConfig []string) []evictionapi.Threshold {
+	for _, key := range allocatableConfig {
+		if key == cm.NodeAllocatableEnforcementKey {
+			return []evictionapi.Threshold{
+				{
+					Signal:   evictionapi.SignalAllocatableMemoryAvailable,
+					Operator: evictionapi.OpLessThan,
+					Value: evictionapi.ThresholdValue{
+						Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
+					},
+					MinReclaim: &evictionapi.ThresholdValue{
+						Quantity: resource.NewQuantity(int64(0), resource.BinarySI),
+					},
+				},
+			}
+		}
+	}
+	return []evictionapi.Threshold{}
+}
+
 // parsePercentage parses a string representing a percentage value
 func parsePercentage(input string) (float32, error) {
 	value, err := strconv.ParseFloat(strings.TrimRight(input, "%"), 32)
@@ -611,11 +637,15 @@ func (a byEvictionPriority) Less(i, j int) bool {
 }
 
 // makeSignalObservations derives observations using the specified summary provider.
-func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObservations, statsFunc, error) {
+func makeSignalObservations(summaryProvider stats.SummaryProvider, nodeProvider NodeProvider) (signalObservations, statsFunc, error) {
 	summary, err := summaryProvider.Get()
 	if err != nil {
 		return nil, nil, err
 	}
+	node, err := nodeProvider.GetNode()
+	if err != nil {
+		return nil, nil, err
+	}
 
 	// build the function to work against for pod stats
 	statsFunc := cachedStatsFunc(summary.Pods)
@@ -663,6 +693,19 @@ func makeSignalObservations(summaryProvider stats.SummaryProvider) (signalObserv
 			}
 		}
 	}
+	if memoryAllocatableCapacity, ok := node.Status.Allocatable[v1.ResourceMemory]; ok {
+		memoryAllocatableAvailable := memoryAllocatableCapacity.Copy()
+		for _, pod := range summary.Pods {
+			mu, err := podMemoryUsage(pod)
+			if err == nil {
+				memoryAllocatableAvailable.Sub(mu[v1.ResourceMemory])
+			}
+		}
+		result[evictionapi.SignalAllocatableMemoryAvailable] = signalObservation{
+			available: memoryAllocatableAvailable,
+			capacity:  memoryAllocatableCapacity.Copy(),
+		}
+	}
 	return result, statsFunc, nil
 }
 

pkg/kubelet/eviction/helpers_test.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/v1"
 	statsapi "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/stats"
+	"k8s.io/kubernetes/pkg/kubelet/cm"
 	evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api"
 	"k8s.io/kubernetes/pkg/quota"
 )
@@ -40,6 +41,7 @@ func quantityMustParse(value string) *resource.Quantity {
 func TestParseThresholdConfig(t *testing.T) {
 	gracePeriod, _ := time.ParseDuration("30s")
 	testCases := map[string]struct {
+		allocatableConfig       []string
 		evictionHard            string
 		evictionSoft            string
 		evictionSoftGracePeriod string
@@ -48,6 +50,7 @@ func TestParseThresholdConfig(t *testing.T) {
 		expectThresholds        []evictionapi.Threshold
 	}{
 		"no values": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -56,6 +59,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"all flag values": {
+			allocatableConfig:       []string{cm.NodeAllocatableEnforcementKey},
 			evictionHard:            "memory.available<150Mi",
 			evictionSoft:            "memory.available<300Mi",
 			evictionSoftGracePeriod: "memory.available=30s",
@@ -63,6 +67,16 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectErr:               false,
 			expectThresholds: []evictionapi.Threshold{
 				{
+					Signal:   evictionapi.SignalAllocatableMemoryAvailable,
+					Operator: evictionapi.OpLessThan,
+					Value: evictionapi.ThresholdValue{
+						Quantity: quantityMustParse("0"),
+					},
+					MinReclaim: &evictionapi.ThresholdValue{
+						Quantity: quantityMustParse("0"),
+					},
+				},
+				{
 					Signal:   evictionapi.SignalMemoryAvailable,
 					Operator: evictionapi.OpLessThan,
 					Value: evictionapi.ThresholdValue{
@@ -86,6 +100,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			},
 		},
 		"all flag values in percentages": {
+			allocatableConfig:       []string{},
 			evictionHard:            "memory.available<10%",
 			evictionSoft:            "memory.available<30%",
 			evictionSoftGracePeriod: "memory.available=30s",
@@ -116,6 +131,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			},
 		},
 		"disk flag values": {
+			allocatableConfig:       []string{},
 			evictionHard:            "imagefs.available<150Mi,nodefs.available<100Mi",
 			evictionSoft:            "imagefs.available<300Mi,nodefs.available<200Mi",
 			evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s",
@@ -167,6 +183,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			},
 		},
 		"disk flag values in percentages": {
+			allocatableConfig:       []string{},
 			evictionHard:            "imagefs.available<15%,nodefs.available<10.5%",
 			evictionSoft:            "imagefs.available<30%,nodefs.available<20.5%",
 			evictionSoftGracePeriod: "imagefs.available=30s,nodefs.available=30s",
@@ -218,6 +235,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			},
 		},
 		"inode flag values": {
+			allocatableConfig:       []string{},
 			evictionHard:            "imagefs.inodesFree<150Mi,nodefs.inodesFree<100Mi",
 			evictionSoft:            "imagefs.inodesFree<300Mi,nodefs.inodesFree<200Mi",
 			evictionSoftGracePeriod: "imagefs.inodesFree=30s,nodefs.inodesFree=30s",
@@ -269,6 +287,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			},
 		},
 		"invalid-signal": {
+			allocatableConfig:       []string{},
 			evictionHard:            "mem.available<150Mi",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -277,6 +296,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"hard-signal-negative": {
+			allocatableConfig:       []string{},
 			evictionHard:            "memory.available<-150Mi",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -285,6 +305,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"hard-signal-negative-percentage": {
+			allocatableConfig:       []string{},
 			evictionHard:            "memory.available<-15%",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -293,6 +314,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"soft-signal-negative": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "memory.available<-150Mi",
 			evictionSoftGracePeriod: "",
@@ -301,6 +323,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"duplicate-signal": {
+			allocatableConfig:       []string{},
 			evictionHard:            "memory.available<150Mi,memory.available<100Mi",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -309,6 +332,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"valid-and-invalid-signal": {
+			allocatableConfig:       []string{},
 			evictionHard:            "memory.available<150Mi,invalid.foo<150Mi",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -317,6 +341,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"soft-no-grace-period": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "memory.available<150Mi",
 			evictionSoftGracePeriod: "",
@@ -325,6 +350,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"soft-neg-grace-period": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "memory.available<150Mi",
 			evictionSoftGracePeriod: "memory.available=-30s",
@@ -333,6 +359,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"neg-reclaim": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -341,6 +368,7 @@ func TestParseThresholdConfig(t *testing.T) {
 			expectThresholds:        []evictionapi.Threshold{},
 		},
 		"duplicate-reclaim": {
+			allocatableConfig:       []string{},
 			evictionHard:            "",
 			evictionSoft:            "",
 			evictionSoftGracePeriod: "",
@@ -350,7 +378,7 @@ func TestParseThresholdConfig(t *testing.T) {
 		},
 	}
 	for testName, testCase := range testCases {
-		thresholds, err := ParseThresholdConfig(testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod, testCase.evictionMinReclaim)
+		thresholds, err := ParseThresholdConfig(testCase.allocatableConfig, testCase.evictionHard, testCase.evictionSoft, testCase.evictionSoftGracePeriod, testCase.evictionMinReclaim)
 		if testCase.expectErr != (err != nil) {
 			t.Errorf("Err not as expected, test: %v, error expected: %v, actual: %v", testName, testCase.expectErr, err)
 		}
@@ -699,6 +727,7 @@ func TestMakeSignalObservations(t *testing.T) {
 	}
 	nodeAvailableBytes := uint64(1024 * 1024 * 1024)
 	nodeWorkingSetBytes := uint64(1024 * 1024 * 1024)
+	allocatableMemoryCapacity := uint64(5 * 1024 * 1024 * 1024)
 	imageFsAvailableBytes := uint64(1024 * 1024)
 	imageFsCapacityBytes := uint64(1024 * 1024 * 2)
 	nodeFsAvailableBytes := uint64(1024)
@@ -738,15 +767,31 @@ func TestMakeSignalObservations(t *testing.T) {
 		podMaker("pod1", "ns2", "uuid2", 1),
 		podMaker("pod3", "ns3", "uuid3", 1),
 	}
-	containerWorkingSetBytes := int64(1024 * 1024)
+	containerWorkingSetBytes := int64(1024 * 1024 * 1024)
 	for _, pod := range pods {
 		fakeStats.Pods = append(fakeStats.Pods, newPodStats(pod, containerWorkingSetBytes))
 	}
-	actualObservations, statsFunc, err := makeSignalObservations(provider)
+	res := quantityMustParse("5Gi")
+	nodeProvider := newMockNodeProvider(v1.ResourceList{v1.ResourceMemory: *res})
+	// Allocatable thresholds are always 100%.  Verify that Threshold == Capacity.
+	if res.CmpInt64(int64(allocatableMemoryCapacity)) != 0 {
+		t.Errorf("Expected Threshold %v to be equal to value %v", res.Value(), allocatableMemoryCapacity)
+	}
+	actualObservations, statsFunc, err := makeSignalObservations(provider, nodeProvider)
 
 	if err != nil {
 		t.Errorf("Unexpected err: %v", err)
 	}
+	allocatableMemQuantity, found := actualObservations[evictionapi.SignalAllocatableMemoryAvailable]
+	if !found {
+		t.Errorf("Expected allocatable memory observation, but didnt find one")
+	}
+	if allocatableMemQuantity.available.Value() != 2*containerWorkingSetBytes {
+		t.Errorf("Expected %v, actual: %v", containerWorkingSetBytes, allocatableMemQuantity.available.Value())
+	}
+	if expectedBytes := int64(allocatableMemoryCapacity); allocatableMemQuantity.capacity.Value() != expectedBytes {
+		t.Errorf("Expected %v, actual: %v", expectedBytes, allocatableMemQuantity.capacity.Value())
+	}
 	memQuantity, found := actualObservations[evictionapi.SignalMemoryAvailable]
 	if !found {
 		t.Errorf("Expected available memory observation: %v", err)

pkg/kubelet/eviction/types.go

@@ -53,7 +53,7 @@ type Config struct {
 // Manager evaluates when an eviction threshold for node stability has been met on the node.
 type Manager interface {
 	// Start starts the control loop to monitor eviction thresholds at specified interval.
-	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, monitoringInterval time.Duration)
+	Start(diskInfoProvider DiskInfoProvider, podFunc ActivePodsFunc, nodeProvider NodeProvider, monitoringInterval time.Duration)
 
 	// IsUnderMemoryPressure returns true if the node is under memory pressure.
 	IsUnderMemoryPressure() bool
@@ -68,6 +68,12 @@ type DiskInfoProvider interface {
 	HasDedicatedImageFs() (bool, error)
 }
 
+// NodeProvider is responsible for providing the node api object describing this node
+type NodeProvider interface {
+	// GetNode returns the node info for this node
+	GetNode() (*v1.Node, error)
+}
+
 // ImageGC is responsible for performing garbage collection of unused images.
 type ImageGC interface {
 	// DeleteUnusedImages deletes unused images and returns the number of bytes freed, or an error.

pkg/kubelet/kubelet.go

@@ -349,7 +349,12 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 		RootFreeDiskMB:   int(kubeCfg.LowDiskSpaceThresholdMB),
 	}
 
-	thresholds, err := eviction.ParseThresholdConfig(kubeCfg.EvictionHard, kubeCfg.EvictionSoft, kubeCfg.EvictionSoftGracePeriod, kubeCfg.EvictionMinimumReclaim)
+	enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
+	if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
+		// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
+		enforceNodeAllocatable = []string{}
+	}
+	thresholds, err := eviction.ParseThresholdConfig(enforceNodeAllocatable, kubeCfg.EvictionHard, kubeCfg.EvictionSoft, kubeCfg.EvictionSoftGracePeriod, kubeCfg.EvictionMinimumReclaim)
 	if err != nil {
 		return nil, err
 	}
@@ -1222,7 +1227,7 @@ func (kl *Kubelet) initializeRuntimeDependentModules() {
 		glog.Fatalf("Failed to start cAdvisor %v", err)
 	}
 	// eviction manager must start after cadvisor because it needs to know if the container runtime has a dedicated imagefs
-	kl.evictionManager.Start(kl, kl.getActivePods, evictionMonitoringPeriod)
+	kl.evictionManager.Start(kl, kl.getActivePods, kl, evictionMonitoringPeriod)
 }
 
 // Run starts the kubelet reacting to config updates

test/e2e_node/BUILD

@@ -57,6 +57,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
+        "allocatable_eviction_test.go",
         "apparmor_test.go",
         "container_manager_test.go",
         "critical_pod_test.go",

test/e2e_node/allocatable_eviction_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/kubelet/cm"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// Eviction Policy is described here:
+// https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/kubelet-eviction.md
+
+var _ = framework.KubeDescribe("AllocatableEviction [Slow] [Serial] [Disruptive] [Flaky]", func() {
+	f := framework.NewDefaultFramework("allocatable-eviction-test")
+
+	podTestSpecs := []podTestSpec{
+		{
+			evictionPriority: 1, // This pod should be evicted before the innocent pod
+			pod:              *getMemhogPod("memory-hog-pod", "memory-hog", v1.ResourceRequirements{}),
+		},
+		{
+			evictionPriority: 0, // This pod should never be evicted
+			pod: v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{Name: "innocent-pod"},
+				Spec: v1.PodSpec{
+					RestartPolicy: v1.RestartPolicyNever,
+					Containers: []v1.Container{
+						{
+							Image: "gcr.io/google_containers/busybox:1.24",
+							Name:  "normal-memory-usage-container",
+							Command: []string{
+								"sh",
+								"-c", //make one big (5 Gb) file
+								"dd if=/dev/urandom of=largefile bs=5000000000 count=1; while true; do sleep 5; done",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	evictionTestTimeout := 40 * time.Minute
+	testCondition := "Memory Pressure"
+	kubeletConfigUpdate := func(initialConfig *componentconfig.KubeletConfiguration) {
+		initialConfig.EvictionHard = "memory.available<10%"
+		// Set large system and kube reserved values to trigger allocatable thresholds far before hard eviction thresholds.
+		initialConfig.SystemReserved = componentconfig.ConfigurationMap(map[string]string{"memory": "1Gi"})
+		initialConfig.KubeReserved = componentconfig.ConfigurationMap(map[string]string{"memory": "1Gi"})
+		initialConfig.EnforceNodeAllocatable = []string{cm.NodeAllocatableEnforcementKey}
+		initialConfig.ExperimentalNodeAllocatableIgnoreEvictionThreshold = false
+		initialConfig.CgroupsPerQOS = true
+	}
+	runEvictionTest(f, testCondition, podTestSpecs, evictionTestTimeout, hasMemoryPressure, kubeletConfigUpdate)
+})
+
+// Returns TRUE if the node has Memory Pressure, FALSE otherwise
+func hasMemoryPressure(f *framework.Framework, testCondition string) (bool, error) {
+	localNodeStatus := getLocalNode(f).Status
+	_, pressure := v1.GetNodeCondition(&localNodeStatus, v1.NodeMemoryPressure)
+	Expect(pressure).NotTo(BeNil())
+	hasPressure := pressure.Status == v1.ConditionTrue
+	By(fmt.Sprintf("checking if pod has %s: %v", testCondition, hasPressure))
+
+	// Additional Logging relating to Memory
+	summary, err := getNodeSummary()
+	if err != nil {
+		return false, err
+	}
+	if summary.Node.Memory != nil && summary.Node.Memory.WorkingSetBytes != nil && summary.Node.Memory.AvailableBytes != nil {
+		framework.Logf("Node.Memory.WorkingSetBytes: %d, summary.Node.Memory.AvailableBytes: %d", *summary.Node.Memory.WorkingSetBytes, *summary.Node.Memory.AvailableBytes)
+	}
+	for _, pod := range summary.Pods {
+		framework.Logf("Pod: %s", pod.PodRef.Name)
+		for _, container := range pod.Containers {
+			if container.Memory != nil && container.Memory.WorkingSetBytes != nil {
+				framework.Logf("--- summary Container: %s WorkingSetBytes: %d", container.Name, *container.Memory.WorkingSetBytes)
+			}
+		}
+	}
+	return hasPressure, nil
+}

test/e2e_node/inode_eviction_test.go

@@ -22,6 +22,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -112,10 +113,11 @@ var _ = framework.KubeDescribe("InodeEviction [Slow] [Serial] [Disruptive] [Flak
 	}
 	evictionTestTimeout := 30 * time.Minute
 	testCondition := "Disk Pressure due to Inodes"
-	// Set the EvictionHard threshold lower to decrease test time
-	evictionHardLimit := "nodefs.inodesFree<50%"
+	kubeletConfigUpdate := func(initialConfig *componentconfig.KubeletConfiguration) {
+		initialConfig.EvictionHard = "nodefs.inodesFree<50%"
+	}
 
-	runEvictionTest(f, testCondition, podTestSpecs, evictionHardLimit, evictionTestTimeout, hasInodePressure)
+	runEvictionTest(f, testCondition, podTestSpecs, evictionTestTimeout, hasInodePressure, kubeletConfigUpdate)
 })
 
 // Struct used by runEvictionTest that specifies the pod, and when that pod should be evicted, relative to other pods
@@ -133,12 +135,12 @@ type podTestSpec struct {
 //		It ensures that lower evictionPriority pods are always evicted before higher evictionPriority pods (2 evicted before 1, etc.)
 //		It ensures that all lower evictionPriority pods are eventually evicted.
 // runEvictionTest then cleans up the testing environment by deleting provided nodes, and ensures that testCondition no longer exists
-func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs []podTestSpec, evictionHard string,
-	evictionTestTimeout time.Duration, hasPressureCondition func(*framework.Framework, string) (bool, error)) {
+func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs []podTestSpec, evictionTestTimeout time.Duration,
+	hasPressureCondition func(*framework.Framework, string) (bool, error), updateFunction func(initialConfig *componentconfig.KubeletConfiguration)) {
 
 	Context(fmt.Sprintf("when we run containers that should cause %s", testCondition), func() {
 
-		tempSetEvictionHard(f, evictionHard)
+		tempSetCurrentKubeletConfig(f, updateFunction)
 		BeforeEach(func() {
 			By("seting up pods to be used by tests")
 			for _, spec := range podTestSpecs {
@@ -148,6 +150,11 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 		})
 
 		It(fmt.Sprintf("should eventually see %s, and then evict all of the correct pods", testCondition), func() {
+			configEnabled, err := isKubeletConfigEnabled(f)
+			framework.ExpectNoError(err)
+			if !configEnabled {
+				framework.Skipf("Dynamic kubelet config must be enabled for this test to run.")
+			}
 			Eventually(func() error {
 				hasPressure, err := hasPressureCondition(f, testCondition)
 				if err != nil {
@@ -299,14 +306,8 @@ func runEvictionTest(f *framework.Framework, testCondition string, podTestSpecs
 
 // Returns TRUE if the node has disk pressure due to inodes exists on the node, FALSE otherwise
 func hasInodePressure(f *framework.Framework, testCondition string) (bool, error) {
-
-	nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
-	framework.ExpectNoError(err, "getting node list")
-	if len(nodeList.Items) != 1 {
-		return false, fmt.Errorf("expected 1 node, but see %d. List: %v", len(nodeList.Items), nodeList.Items)
-	}
-
-	_, pressure := v1.GetNodeCondition(&nodeList.Items[0].Status, v1.NodeDiskPressure)
+	localNodeStatus := getLocalNode(f).Status
+	_, pressure := v1.GetNodeCondition(&localNodeStatus, v1.NodeDiskPressure)
 	Expect(pressure).NotTo(BeNil())
 	hasPressure := pressure.Status == v1.ConditionTrue
 	By(fmt.Sprintf("checking if pod has %s: %v", testCondition, hasPressure))

test/e2e_node/memory_eviction_test.go

@@ -136,7 +136,7 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu
 					By("creating a guaranteed pod, a burstable pod, and a besteffort pod.")
 
 					// A pod is guaranteed only when requests and limits are specified for all the containers and they are equal.
-					guaranteed := createMemhogPod(f, "guaranteed-", "guaranteed", v1.ResourceRequirements{
+					guaranteed := getMemhogPod("guaranteed-pod", "guaranteed", v1.ResourceRequirements{
 						Requests: v1.ResourceList{
 							"cpu":    resource.MustParse("100m"),
 							"memory": resource.MustParse("100Mi"),
@@ -145,16 +145,22 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu
 							"cpu":    resource.MustParse("100m"),
 							"memory": resource.MustParse("100Mi"),
 						}})
+					guaranteed = f.PodClient().CreateSync(guaranteed)
+					glog.Infof("pod created with name: %s", guaranteed.Name)
 
 					// A pod is burstable if limits and requests do not match across all containers.
-					burstable := createMemhogPod(f, "burstable-", "burstable", v1.ResourceRequirements{
+					burstable := getMemhogPod("burstable-pod", "burstable", v1.ResourceRequirements{
 						Requests: v1.ResourceList{
 							"cpu":    resource.MustParse("100m"),
 							"memory": resource.MustParse("100Mi"),
 						}})
+					burstable = f.PodClient().CreateSync(burstable)
+					glog.Infof("pod created with name: %s", burstable.Name)
 
-					// A pod is besteffort if none of its containers have specified any requests or limits.
-					besteffort := createMemhogPod(f, "besteffort-", "besteffort", v1.ResourceRequirements{})
+					// A pod is besteffort if none of its containers have specified any requests or limits	.
+					besteffort := getMemhogPod("besteffort-pod", "besteffort", v1.ResourceRequirements{})
+					besteffort = f.PodClient().CreateSync(besteffort)
+					glog.Infof("pod created with name: %s", besteffort.Name)
 
 					// We poll until timeout or all pods are killed.
 					// Inside the func, we check that all pods are in a valid phase with
@@ -232,7 +238,7 @@ var _ = framework.KubeDescribe("MemoryEviction [Slow] [Serial] [Disruptive]", fu
 
 })
 
-func createMemhogPod(f *framework.Framework, genName string, ctnName string, res v1.ResourceRequirements) *v1.Pod {
+func getMemhogPod(podName string, ctnName string, res v1.ResourceRequirements) *v1.Pod {
 	env := []v1.EnvVar{
 		{
 			Name: "MEMORY_LIMIT",
@@ -256,9 +262,9 @@ func createMemhogPod(f *framework.Framework, genName string, ctnName string, res
 		memLimit = "$(MEMORY_LIMIT)"
 	}
 
-	pod := &v1.Pod{
+	return &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: genName,
+			Name: podName,
 		},
 		Spec: v1.PodSpec{
 			RestartPolicy: v1.RestartPolicyNever,
@@ -277,8 +283,4 @@ func createMemhogPod(f *framework.Framework, genName string, ctnName string, res
 			},
 		},
 	}
-	// The generated pod.Name will be on the pod spec returned by CreateSync
-	pod = f.PodClient().CreateSync(pod)
-	glog.Infof("pod created with name: %s", pod.Name)
-	return pod
 }

test/e2e_node/util.go

@@ -86,13 +86,6 @@ func getCurrentKubeletConfig() (*componentconfig.KubeletConfiguration, error) {
 	return kubeCfg, nil
 }
 
-// Convenience method to set the evictionHard threshold during the current context.
-func tempSetEvictionHard(f *framework.Framework, evictionHard string) {
-	tempSetCurrentKubeletConfig(f, func(initialConfig *componentconfig.KubeletConfiguration) {
-		initialConfig.EvictionHard = evictionHard
-	})
-}
-
 // Must be called within a Context. Allows the function to modify the KubeletConfiguration during the BeforeEach of the context.
 // The change is reverted in the AfterEach of the context.
 // Returns true on success.