critical pods can preempt other pods to be admitted

pkg/api/v1/resource_helpers.go

@@ -227,3 +227,31 @@ func PodRequestsAndLimits(pod *Pod) (reqs map[ResourceName]resource.Quantity, li
 	}
 	return
 }
+
+// finds and returns the request for a specific resource.
+func GetResourceRequest(pod *Pod, resource ResourceName) int64 {
+	if resource == ResourcePods {
+		return 1
+	}
+	totalResources := int64(0)
+	for _, container := range pod.Spec.Containers {
+		if rQuantity, ok := container.Resources.Requests[resource]; ok {
+			if resource == ResourceCPU {
+				totalResources += rQuantity.MilliValue()
+			} else {
+				totalResources += rQuantity.Value()
+			}
+		}
+	}
+	// take max_resource(sum_pod, any_init_container)
+	for _, container := range pod.Spec.InitContainers {
+		if rQuantity, ok := container.Resources.Requests[resource]; ok {
+			if resource == ResourceCPU && rQuantity.MilliValue() > totalResources {
+				totalResources = rQuantity.MilliValue()
+			} else if rQuantity.Value() > totalResources {
+				totalResources = rQuantity.Value()
+			}
+		}
+	}
+	return totalResources
+}

pkg/kubelet/BUILD

@@ -65,6 +65,7 @@ go_library(
         "//pkg/kubelet/network:go_default_library",
         "//pkg/kubelet/pleg:go_default_library",
         "//pkg/kubelet/pod:go_default_library",
+        "//pkg/kubelet/preemption:go_default_library",
         "//pkg/kubelet/prober:go_default_library",
         "//pkg/kubelet/prober/results:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
@@ -253,6 +254,7 @@ filegroup(
         "//pkg/kubelet/network:all-srcs",
         "//pkg/kubelet/pleg:all-srcs",
         "//pkg/kubelet/pod:all-srcs",
+        "//pkg/kubelet/preemption:all-srcs",
         "//pkg/kubelet/prober:all-srcs",
         "//pkg/kubelet/qos:all-srcs",
         "//pkg/kubelet/remote:all-srcs",

pkg/kubelet/events/event.go

@@ -28,6 +28,7 @@ const (
 	FailedToCreateContainer = "Failed"
 	FailedToStartContainer  = "Failed"
 	KillingContainer        = "Killing"
+	PreemptContainer        = "Preempting"
 	BackOffStartContainer   = "BackOff"
 	ExceededGracePeriod     = "ExceededGracePeriod"
 

pkg/kubelet/kubelet.go

@@ -75,6 +75,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/network"
 	"k8s.io/kubernetes/pkg/kubelet/pleg"
 	kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
+	"k8s.io/kubernetes/pkg/kubelet/preemption"
 	"k8s.io/kubernetes/pkg/kubelet/prober"
 	proberesults "k8s.io/kubernetes/pkg/kubelet/prober/results"
 	"k8s.io/kubernetes/pkg/kubelet/remote"
@@ -783,7 +784,8 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Kub
 	klet.AddPodSyncLoopHandler(activeDeadlineHandler)
 	klet.AddPodSyncHandler(activeDeadlineHandler)
 
-	klet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(klet.getNodeAnyWay))
+	criticalPodAdmissionHandler := preemption.NewCriticalPodAdmissionHandler(klet.getActivePods, killPodNow(klet.podWorkers, kubeDeps.Recorder), kubeDeps.Recorder)
+	klet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(klet.getNodeAnyWay, criticalPodAdmissionHandler))
 	// apply functional Option's
 	for _, opt := range kubeDeps.Options {
 		opt(klet)

pkg/kubelet/kubelet_test.go

@@ -248,7 +248,7 @@ func newTestKubeletWithImageList(
 	kubelet.evictionManager = evictionManager
 	kubelet.admitHandlers.AddPodAdmitHandler(evictionAdmitHandler)
 	// Add this as cleanup predicate pod admitter
-	kubelet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(kubelet.getNodeAnyWay))
+	kubelet.admitHandlers.AddPodAdmitHandler(lifecycle.NewPredicateAdmitHandler(kubelet.getNodeAnyWay, lifecycle.NewAdmissionFailureHandlerStub()))
 
 	plug := &volumetest.FakeVolumePlugin{PluginName: "fake", Host: nil}
 	kubelet.volumePluginMgr, err =

pkg/kubelet/lifecycle/BUILD

@@ -11,6 +11,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "admission_failure_handler_stub.go",
         "doc.go",
         "fake_handler_runner.go",
         "handlers.go",
@@ -24,6 +25,7 @@ go_library(
         "//pkg/kubelet/types:go_default_library",
         "//pkg/kubelet/util/format:go_default_library",
         "//pkg/security/apparmor:go_default_library",
+        "//plugin/pkg/scheduler/algorithm:go_default_library",
         "//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
         "//plugin/pkg/scheduler/schedulercache:go_default_library",
         "//vendor:github.com/golang/glog",

pkg/kubelet/lifecycle/admission_failure_handler_stub.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lifecycle
+
+import (
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
+)
+
+// AdmissionFailureHandlerStub is an AdmissionFailureHandler that does not perform any handling of admission failure.
+// It simply passes the failure on.
+type AdmissionFailureHandlerStub struct{}
+
+var _ AdmissionFailureHandler = &AdmissionFailureHandlerStub{}
+
+func NewAdmissionFailureHandlerStub() *AdmissionFailureHandlerStub {
+	return &AdmissionFailureHandlerStub{}
+}
+
+func (n *AdmissionFailureHandlerStub) HandleAdmissionFailure(pod *v1.Pod, failureReasons []algorithm.PredicateFailureReason) (bool, []algorithm.PredicateFailureReason, error) {
+	return false, failureReasons, nil
+}

pkg/kubelet/lifecycle/predicate.go

@@ -22,20 +22,30 @@ import (
 	"github.com/golang/glog"
 	"k8s.io/kubernetes/pkg/api/v1"
 	"k8s.io/kubernetes/pkg/kubelet/util/format"
+	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
 	"k8s.io/kubernetes/plugin/pkg/scheduler/schedulercache"
 )
 
 type getNodeAnyWayFuncType func() (*v1.Node, error)
+
+// AdmissionFailureHandler is an interface which defines how to deal with a failure to admit a pod.
+// This allows for the graceful handling of pod admission failure.
+type AdmissionFailureHandler interface {
+	HandleAdmissionFailure(pod *v1.Pod, failureReasons []algorithm.PredicateFailureReason) (bool, []algorithm.PredicateFailureReason, error)
+}
+
 type predicateAdmitHandler struct {
-	getNodeAnyWayFunc getNodeAnyWayFuncType
+	getNodeAnyWayFunc       getNodeAnyWayFuncType
+	admissionFailureHandler AdmissionFailureHandler
 }
 
 var _ PodAdmitHandler = &predicateAdmitHandler{}
 
-func NewPredicateAdmitHandler(getNodeAnyWayFunc getNodeAnyWayFuncType) *predicateAdmitHandler {
+func NewPredicateAdmitHandler(getNodeAnyWayFunc getNodeAnyWayFuncType, admissionFailureHandler AdmissionFailureHandler) *predicateAdmitHandler {
 	return &predicateAdmitHandler{
 		getNodeAnyWayFunc,
+		admissionFailureHandler,
 	}
 }
 
@@ -59,11 +69,23 @@ func (w *predicateAdmitHandler) Admit(attrs *PodAdmitAttributes) PodAdmitResult
 		glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
 		return PodAdmitResult{
 			Admit:   fit,
-			Reason:  "UnexpectedError",
+			Reason:  "UnexpectedAdmissionError",
 			Message: message,
 		}
 	}
 	if !fit {
+		fit, reasons, err = w.admissionFailureHandler.HandleAdmissionFailure(pod, reasons)
+		if err != nil {
+			message := fmt.Sprintf("Unexpected error while attempting to recover from admission failure: %v", err)
+			glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), message)
+			return PodAdmitResult{
+				Admit:   fit,
+				Reason:  "UnexpectedAdmissionError",
+				Message: message,
+			}
+		}
+	}
+	if !fit {
 		var reason string
 		var message string
 		if len(reasons) == 0 {

pkg/kubelet/preemption/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["preemption.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/v1:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/kubelet/events:go_default_library",
+        "//pkg/kubelet/eviction:go_default_library",
+        "//pkg/kubelet/lifecycle:go_default_library",
+        "//pkg/kubelet/qos:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
+        "//pkg/kubelet/util/format:go_default_library",
+        "//plugin/pkg/scheduler/algorithm:go_default_library",
+        "//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
+        "//vendor:github.com/golang/glog",
+        "//vendor:k8s.io/apiserver/pkg/util/feature",
+        "//vendor:k8s.io/client-go/tools/record",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["preemption_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/v1:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
+        "//vendor:k8s.io/apimachinery/pkg/api/resource",
+        "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
+        "//vendor:k8s.io/client-go/tools/record",
+    ],
+)

plugin/pkg/scheduler/algorithm/predicates/error.go

@@ -70,6 +70,10 @@ func (e *InsufficientResourceError) GetReason() string {
 	return fmt.Sprintf("Insufficient %v", e.ResourceName)
 }
 
+func (e *InsufficientResourceError) GetInsufficientAmount() int64 {
+	return e.requested - (e.capacity - e.used)
+}
+
 type PredicateFailureError struct {
 	PredicateName string
 }

test/e2e/framework/pods.go

@@ -72,16 +72,21 @@ func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
 	return p
 }
 
-// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
-func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
+// CreateSync creates a new pod according to the framework specifications in the given namespace, and waits for it to start.
+func (c *PodClient) CreateSyncInNamespace(pod *v1.Pod, namespace string) *v1.Pod {
 	p := c.Create(pod)
-	ExpectNoError(c.f.WaitForPodRunning(p.Name))
+	ExpectNoError(WaitForPodNameRunningInNamespace(c.f.ClientSet, p.Name, namespace))
 	// Get the newest pod after it becomes running, some status may change after pod created, such as pod ip.
 	p, err := c.Get(p.Name, metav1.GetOptions{})
 	ExpectNoError(err)
 	return p
 }
 
+// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
+func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
+	return c.CreateSyncInNamespace(pod, c.f.Namespace.Name)
+}
+
 // CreateBatch create a batch of pods. All pods are created before waiting.
 func (c *PodClient) CreateBatch(pods []*v1.Pod) []*v1.Pod {
 	ps := make([]*v1.Pod, len(pods))
@@ -124,11 +129,17 @@ func (c *PodClient) Update(name string, updateFn func(pod *v1.Pod)) {
 // DeleteSync deletes the pod and wait for the pod to disappear for `timeout`. If the pod doesn't
 // disappear before the timeout, it will fail the test.
 func (c *PodClient) DeleteSync(name string, options *metav1.DeleteOptions, timeout time.Duration) {
+	c.DeleteSyncInNamespace(name, c.f.Namespace.Name, options, timeout)
+}
+
+// DeleteSyncInNamespace deletes the pod from the namespace and wait for the pod to disappear for `timeout`. If the pod doesn't
+// disappear before the timeout, it will fail the test.
+func (c *PodClient) DeleteSyncInNamespace(name string, namespace string, options *metav1.DeleteOptions, timeout time.Duration) {
 	err := c.Delete(name, options)
 	if err != nil && !errors.IsNotFound(err) {
 		Failf("Failed to delete pod %q: %v", name, err)
 	}
-	Expect(WaitForPodToDisappear(c.f.ClientSet, c.f.Namespace.Name, name, labels.Everything(),
+	Expect(WaitForPodToDisappear(c.f.ClientSet, namespace, name, labels.Everything(),
 		2*time.Second, timeout)).To(Succeed(), "wait for pod %q to disappear", name)
 }
 

test/e2e_node/BUILD

@@ -52,6 +52,7 @@ go_test(
         "apparmor_test.go",
         "cgroup_manager_test.go",
         "container_manager_test.go",
+        "critical_pod_test.go",
         "density_test.go",
         "disk_eviction_test.go",
         "dynamic_kubelet_configuration_test.go",
@@ -76,7 +77,9 @@ go_test(
         "integration",
     ],
     deps = [
+        "//pkg/api:go_default_library",
         "//pkg/api/v1:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
         "//pkg/kubelet:go_default_library",
         "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
@@ -85,6 +88,7 @@ go_test(
         "//pkg/kubelet/dockertools:go_default_library",
         "//pkg/kubelet/images:go_default_library",
         "//pkg/kubelet/metrics:go_default_library",
+        "//pkg/kubelet/types:go_default_library",
         "//pkg/metrics:go_default_library",
         "//pkg/security/apparmor:go_default_library",
         "//test/e2e/common:go_default_library",

test/e2e_node/critical_pod_test.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeapi "k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	kubelettypes "k8s.io/kubernetes/pkg/kubelet/types"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	criticalPodName   = "critical-pod"
+	guaranteedPodName = "guaranteed"
+	burstablePodName  = "burstable"
+	bestEffortPodName = "best-effort"
+)
+
+var _ = framework.KubeDescribe("CriticalPod [Serial] [Disruptive]", func() {
+	f := framework.NewDefaultFramework("critical-pod-test")
+
+	Context("when we need to admit a critical pod", func() {
+		tempSetCurrentKubeletConfig(f, func(initialConfig *componentconfig.KubeletConfiguration) {
+			initialConfig.FeatureGates += ", ExperimentalCriticalPodAnnotation=true"
+		})
+
+		It("should be able to create and delete a critical pod", func() {
+			configEnabled, err := isKubeletConfigEnabled(f)
+			framework.ExpectNoError(err)
+			if !configEnabled {
+				framework.Skipf("unable to run test without dynamic kubelet config enabled.")
+			}
+
+			// Define test pods
+			nonCriticalGuaranteed := getTestPod(false, guaranteedPodName, v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					"cpu":    resource.MustParse("100m"),
+					"memory": resource.MustParse("100Mi"),
+				},
+				Limits: v1.ResourceList{
+					"cpu":    resource.MustParse("100m"),
+					"memory": resource.MustParse("100Mi"),
+				},
+			})
+			nonCriticalBurstable := getTestPod(false, burstablePodName, v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					"cpu":    resource.MustParse("100m"),
+					"memory": resource.MustParse("100Mi"),
+				},
+			})
+			nonCriticalBestEffort := getTestPod(false, bestEffortPodName, v1.ResourceRequirements{})
+			criticalPod := getTestPod(true, criticalPodName, v1.ResourceRequirements{
+				// request the entire resource capacity of the node, so that
+				// admitting this pod requires the other pod to be preempted
+				Requests: getNodeCPUAndMemoryCapacity(f),
+			})
+
+			// Create pods, starting with non-critical so that the critical preempts the other pods.
+			f.PodClient().CreateBatch([]*v1.Pod{nonCriticalBestEffort, nonCriticalBurstable, nonCriticalGuaranteed})
+			f.PodClientNS(kubeapi.NamespaceSystem).CreateSyncInNamespace(criticalPod, kubeapi.NamespaceSystem)
+
+			// Check that non-critical pods other than the besteffort have been evicted
+			updatedPodList, err := f.ClientSet.Core().Pods(f.Namespace.Name).List(metav1.ListOptions{})
+			framework.ExpectNoError(err)
+			for _, p := range updatedPodList.Items {
+				if p.Name == nonCriticalBestEffort.Name {
+					Expect(p.Status.Phase).NotTo(Equal(v1.PodFailed), fmt.Sprintf("pod: %v should be preempted", p.Name))
+				} else {
+					Expect(p.Status.Phase).To(Equal(v1.PodFailed), fmt.Sprintf("pod: %v should not be preempted", p.Name))
+				}
+			}
+		})
+		AfterEach(func() {
+			// Delete Pods
+			f.PodClient().DeleteSync(guaranteedPodName, &metav1.DeleteOptions{}, podDisappearTimeout)
+			f.PodClient().DeleteSync(burstablePodName, &metav1.DeleteOptions{}, podDisappearTimeout)
+			f.PodClient().DeleteSync(bestEffortPodName, &metav1.DeleteOptions{}, podDisappearTimeout)
+			f.PodClientNS(kubeapi.NamespaceSystem).DeleteSyncInNamespace(criticalPodName, kubeapi.NamespaceSystem, &metav1.DeleteOptions{}, podDisappearTimeout)
+			// Log Events
+			logPodEvents(f)
+			logNodeEvents(f)
+
+		})
+	})
+})
+
+func getNodeCPUAndMemoryCapacity(f *framework.Framework) v1.ResourceList {
+	nodeList, err := f.ClientSet.Core().Nodes().List(metav1.ListOptions{})
+	framework.ExpectNoError(err)
+	// Assuming that there is only one node, because this is a node e2e test.
+	Expect(len(nodeList.Items)).To(Equal(1))
+	capacity := nodeList.Items[0].Status.Capacity
+	return v1.ResourceList{
+		v1.ResourceCPU:    capacity[v1.ResourceCPU],
+		v1.ResourceMemory: capacity[v1.ResourceMemory],
+	}
+}
+
+func getTestPod(critical bool, name string, resources v1.ResourceRequirements) *v1.Pod {
+	pod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{Name: name},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:      "container",
+					Image:     framework.GetPauseImageNameForHostArch(),
+					Resources: resources,
+				},
+			},
+		},
+	}
+	if critical {
+		pod.ObjectMeta.Namespace = kubeapi.NamespaceSystem
+		pod.ObjectMeta.Annotations = map[string]string{
+			kubelettypes.CriticalPodAnnotationKey: "",
+		}
+		Expect(kubelettypes.IsCriticalPod(pod)).To(BeTrue(), "pod should be a critical pod")
+	} else {
+		Expect(kubelettypes.IsCriticalPod(pod)).To(BeFalse(), "pod should not be a critical pod")
+	}
+	return pod
+}