Merge pull request #58712 from bsalamat/fix_cache_filter

Automatic merge from submit-queue (batch tested with PRs 54242, 58522, 58704, 58708, 58712). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Add better event handling for deleted Pods **What this PR does / why we need it**: We have observed cases where scheduler cache has not been updated after Pod deletions. This PR makes the event handling logic more robust by adding logic to check for `cache.DeletedFinalStateUnknown` in our event handler filters. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note Stability: Make Pod delete event handling of scheduler more robust. ``` /sig scheduling

Merge pull request #58712 from bsalamat/fix_cache_filter
cdf26b05 · Kubernetes Submit Queue · GitHub · 36ea7614 · 79601acb · cdf26b05
Unverified Commit cdf26b05 authored Jan 23, 2018 by Kubernetes Submit Queue Committed by GitHub Jan 23, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 58 additions and 24 deletions

factory.go pkg/scheduler/factory/factory.go +58 -24

No files found.
--- a/pkg/scheduler/factory/factory.go
+++ b/pkg/scheduler/factory/factory.go
@@ -189,6 +189,13 @@ func NewConfigFactory(
 				switch t := obj.(type) {
 				case *v1.Pod:
 					return assignedNonTerminatedPod(t)
+				case cache.DeletedFinalStateUnknown:
+					if pod, ok := t.Obj.(*v1.Pod); ok {
+						return assignedNonTerminatedPod(pod)
+					} else {
+						runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
+						return false
+					}
 				default:
 					runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
 					return false
@@ -208,36 +215,22 @@ func NewConfigFactory(
 				switch t := obj.(type) {
 				case *v1.Pod:
 					return unassignedNonTerminatedPod(t)
+				case cache.DeletedFinalStateUnknown:
+					if pod, ok := t.Obj.(*v1.Pod); ok {
+						return unassignedNonTerminatedPod(pod)
+					} else {
+						runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
+						return false
+					}
 				default:
 					runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
 					return false
 				}
 			},
 			Handler: cache.ResourceEventHandlerFuncs{
-				AddFunc: func(obj interface{}) {
-					if err := c.podQueue.Add(obj.(*v1.Pod)); err != nil {
-						runtime.HandleError(fmt.Errorf("unable to queue %T: %v", obj, err))
-					}
-				},
-				UpdateFunc: func(oldObj, newObj interface{}) {
-					pod := newObj.(*v1.Pod)
-					if c.skipPodUpdate(pod) {
-						return
-					}
-					if err := c.podQueue.Update(pod); err != nil {
-						runtime.HandleError(fmt.Errorf("unable to update %T: %v", newObj, err))
-					}
-				},
-				DeleteFunc: func(obj interface{}) {
-					pod := obj.(*v1.Pod)
-					if err := c.podQueue.Delete(pod); err != nil {
-						runtime.HandleError(fmt.Errorf("unable to dequeue %T: %v", obj, err))
-					}
-					if c.volumeBinder != nil {
-						// Volume binder only wants to keep unassigned pods
-						c.volumeBinder.DeletePodBindings(pod)
-					}
-				},
+				AddFunc:    c.addPodToSchedulingQueue,
+				UpdateFunc: c.updatePodInSchedulingQueue,
+				DeleteFunc: c.deletePodFromSchedulingQueue,
 			},
 		},
 	)
@@ -591,6 +584,47 @@ func (c *configFactory) updatePodInCache(oldObj, newObj interface{}) {
 	c.podQueue.AssignedPodUpdated(newPod)
 }

+func (c *configFactory) addPodToSchedulingQueue(obj interface{}) {
+	if err := c.podQueue.Add(obj.(*v1.Pod)); err != nil {
+		runtime.HandleError(fmt.Errorf("unable to queue %T: %v", obj, err))
+	}
+}
+
+func (c *configFactory) updatePodInSchedulingQueue(oldObj, newObj interface{}) {
+	pod := newObj.(*v1.Pod)
+	if c.skipPodUpdate(pod) {
+		return
+	}
+	if err := c.podQueue.Update(pod); err != nil {
+		runtime.HandleError(fmt.Errorf("unable to update %T: %v", newObj, err))
+	}
+}
+
+func (c *configFactory) deletePodFromSchedulingQueue(obj interface{}) {
+	var pod *v1.Pod
+	switch t := obj.(type) {
+	case *v1.Pod:
+		pod = obj.(*v1.Pod)
+	case cache.DeletedFinalStateUnknown:
+		var ok bool
+		pod, ok = t.Obj.(*v1.Pod)
+		if !ok {
+			runtime.HandleError(fmt.Errorf("unable to convert object %T to *v1.Pod in %T", obj, c))
+			return
+		}
+	default:
+		runtime.HandleError(fmt.Errorf("unable to handle object in %T: %T", c, obj))
+		return
+	}
+	if err := c.podQueue.Delete(pod); err != nil {
+		runtime.HandleError(fmt.Errorf("unable to dequeue %T: %v", obj, err))
+	}
+	if c.volumeBinder != nil {
+		// Volume binder only wants to keep unassigned pods
+		c.volumeBinder.DeletePodBindings(pod)
+	}
+}
+
 func (c *configFactory) invalidateCachedPredicatesOnUpdatePod(newPod *v1.Pod, oldPod *v1.Pod) {
 	if c.enableEquivalenceClassCache {
 		// if the pod does not have bound node, updating equivalence cache is meaningless;