Skip to content

K
k3s
Unverified Commit 7548764f authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #67788 from mohamed-mehany/inter-pod-affinity-optimization

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md. Affinity/Anti-Affinity Optimization of Pod Being Scheduled **What this PR does / why we need it**: Following #66948, it was noticed that the applied optimizations for anti-affinity rules lookup of existing pods could be further applied to checking affinity and anti-affinity terms of the Pod being scheduled. This is done by mapping topology pairs to pods that potentially match the pod being scheduled instead of mapping nodes to matching pods, and accordingly the search space is reduced. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes #67738 **Special notes for your reviewer**: /sig scheduling /sig scalability **Release note**: ```release-note Improve performance of Pod affinity/anti-affinity in the scheduler ```
parents ed958b7d ee393b4e
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 76 deletions
pkg/scheduler/algorithm/predicates/metadata_test.go
View file @ 7548764f
...@@ -52,13 +52,6 @@ func (s sortableServices) Swap(i, j int) { s[i], s[j] = s[j], s[i] } ...@@ -52,13 +52,6 @@ func (s sortableServices) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
var _ = sort.Interface(&sortableServices{}) var _ = sort.Interface(&sortableServices{})
func sortNodePodMap(np map[string][]*v1.Pod) {
for _, pl := range np {
sortablePods := sortablePods(pl)
sort.Sort(sortablePods)
}
}
// predicateMetadataEquivalent returns true if the two metadata are equivalent. // predicateMetadataEquivalent returns true if the two metadata are equivalent.
// Note: this function does not compare podRequest. // Note: this function does not compare podRequest.
func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error { func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error {
...@@ -77,15 +70,11 @@ func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error { ...@@ -77,15 +70,11 @@ func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error {
for !reflect.DeepEqual(meta1.podPorts, meta2.podPorts) { for !reflect.DeepEqual(meta1.podPorts, meta2.podPorts) {
return fmt.Errorf("podPorts are not equal") return fmt.Errorf("podPorts are not equal")
} }
sortNodePodMap(meta1.nodeNameToMatchingAffinityPods) if !reflect.DeepEqual(meta1.topologyPairsPotentialAffinityPods, meta2.topologyPairsPotentialAffinityPods) {
sortNodePodMap(meta2.nodeNameToMatchingAffinityPods) return fmt.Errorf("topologyPairsPotentialAffinityPods are not equal")
if !reflect.DeepEqual(meta1.nodeNameToMatchingAffinityPods, meta2.nodeNameToMatchingAffinityPods) {
return fmt.Errorf("nodeNameToMatchingAffinityPods are not euqal")
} }
sortNodePodMap(meta1.nodeNameToMatchingAntiAffinityPods) if !reflect.DeepEqual(meta1.topologyPairsPotentialAntiAffinityPods, meta2.topologyPairsPotentialAntiAffinityPods) {
sortNodePodMap(meta2.nodeNameToMatchingAntiAffinityPods) return fmt.Errorf("topologyPairsPotentialAntiAffinityPods are not equal")
if !reflect.DeepEqual(meta1.nodeNameToMatchingAntiAffinityPods, meta2.nodeNameToMatchingAntiAffinityPods) {
return fmt.Errorf("nodeNameToMatchingAntiAffinityPods are not euqal")
} }
if !reflect.DeepEqual(meta1.topologyPairsAntiAffinityPodsMap.podToTopologyPairs, if !reflect.DeepEqual(meta1.topologyPairsAntiAffinityPodsMap.podToTopologyPairs,
meta2.topologyPairsAntiAffinityPodsMap.podToTopologyPairs) { meta2.topologyPairsAntiAffinityPodsMap.podToTopologyPairs) {
...@@ -454,42 +443,71 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) { ...@@ -454,42 +443,71 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) {
}, },
}, },
}, },
nodeNameToMatchingAffinityPods: map[string][]*v1.Pod{ topologyPairsPotentialAffinityPods: &topologyPairsMaps{
"nodeA": { topologyPairToPods: map[topologyPair]podSet{
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, {key: "name", value: "nodeA"}: {
Spec: v1.PodSpec{NodeName: "nodeA"}, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1},
Spec: v1.PodSpec{NodeName: "nodeA"},
}: struct{}{},
},
{key: "name", value: "nodeC"}: {
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p2"},
Spec: v1.PodSpec{
NodeName: "nodeC",
},
}: struct{}{},
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p6", Labels: selector1},
Spec: v1.PodSpec{NodeName: "nodeC"},
}: struct{}{},
}, },
}, },
"nodeC": { podToTopologyPairs: map[string]topologyPairSet{
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p2"}, "p1_": {
Spec: v1.PodSpec{ topologyPair{key: "name", value: "nodeA"}: struct{}{},
NodeName: "nodeC",
},
}, },
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p6", Labels: selector1}, "p2_": {
Spec: v1.PodSpec{NodeName: "nodeC"}, topologyPair{key: "name", value: "nodeC"}: struct{}{},
},
"p6_": {
topologyPair{key: "name", value: "nodeC"}: struct{}{},
}, },
}, },
}, },
nodeNameToMatchingAntiAffinityPods: map[string][]*v1.Pod{ topologyPairsPotentialAntiAffinityPods: &topologyPairsMaps{
"nodeN": { topologyPairToPods: map[topologyPair]podSet{
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1}, {key: "name", value: "nodeN"}: {
Spec: v1.PodSpec{NodeName: "nodeN"}, &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1},
}, Spec: v1.PodSpec{NodeName: "nodeN"},
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p2"}, }: struct{}{},
Spec: v1.PodSpec{ &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p2"},
NodeName: "nodeM", Spec: v1.PodSpec{
}, NodeName: "nodeM",
},
}: struct{}{},
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p3"},
Spec: v1.PodSpec{
NodeName: "nodeM",
},
}: struct{}{},
}, },
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p3"}, {key: "name", value: "nodeM"}: {
Spec: v1.PodSpec{ &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p6", Labels: selector1},
NodeName: "nodeM", Spec: v1.PodSpec{NodeName: "nodeM"},
}, }: struct{}{},
}, },
}, },
"nodeM": { podToTopologyPairs: map[string]topologyPairSet{
&v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "p6", Labels: selector1}, "p1_": {
Spec: v1.PodSpec{NodeName: "nodeM"}, topologyPair{key: "name", value: "nodeN"}: struct{}{},
},
"p2_": {
topologyPair{key: "name", value: "nodeN"}: struct{}{},
},
"p3_": {
topologyPair{key: "name", value: "nodeN"}: struct{}{},
},
"p6_": {
topologyPair{key: "name", value: "nodeM"}: struct{}{},
}, },
}, },
}, },
......
pkg/scheduler/algorithm/predicates/predicates.go
View file @ 7548764f
...@@ -1393,30 +1393,21 @@ func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta ...@@ -1393,30 +1393,21 @@ func (c *PodAffinityChecker) satisfiesExistingPodsAntiAffinity(pod *v1.Pod, meta
return nil, nil return nil, nil
} }
// anyPodsMatchingTopologyTerms checks whether any of the nodes given via // nodeMatchesTopologyTerms checks whether "nodeInfo" matches
// "targetPods" matches topology of all the "terms" for the give "pod" and "nodeInfo". // topology of all the "terms" for the given "pod".
func (c *PodAffinityChecker) anyPodsMatchingTopologyTerms(pod *v1.Pod, targetPods map[string][]*v1.Pod, nodeInfo *schedulercache.NodeInfo, terms []v1.PodAffinityTerm) (bool, error) { func (c *PodAffinityChecker) nodeMatchesTopologyTerms(pod *v1.Pod, topologyPairs *topologyPairsMaps, nodeInfo *schedulercache.NodeInfo, terms []v1.PodAffinityTerm) bool {
for nodeName, targetPods := range targetPods { node := nodeInfo.Node()
targetPodNodeInfo, err := c.info.GetNodeInfo(nodeName) for _, term := range terms {
if err != nil { if topologyValue, ok := node.Labels[term.TopologyKey]; ok {
return false, err pair := topologyPair{key: term.TopologyKey, value: topologyValue}
} if _, ok := topologyPairs.topologyPairToPods[pair]; !ok {
if len(targetPods) > 0 { return false
allTermsMatched := true
for _, term := range terms {
if !priorityutil.NodesHaveSameTopologyKey(nodeInfo.Node(), targetPodNodeInfo, term.TopologyKey) {
allTermsMatched = false
break
}
}
if allTermsMatched {
// We have 1 or more pods on the target node that have already matched namespace and selector
// and all of the terms topologies matched the target node. So, there is at least 1 matching pod on the node.
return true, nil
} }
} else {
return false
} }
} }
return false, nil return true
} }
// Checks if scheduling the pod onto this node would break any rules of this pod. // Checks if scheduling the pod onto this node would break any rules of this pod.
...@@ -1429,20 +1420,15 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, ...@@ -1429,20 +1420,15 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod,
} }
if predicateMeta, ok := meta.(*predicateMetadata); ok { if predicateMeta, ok := meta.(*predicateMetadata); ok {
// Check all affinity terms. // Check all affinity terms.
matchingPods := predicateMeta.nodeNameToMatchingAffinityPods topologyPairsPotentialAffinityPods := predicateMeta.topologyPairsPotentialAffinityPods
if affinityTerms := GetPodAffinityTerms(affinity.PodAffinity); len(affinityTerms) > 0 { if affinityTerms := GetPodAffinityTerms(affinity.PodAffinity); len(affinityTerms) > 0 {
matchExists, err := c.anyPodsMatchingTopologyTerms(pod, matchingPods, nodeInfo, affinityTerms) matchExists := c.nodeMatchesTopologyTerms(pod, topologyPairsPotentialAffinityPods, nodeInfo, affinityTerms)
if err != nil {
errMessage := fmt.Sprintf("Cannot schedule pod %+v onto node %v, because of PodAffinity, err: %v", podName(pod), node.Name, err)
glog.Errorf(errMessage)
return ErrPodAffinityRulesNotMatch, errors.New(errMessage)
}
if !matchExists { if !matchExists {
// This pod may the first pod in a series that have affinity to themselves. In order // This pod may the first pod in a series that have affinity to themselves. In order
// to not leave such pods in pending state forever, we check that if no other pod // to not leave such pods in pending state forever, we check that if no other pod
// in the cluster matches the namespace and selector of this pod and the pod matches // in the cluster matches the namespace and selector of this pod and the pod matches
// its own terms, then we allow the pod to pass the affinity check. // its own terms, then we allow the pod to pass the affinity check.
if !(len(matchingPods) == 0 && targetPodMatchesAffinityOfPod(pod, pod)) { if !(len(topologyPairsPotentialAffinityPods.topologyPairToPods) == 0 && targetPodMatchesAffinityOfPod(pod, pod)) {
glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity", glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAffinity",
podName(pod), node.Name) podName(pod), node.Name)
return ErrPodAffinityRulesNotMatch, nil return ErrPodAffinityRulesNotMatch, nil
...@@ -1451,12 +1437,12 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod, ...@@ -1451,12 +1437,12 @@ func (c *PodAffinityChecker) satisfiesPodsAffinityAntiAffinity(pod *v1.Pod,
} }
// Check all anti-affinity terms. // Check all anti-affinity terms.
matchingPods = predicateMeta.nodeNameToMatchingAntiAffinityPods topologyPairsPotentialAntiAffinityPods := predicateMeta.topologyPairsPotentialAntiAffinityPods
if antiAffinityTerms := GetPodAntiAffinityTerms(affinity.PodAntiAffinity); len(antiAffinityTerms) > 0 { if antiAffinityTerms := GetPodAntiAffinityTerms(affinity.PodAntiAffinity); len(antiAffinityTerms) > 0 {
matchExists, err := c.anyPodsMatchingTopologyTerms(pod, matchingPods, nodeInfo, antiAffinityTerms) matchExists := c.nodeMatchesTopologyTerms(pod, topologyPairsPotentialAntiAffinityPods, nodeInfo, antiAffinityTerms)
if err != nil || matchExists { if matchExists {
glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinity, err: %v", glog.V(10).Infof("Cannot schedule pod %+v onto node %v, because of PodAntiAffinity",
podName(pod), node.Name, err) podName(pod), node.Name)
return ErrPodAntiAffinityRulesNotMatch, nil return ErrPodAntiAffinityRulesNotMatch, nil
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment