Merge pull request #65396 from bsalamat/sched_no_sort

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. Improve scheduler's performance by eliminating sorting of nodes by their score **What this PR does / why we need it**: Profiling scheduler, I noticed that scheduler spends a significant amount of time in sorting the nodes after we score them to find nodes with the highest score. Finding nodes with the highest score does not need sorting the array. This PR replaces the sort with a linear scan. Eliminating the sort results in over 10% improvement in throughput of the scheduler. Before (3 runs for 5000 nodes, scheduling 1000 pods in a cluster running 2000 pods): BenchmarkScheduling/5000Nodes/2000Pods-12 1000 20682552 ns/op BenchmarkScheduling/5000Nodes/2000Pods-12 1000 20464729 ns/op BenchmarkScheduling/5000Nodes/2000Pods-12 1000 21188906 ns/op After: BenchmarkScheduling/5000Nodes/2000Pods-12 1000 18485866 ns/op BenchmarkScheduling/5000Nodes/2000Pods-12 1000 18457749 ns/op BenchmarkScheduling/5000Nodes/2000Pods-12 1000 18418200 ns/op **Release note**: ```release-note Improve scheduler's performance by eliminating sorting of nodes by their score. ```

Merge pull request #65396 from bsalamat/sched_no_sort
f0311d82 · Kubernetes Submit Queue · GitHub · 582b88c8 · ffc8cc2f · f0311d82
Unverified Commit f0311d82 authored Jun 23, 2018 by Kubernetes Submit Queue Committed by GitHub Jun 23, 2018
Hide whitespace changes
Inline Side-by-side

Showing with 20 additions and 10 deletions

extender_test.go pkg/scheduler/core/extender_test.go +1 -1

generic_scheduler.go pkg/scheduler/core/generic_scheduler.go +19 -9

No files found.
--- a/pkg/scheduler/core/extender_test.go
+++ b/pkg/scheduler/core/extender_test.go
@@ -454,7 +454,7 @@ func TestGenericSchedulerWithExtenders(t *testing.T) {
 			// because of the errors from errorPredicateExtender and/or
 			// errorPrioritizerExtender.
 			predicates:   map[string]algorithm.FitPredicate{"true": truePredicate},
-			prioritizers: []algorithm.PriorityConfig{{Map: EqualPriorityMap, Weight: 1}},
+			prioritizers: []algorithm.PriorityConfig{{Function: machine2Prioritizer, Weight: 1}},
 			extenders: []FakeExtender{
 				{
 					predicates:   []fitPredicate{errorPredicateExtender},

--- a/pkg/scheduler/core/generic_scheduler.go
+++ b/pkg/scheduler/core/generic_scheduler.go
@@ -93,7 +93,6 @@ type genericScheduler struct {
 	predicateMetaProducer    algorithm.PredicateMetadataProducer
 	prioritizers             []algorithm.PriorityConfig
 	extenders                []algorithm.SchedulerExtender
-	lastNodeIndexLock        sync.Mutex
 	lastNodeIndex            uint64
 	alwaysCheckAllPredicates bool
 	cachedNodeInfoMap        map[string]*schedulercache.NodeInfo
@@ -176,6 +175,22 @@ func (g *genericScheduler) Predicates() map[string]algorithm.FitPredicate {
 	return g.predicates
 }

+// findMaxScores returns the indexes of nodes in the "priorityList" that has the highest "Score".
+func findMaxScores(priorityList schedulerapi.HostPriorityList) []int {
+	maxScoreIndexes := make([]int, 0, len(priorityList)/2)
+	maxScore := priorityList[0].Score
+	for i, hp := range priorityList {
+		if hp.Score > maxScore {
+			maxScore = hp.Score
+			maxScoreIndexes = maxScoreIndexes[:0]
+			maxScoreIndexes = append(maxScoreIndexes, i)
+		} else if hp.Score == maxScore {
+			maxScoreIndexes = append(maxScoreIndexes, i)
+		}
+	}
+	return maxScoreIndexes
+}
+
 // selectHost takes a prioritized list of nodes and then picks one
 // in a round-robin manner from the nodes that had the highest score.
 func (g *genericScheduler) selectHost(priorityList schedulerapi.HostPriorityList) (string, error) {
@@ -183,16 +198,11 @@ func (g *genericScheduler) selectHost(priorityList schedulerapi.HostPriorityList
 		return "", fmt.Errorf("empty priorityList")
 	}

-	sort.Sort(sort.Reverse(priorityList))
-	maxScore := priorityList[0].Score
-	firstAfterMaxScore := sort.Search(len(priorityList), func(i int) bool { return priorityList[i].Score < maxScore })
-
-	g.lastNodeIndexLock.Lock()
-	ix := int(g.lastNodeIndex % uint64(firstAfterMaxScore))
+	maxScores := findMaxScores(priorityList)
+	ix := int(g.lastNodeIndex % uint64(len(maxScores)))
 	g.lastNodeIndex++
-	g.lastNodeIndexLock.Unlock()

-	return priorityList[ix].Host, nil
+	return priorityList[maxScores[ix]].Host, nil
 }

 // preempt finds nodes with pods that can be preempted to make room for "pod" to