Skip to content

K
k3s
Commit 0e851787 authored by Harry Zhang's avatar Harry Zhang
Browse files
Options

Update factory.go informers to update equivalence cache

Fix tombstone Add e2e to verify equivalence cache Addressing nits in factory,go and e2e Update build files
parent 4103f40f
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 80 additions and 25 deletions
pkg/features/kube_features.go
View file @ 0e851787
......@@ -120,6 +120,12 @@ const (
//
// Add priority to pods. Priority affects scheduling and preemption of pods.
PodPriority utilfeature.Feature = "PodPriority"
// owner: @resouer
// alpha: v1.8
//
// Enable equivalence class cache for scheduler.
EnableEquivalenceClassCache utilfeature.Feature = "EnableEquivalenceClassCache"
)
func init() {
......@@ -144,6 +150,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
LocalStorageCapacityIsolation: {Default: false, PreRelease: utilfeature.Alpha},
DebugContainers: {Default: false, PreRelease: utilfeature.Alpha},
PodPriority: {Default: false, PreRelease: utilfeature.Alpha},
EnableEquivalenceClassCache: {Default: false, PreRelease: utilfeature.Alpha},
// inherited features from generic apiserver, relisted here to get a conflict if it is changed
// unintentionally on either side:
......
plugin/cmd/kube-scheduler/app/configurator.go
View file @ 0e851787
......@@ -28,6 +28,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
......@@ -39,6 +40,7 @@ import (
clientv1 "k8s.io/api/core/v1"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/plugin/pkg/scheduler"
_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"
schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"
......@@ -104,6 +106,7 @@ func CreateScheduler(
statefulSetInformer,
serviceInformer,
s.HardPodAffinitySymmetricWeight,
utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
)
// Rebuild the configurator with a default Create(...) method.
......
plugin/pkg/scheduler/algorithmprovider/defaults/compatibility_test.go
View file @ 0e851787
......@@ -37,6 +37,8 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/factory"
)
const enableEquivalenceCache = true
func TestCompatibility_v1_Scheduler(t *testing.T) {
// Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases
schedulerFiles := map[string]struct {
......@@ -432,6 +434,7 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
).CreateFromConfig(policy); err != nil {
t.Errorf("%s: Error constructing: %v", v, err)
continue
......
plugin/pkg/scheduler/core/equivalence_cache.go
View file @ 0e851787
......@@ -158,7 +158,8 @@ func (ec *EquivalenceCache) InvalidateAllCachedPredicateItemOfNode(nodeName stri
glog.V(5).Infof("Done invalidating all cached predicates on node: %s", nodeName)
}
// InvalidateCachedPredicateItemForPod marks item of given predicateKeys, of given pod, on the given node as invalid
// InvalidateCachedPredicateItemForPod marks item of given predicateKeys, of given pod (i.e. equivalenceHash),
// on the given node as invalid
func (ec *EquivalenceCache) InvalidateCachedPredicateItemForPod(nodeName string, predicateKeys sets.String, pod *v1.Pod) {
if len(predicateKeys) == 0 {
return
......
plugin/pkg/scheduler/factory/factory_test.go
View file @ 0e851787
......@@ -40,6 +40,8 @@ import (
"k8s.io/kubernetes/plugin/pkg/scheduler/util"
)
const enableEquivalenceCache = true
func TestCreate(t *testing.T) {
handler := utiltesting.FakeHandler{
StatusCode: 500,
......@@ -62,6 +64,7 @@ func TestCreate(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
factory.Create()
}
......@@ -93,6 +96,7 @@ func TestCreateFromConfig(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
// Pre-register some predicate and priority functions
......@@ -151,6 +155,7 @@ func TestCreateFromConfigWithHardPodAffinitySymmetricWeight(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
// Pre-register some predicate and priority functions
......@@ -210,6 +215,7 @@ func TestCreateFromEmptyConfig(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
configData = []byte(`{}`)
......@@ -266,6 +272,7 @@ func TestDefaultErrorFunc(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
queue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
podBackoff := util.CreatePodBackoff(1*time.Millisecond, 1*time.Second)
......@@ -378,6 +385,7 @@ func TestResponsibleForPod(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
// factory of "foo-scheduler"
factoryFooScheduler := NewConfigFactory(
......@@ -392,6 +400,7 @@ func TestResponsibleForPod(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
// scheduler annotations to be tested
schedulerFitsDefault := "default-scheduler"
......@@ -461,6 +470,7 @@ func TestInvalidHardPodAffinitySymmetricWeight(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
-1,
enableEquivalenceCache,
)
_, err := factory.Create()
if err == nil {
......@@ -506,6 +516,7 @@ func TestInvalidFactoryArgs(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
test.hardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
_, err := factory.Create()
if err == nil {
......
test/e2e/autoscaling/cluster_size_autoscaling.go
View file @ 0e851787
......@@ -411,7 +411,7 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Slow]", func() {
framework.AddOrUpdateLabelOnNode(c, node, labelKey, labelValue)
}
CreateNodeSelectorPods(f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false)
scheduling.CreateNodeSelectorPods(f, "node-selector", minSize+1, map[string]string{labelKey: labelValue}, false)
By("Waiting for new node to appear and annotating it")
framework.WaitForGroupSize(minMig, int32(minSize+1))
......@@ -907,26 +907,6 @@ func doPut(url, content string) (string, error) {
return strBody, nil
}
func CreateNodeSelectorPods(f *framework.Framework, id string, replicas int, nodeSelector map[string]string, expectRunning bool) {
By(fmt.Sprintf("Running RC which reserves host port and defines node selector"))
config := &testutils.RCConfig{
Client: f.ClientSet,
InternalClient: f.InternalClientset,
Name: id,
Namespace: f.Namespace.Name,
Timeout: defaultTimeout,
Image: framework.GetPauseImageName(f.ClientSet),
Replicas: replicas,
HostPorts: map[string]int{"port1": 4321},
NodeSelector: nodeSelector,
}
err := framework.RunRC(*config)
if expectRunning {
framework.ExpectNoError(err)
}
}
func ReserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool, timeout time.Duration) {
By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes))
request := int64(1024 * 1024 * megabytes / replicas)
......
test/e2e/framework/util.go
View file @ 0e851787
......@@ -2462,6 +2462,15 @@ func AddOrUpdateLabelOnNode(c clientset.Interface, nodeName string, labelKey, la
ExpectNoError(testutil.AddLabelsToNode(c, nodeName, map[string]string{labelKey: labelValue}))
}
func AddOrUpdateLabelOnNodeAndReturnOldValue(c clientset.Interface, nodeName string, labelKey, labelValue string) string {
var oldValue string
node, err := c.Core().Nodes().Get(nodeName, metav1.GetOptions{})
ExpectNoError(err)
oldValue = node.Labels[labelKey]
ExpectNoError(testutil.AddLabelsToNode(c, nodeName, map[string]string{labelKey: labelValue}))
return oldValue
}
func ExpectNodeHasLabel(c clientset.Interface, nodeName string, labelKey string, labelValue string) {
By("verifying the node has the label " + labelKey + " " + labelValue)
node, err := c.Core().Nodes().Get(nodeName, metav1.GetOptions{})
......
test/e2e/scheduling/predicates.go
View file @ 0e851787
......@@ -24,6 +24,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/uuid"
clientset "k8s.io/client-go/kubernetes"
......@@ -51,6 +52,8 @@ type pausePodConfig struct {
Resources *v1.ResourceRequirements
Tolerations []v1.Toleration
NodeName string
Ports []v1.ContainerPort
OwnerReferences []metav1.OwnerReference
}
var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
......@@ -749,9 +752,10 @@ var _ = SIGDescribe("SchedulerPredicates [Serial]", func() {
func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: conf.Name,
Labels: conf.Labels,
Annotations: conf.Annotations,
Name: conf.Name,
Labels: conf.Labels,
Annotations: conf.Annotations,
OwnerReferences: conf.OwnerReferences,
},
Spec: v1.PodSpec{
NodeSelector: conf.NodeSelector,
......@@ -760,6 +764,7 @@ func initPausePod(f *framework.Framework, conf pausePodConfig) *v1.Pod {
{
Name: conf.Name,
Image: framework.GetPauseImageName(f.ClientSet),
Ports: conf.Ports,
},
},
Tolerations: conf.Tolerations,
......@@ -948,6 +953,32 @@ func verifyResult(c clientset.Interface, expectedScheduled int, expectedNotSched
Expect(len(scheduledPods)).To(Equal(expectedScheduled), printOnce(fmt.Sprintf("Scheduled Pods: %#v", scheduledPods)))
}
// verifyReplicasResult is wrapper of verifyResult for a group pods with same "name: labelName" label, which means they belong to same RC
func verifyReplicasResult(c clientset.Interface, expectedScheduled int, expectedNotScheduled int, ns string, labelName string) {
allPods := getPodsByLabels(c, ns, map[string]string{"name": labelName})
scheduledPods, notScheduledPods := framework.GetPodsScheduled(masterNodes, allPods)
printed := false
printOnce := func(msg string) string {
if !printed {
printed = true
return msg
} else {
return ""
}
}
Expect(len(notScheduledPods)).To(Equal(expectedNotScheduled), printOnce(fmt.Sprintf("Not scheduled Pods: %#v", notScheduledPods)))
Expect(len(scheduledPods)).To(Equal(expectedScheduled), printOnce(fmt.Sprintf("Scheduled Pods: %#v", scheduledPods)))
}
func getPodsByLabels(c clientset.Interface, ns string, labelsMap map[string]string) *v1.PodList {
selector := labels.SelectorFromSet(labels.Set(labelsMap))
allPods, err := c.Core().Pods(ns).List(metav1.ListOptions{LabelSelector: selector.String()})
framework.ExpectNoError(err)
return allPods
}
func runAndKeepPodWithLabelAndGetNodeName(f *framework.Framework) (string, string) {
// launch a pod to find a node which can launch a pod. We intentionally do
// not just take the node list and choose the first of them. Depending on the
......
test/integration/scheduler/extender_test.go
View file @ 0e851787
......@@ -370,6 +370,7 @@ func TestSchedulerExtender(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
schedulerConfig, err := schedulerConfigFactory.CreateFromConfig(policy)
if err != nil {
......
test/integration/scheduler/scheduler_test.go
View file @ 0e851787
......@@ -50,6 +50,8 @@ import (
"k8s.io/kubernetes/test/integration/framework"
)
const enableEquivalenceCache = true
type nodeMutationFunc func(t *testing.T, n *v1.Node, nodeLister corelisters.NodeLister, c clientset.Interface)
type nodeStateManager struct {
......@@ -257,6 +259,7 @@ func TestUnschedulableNodes(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
......@@ -540,6 +543,7 @@ func TestMultiScheduler(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
......@@ -626,6 +630,7 @@ func TestMultiScheduler(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
schedulerConfig2, err := schedulerConfigFactory2.Create()
if err != nil {
......@@ -736,6 +741,7 @@ func TestAllocatable(t *testing.T) {
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
......
test/integration/scheduler_perf/util.go
View file @ 0e851787
......@@ -36,6 +36,8 @@ import (
"k8s.io/kubernetes/test/integration/framework"
)
const enableEquivalenceCache = true
// mustSetupScheduler starts the following components:
// - k8s api server (a.k.a. master)
// - scheduler
......@@ -74,6 +76,7 @@ func mustSetupScheduler() (schedulerConfigurator scheduler.Configurator, destroy
informerFactory.Apps().V1beta1().StatefulSets(),
informerFactory.Core().V1().Services(),
v1.DefaultHardPodAffinitySymmetricWeight,
enableEquivalenceCache,
)
eventBroadcaster := record.NewBroadcaster()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment