Skip to content

K
k3s
Commit c9d142d7 authored by Kubernetes Submit Queue's avatar Kubernetes Submit Queue Committed by GitHub
Browse files
Options

Merge pull request #49382 from bskiba/kubemark_e2e_nm

Automatic merge from submit-queue Add a simple cloud provider for e2e tests on kubemark **What this PR does / why we need it**: Adds a simplified cloud provider for kubemark. This enables us to add and remove nodes and operate on nodegroups while running tests on kubemark. This is needed to run scalability tests for cluster autoscaler on kubemark. See https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/kubemark_integration.md **Release note**: ``` NONE ```
parents f6cb2fce 2f747f3d
Expand all Show whitespace changes
Inline Side-by-side
Showing with 47 additions and 0 deletions
hack/verify-flags/known-flags.txt
View file @ c9d142d7
...@@ -426,6 +426,7 @@ kube-reserved ...@@ -426,6 +426,7 @@ kube-reserved
kube-reserved-cgroup kube-reserved-cgroup
kube-master-url kube-master-url
kube-reserved kube-reserved
kubemark-external-kubeconfig
kubernetes-anywhere-cluster kubernetes-anywhere-cluster
kubernetes-anywhere-path kubernetes-anywhere-path
kubernetes-anywhere-phase2-provider kubernetes-anywhere-phase2-provider
......
pkg/kubemark/BUILD
View file @ c9d142d7
...@@ -10,6 +10,7 @@ load( ...@@ -10,6 +10,7 @@ load(
go_library( go_library(
name = "go_default_library", name = "go_default_library",
srcs = [ srcs = [
"controller.go",
"hollow_kubelet.go", "hollow_kubelet.go",
"hollow_proxy.go", "hollow_proxy.go",
], ],
...@@ -22,6 +23,7 @@ go_library( ...@@ -22,6 +23,7 @@ go_library(
"//pkg/apis/componentconfig:go_default_library", "//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library", "//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/kubelet:go_default_library", "//pkg/kubelet:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library", "//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/cm:go_default_library", "//pkg/kubelet/cm:go_default_library",
...@@ -44,9 +46,15 @@ go_library( ...@@ -44,9 +46,15 @@ go_library(
"//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//vendor/k8s.io/client-go/listers/core/v1:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/tools/record:go_default_library", "//vendor/k8s.io/client-go/tools/record:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library",
], ],
......
test/e2e/framework/BUILD
View file @ c9d142d7
...@@ -68,6 +68,7 @@ go_library( ...@@ -68,6 +68,7 @@ go_library(
"//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/sysctl:go_default_library", "//pkg/kubelet/sysctl:go_default_library",
"//pkg/kubelet/util/format:go_default_library", "//pkg/kubelet/util/format:go_default_library",
"//pkg/kubemark:go_default_library",
"//pkg/master/ports:go_default_library", "//pkg/master/ports:go_default_library",
"//pkg/ssh:go_default_library", "//pkg/ssh:go_default_library",
"//pkg/util/file:go_default_library", "//pkg/util/file:go_default_library",
...@@ -129,6 +130,7 @@ go_library( ...@@ -129,6 +130,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/client-go/discovery:go_default_library", "//vendor/k8s.io/client-go/discovery:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library", "//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library", "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
......
test/e2e/framework/framework.go
View file @ c9d142d7
...@@ -37,12 +37,15 @@ import ( ...@@ -37,12 +37,15 @@ import (
"k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes"
staging "k8s.io/client-go/kubernetes" staging "k8s.io/client-go/kubernetes"
clientreporestclient "k8s.io/client-go/rest" clientreporestclient "k8s.io/client-go/rest"
restclient "k8s.io/client-go/rest" restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/kubemark"
"k8s.io/kubernetes/test/e2e/framework/metrics" "k8s.io/kubernetes/test/e2e/framework/metrics"
testutils "k8s.io/kubernetes/test/utils" testutils "k8s.io/kubernetes/test/utils"
...@@ -95,6 +98,8 @@ type Framework struct { ...@@ -95,6 +98,8 @@ type Framework struct {
// Place where various additional data is stored during test run to be printed to ReportDir, // Place where various additional data is stored during test run to be printed to ReportDir,
// or stdout if ReportDir is not set once test ends. // or stdout if ReportDir is not set once test ends.
TestSummaries []TestDataSummary TestSummaries []TestDataSummary
kubemarkControllerCloseChannel chan struct{}
} }
type TestDataSummary interface { type TestDataSummary interface {
...@@ -190,6 +195,23 @@ func (f *Framework) BeforeEach() { ...@@ -190,6 +195,23 @@ func (f *Framework) BeforeEach() {
f.StagingClient, err = staging.NewForConfig(clientRepoConfig) f.StagingClient, err = staging.NewForConfig(clientRepoConfig)
Expect(err).NotTo(HaveOccurred()) Expect(err).NotTo(HaveOccurred())
f.ClientPool = dynamic.NewClientPool(config, api.Registry.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) f.ClientPool = dynamic.NewClientPool(config, api.Registry.RESTMapper(), dynamic.LegacyAPIPathResolverFunc)
if ProviderIs("kubemark") && TestContext.KubemarkExternalKubeConfig != "" && TestContext.CloudConfig.KubemarkController == nil {
externalConfig, err := clientcmd.BuildConfigFromFlags("", TestContext.KubemarkExternalKubeConfig)
externalConfig.QPS = f.Options.ClientQPS
externalConfig.Burst = f.Options.ClientBurst
Expect(err).NotTo(HaveOccurred())
externalClient, err := clientset.NewForConfig(externalConfig)
Expect(err).NotTo(HaveOccurred())
f.kubemarkControllerCloseChannel = make(chan struct{})
externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0)
kubemarkInformerFactory := informers.NewSharedInformerFactory(f.ClientSet, 0)
kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes()
go kubemarkNodeInformer.Informer().Run(f.kubemarkControllerCloseChannel)
TestContext.CloudConfig.KubemarkController, err = kubemark.NewKubemarkController(externalClient, externalInformerFactory, f.ClientSet, kubemarkNodeInformer)
Expect(err).NotTo(HaveOccurred())
externalInformerFactory.Start(f.kubemarkControllerCloseChannel)
TestContext.CloudConfig.KubemarkController.Init(f.kubemarkControllerCloseChannel)
}
} }
if !f.SkipNamespaceCreation { if !f.SkipNamespaceCreation {
...@@ -342,6 +364,10 @@ func (f *Framework) AfterEach() { ...@@ -342,6 +364,10 @@ func (f *Framework) AfterEach() {
} }
} }
if TestContext.CloudConfig.KubemarkController != nil {
close(f.kubemarkControllerCloseChannel)
}
PrintSummaries(f.TestSummaries, f.BaseName) PrintSummaries(f.TestSummaries, f.BaseName)
// Check whether all nodes are ready after the test. // Check whether all nodes are ready after the test.
......
test/e2e/framework/size.go
View file @ c9d142d7
...@@ -51,6 +51,8 @@ func ResizeGroup(group string, size int32) error { ...@@ -51,6 +51,8 @@ func ResizeGroup(group string, size int32) error {
} else if TestContext.Provider == "aws" { } else if TestContext.Provider == "aws" {
client := autoscaling.New(session.New()) client := autoscaling.New(session.New())
return awscloud.ResizeInstanceGroup(client, group, int(size)) return awscloud.ResizeInstanceGroup(client, group, int(size))
} else if TestContext.Provider == "kubemark" {
return TestContext.CloudConfig.KubemarkController.SetNodeGroupSize(group, int(size))
} else { } else {
return fmt.Errorf("Provider does not support InstanceGroups") return fmt.Errorf("Provider does not support InstanceGroups")
} }
...@@ -72,6 +74,8 @@ func GetGroupNodes(group string) ([]string, error) { ...@@ -72,6 +74,8 @@ func GetGroupNodes(group string) ([]string, error) {
lines[i] = line[:strings.Index(line, " ")] lines[i] = line[:strings.Index(line, " ")]
} }
return lines, nil return lines, nil
} else if TestContext.Provider == "kubemark" {
return TestContext.CloudConfig.KubemarkController.GetNodeNamesForNodegroup(group)
} else { } else {
return nil, fmt.Errorf("provider does not support InstanceGroups") return nil, fmt.Errorf("provider does not support InstanceGroups")
} }
...@@ -99,6 +103,8 @@ func GroupSize(group string) (int, error) { ...@@ -99,6 +103,8 @@ func GroupSize(group string) (int, error) {
return -1, fmt.Errorf("instance group not found: %s", group) return -1, fmt.Errorf("instance group not found: %s", group)
} }
return instanceGroup.CurrentSize() return instanceGroup.CurrentSize()
} else if TestContext.Provider == "kubemark" {
return TestContext.CloudConfig.KubemarkController.GetNodeGroupSize(group)
} else { } else {
return -1, fmt.Errorf("provider does not support InstanceGroups") return -1, fmt.Errorf("provider does not support InstanceGroups")
} }
......
test/e2e/framework/test_context.go
View file @ c9d142d7
...@@ -27,12 +27,14 @@ import ( ...@@ -27,12 +27,14 @@ import (
"k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd"
"k8s.io/kubernetes/pkg/apis/componentconfig" "k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/kubemark"
) )
const defaultHost = "http://127.0.0.1:8080" const defaultHost = "http://127.0.0.1:8080"
type TestContextType struct { type TestContextType struct {
KubeConfig string KubeConfig string
KubemarkExternalKubeConfig string
KubeContext string KubeContext string
KubeAPIContentType string KubeAPIContentType string
KubeVolumeDir string KubeVolumeDir string
...@@ -159,6 +161,7 @@ type CloudConfig struct { ...@@ -159,6 +161,7 @@ type CloudConfig struct {
MasterTag string MasterTag string
Provider cloudprovider.Interface Provider cloudprovider.Interface
KubemarkController *kubemark.KubemarkController
} }
var TestContext TestContextType var TestContext TestContextType
...@@ -201,6 +204,7 @@ func RegisterCommonFlags() { ...@@ -201,6 +204,7 @@ func RegisterCommonFlags() {
func RegisterClusterFlags() { func RegisterClusterFlags() {
flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.") flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.") flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
flag.StringVar(&TestContext.KubemarkExternalKubeConfig, fmt.Sprintf("%s-%s", "kubemark-external", clientcmd.RecommendedConfigPathFlag), "", "Path to kubeconfig containing embedded authinfo for external cluster.")
flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'") flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver") flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.") flag.StringVar(&TestContext.FederatedKubeContext, "federated-kube-context", "e2e-federation", "kubeconfig context for federation.")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment