Skip to content

K
k3s
Commit a0cc59f2 authored by k8s-merge-robot's avatar k8s-merge-robot
Browse files
Options

Merge pull request #25658 from jszczepkowski/kac-test

Automatic merge from submit-queue Updated e2e tests for cluster autoscaling. [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/.github/PULL_REQUEST_TEMPLATE.md?pixel)]() Updated e2e tests for cluster autoscaling to work with the new cluster autoscaler.
parents 8d90427c 96818874
Hide whitespace changes
Inline Side-by-side
Showing with 31 additions and 99 deletions
cluster/saltbase/salt/cluster-autoscaler/cluster-autoscaler.manifest
View file @ a0cc59f2
......@@ -15,7 +15,7 @@
"containers": [
{
"name": "cluster-autoscaler",
"image": "gcr.io/mwielgus-proj/cluster-autoscaler:v0.0.1-alpha2-4",
"image": "gcr.io/dev-jsz/cluster-autoscaler:0.1",
"command": [
"./cluster-autoscaler",
"--kubernetes=http://127.0.0.1:8080?inClusterConfig=f",
......
test/e2e/cluster_size_autoscaling.go
View file @ a0cc59f2
......@@ -18,7 +18,6 @@ package e2e
import (
"fmt"
"os/exec"
"time"
"k8s.io/kubernetes/pkg/api"
......@@ -29,15 +28,9 @@ import (
)
const (
scaleUpTimeout = 20 * time.Minute
scaleDownTimeout = 30 * time.Minute
scaleTimeout = 2 * time.Minute
)
// [Feature:ClusterSizeAutoscaling]: Cluster size autoscaling is experimental
// and require Google Cloud Monitoring to be enabled, so these tests are not
// run by default.
//
// These tests take ~20 minutes to run each.
var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAutoscaling] [Slow]", func() {
f := framework.NewDefaultFramework("autoscaling")
var nodeCount int
......@@ -56,117 +49,56 @@ var _ = framework.KubeDescribe("Cluster size autoscaling [Feature:ClusterSizeAut
memCapacityMb = int((&mem).Value() / 1024 / 1024)
})
AfterEach(func() {
cleanUpAutoscaler()
})
It("Should scale cluster size based on cpu utilization", func() {
setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1)
// Consume 50% CPU
rcs := createConsumingRCs(f, "cpu-utilization", nodeCount*coresPerNode, 500, 0)
err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)
for _, rc := range rcs {
rc.CleanUp()
}
framework.ExpectNoError(err)
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
})
It("Should scale cluster size based on cpu reservation", func() {
setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1)
It("Should correctly handle pending pods", func() {
By("Too large pending pod does not increase cluster size")
ReserveMemory(f, "memory-reservation", 1, memCapacityMb, false)
// Verify, that cluster size is not changed.
// TODO: find a better way of verification that the cluster size will remain unchanged.
time.Sleep(scaleTimeout)
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout))
ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode)
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation"))
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
})
It("Should scale cluster size based on memory utilization", func() {
setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1)
// Consume 60% of total memory capacity
megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode)
rcs := createConsumingRCs(f, "mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica)
err := framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout)
for _, rc := range rcs {
rc.CleanUp()
}
framework.ExpectNoError(err)
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
})
It("Should scale cluster size based on memory reservation", func() {
setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1)
ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10)
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleTimeout))
By("Small pending pods increase cluster size")
ReserveMemory(f, "memory-reservation", 100, nodeCount*memCapacityMb, false)
// Verify, that cluster size is increased
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount+1, scaleTimeout))
framework.ExpectNoError(framework.DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
// TODO(jsz): Enable code bellow when scale down is implemented.
// framework.ExpectNoError(framework.WaitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
})
})
func setUpAutoscaler(metric string, target float64, min, max int) {
// TODO integrate with kube-up.sh script once it will support autoscaler setup.
By("Setting up autoscaler to scale based on " + metric)
out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling",
framework.TestContext.CloudConfig.NodeInstanceGroup,
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone,
"--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE",
fmt.Sprintf("--min-num-replicas=%v", min),
fmt.Sprintf("--max-num-replicas=%v", max),
).CombinedOutput()
framework.ExpectNoError(err, "Output: "+string(out))
}
func createConsumingRCs(f *framework.Framework, name string, count, cpuPerReplica, memPerReplica int) []*ResourceConsumer {
var res []*ResourceConsumer
for i := 1; i <= count; i++ {
name := fmt.Sprintf("%s-%d", name, i)
res = append(res, NewStaticResourceConsumer(name, 1, cpuPerReplica, memPerReplica, 0, int64(cpuPerReplica), int64(memPerReplica+100), f))
}
return res
}
func cleanUpAutoscaler() {
By("Removing autoscaler")
out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling",
framework.TestContext.CloudConfig.NodeInstanceGroup,
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone,
).CombinedOutput()
framework.ExpectNoError(err, "Output: "+string(out))
}
func ReserveCpu(f *framework.Framework, id string, millicores int) {
func ReserveCpu(f *framework.Framework, id string, replicas, millicores int) {
By(fmt.Sprintf("Running RC which reserves %v millicores", millicores))
request := int64(millicores / replicas)
config := &framework.RCConfig{
Client: f.Client,
Name: id,
Namespace: f.Namespace.Name,
Timeout: 10 * time.Minute,
Timeout: scaleTimeout,
Image: "gcr.io/google_containers/pause-amd64:3.0",
Replicas: millicores / 100,
CpuRequest: 100,
Replicas: replicas,
CpuRequest: request,
}
framework.ExpectNoError(framework.RunRC(*config))
}
func ReserveMemory(f *framework.Framework, id string, megabytes int) {
func ReserveMemory(f *framework.Framework, id string, replicas, megabytes int, expectRunning bool) {
By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes))
request := int64(1024 * 1024 * megabytes / replicas)
config := &framework.RCConfig{
Client: f.Client,
Name: id,
Namespace: f.Namespace.Name,
Timeout: 10 * time.Minute,
Timeout: scaleTimeout,
Image: "gcr.io/google_containers/pause-amd64:3.0",
Replicas: megabytes / 500,
MemRequest: 500 * 1024 * 1024,
Replicas: replicas,
MemRequest: request,
}
err := framework.RunRC(*config)
if expectRunning {
framework.ExpectNoError(err)
}
framework.ExpectNoError(framework.RunRC(*config))
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment