Merge pull request #54111 from freehan/neg-test

Automatic merge from submit-queue (batch tested with PRs 54107, 54184, 54377, 54094, 54111). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. add e2e tests for NEG This PR includes tests: - ingress conformance test - scaling up and down backends - switching backend between IG and NEG - rolling update backend should not cause service disruption ```release-note NONE ```

Merge pull request #54111 from freehan/neg-test
8d3a1922 · Kubernetes Submit Queue · GitHub · 8f79857b · 4a24ebfa · 8d3a1922
Commit 8d3a1922 authored Oct 24, 2017 by Kubernetes Submit Queue Committed by GitHub Oct 24, 2017
7 changed files
--- a/test/e2e/e2e.go
+++ b/test/e2e/e2e.go
@@ -74,7 +74,7 @@ func setupProviderConfig() error {
 			managedZones = []string{zone}
 		}

-		gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{})
+		gceAlphaFeatureGate, err := gcecloud.NewAlphaFeatureGate([]string{gcecloud.AlphaFeatureNetworkEndpointGroup})
 		if err != nil {
 			glog.Errorf("Encountered error for creating alpha feature gate: %v", err)
 		}

--- a/test/e2e/framework/ingress_utils.go
+++ b/test/e2e/framework/ingress_utils.go
--- a/test/e2e/network/ingress.go
+++ b/test/e2e/network/ingress.go
@@ -22,7 +22,9 @@ import (
 	"time"

 	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/kubernetes/test/e2e/framework"

@@ -30,6 +32,11 @@ import (
 	. "github.com/onsi/gomega"
 )

+const (
+	NEGAnnotation    = "alpha.cloud.google.com/load-balancer-neg"
+	NEGUpdateTimeout = 2 * time.Minute
+)
+
 var _ = SIGDescribe("Loadbalancing: L7", func() {
 	defer GinkgoRecover()
 	var (
@@ -96,7 +103,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		})

 		It("should conform to Ingress spec", func() {
-			conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
 			for _, t := range conformanceTests {
 				By(t.EntryLog)
 				t.Execute()
@@ -113,7 +120,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns, map[string]string{
 				"kubernetes.io/ingress.global-static-ip-name": ns,
 				"kubernetes.io/ingress.allow-http":            "false",
-			})
+			}, map[string]string{})

 			By("waiting for Ingress to come up with ip: " + ip)
 			httpClient := framework.BuildInsecureClient(framework.IngressReqTimeout)
@@ -149,6 +156,182 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 		// TODO: Implement a multizone e2e that verifies traffic reaches each
 		// zone based on pod labels.
 	})
+	Describe("GCE [Slow] [Feature:NEG]", func() {
+		var gceController *framework.GCEIngressController
+
+		// Platform specific setup
+		BeforeEach(func() {
+			framework.SkipUnlessProviderIs("gce", "gke")
+			By("Initializing gce controller")
+			gceController = &framework.GCEIngressController{
+				Ns:     ns,
+				Client: jig.Client,
+				Cloud:  framework.TestContext.CloudConfig,
+			}
+			gceController.Init()
+		})
+
+		// Platform specific cleanup
+		AfterEach(func() {
+			if CurrentGinkgoTestDescription().Failed {
+				framework.DescribeIng(ns)
+			}
+			if jig.Ingress == nil {
+				By("No ingress created, no cleanup necessary")
+				return
+			}
+			By("Deleting ingress")
+			jig.TryDeleteIngress()
+
+			By("Cleaning up cloud resources")
+			framework.CleanupGCEIngressController(gceController)
+		})
+
+		It("should conform to Ingress spec", func() {
+			jig.PollInterval = 5 * time.Second
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{
+				NEGAnnotation: "true",
+			})
+			for _, t := range conformanceTests {
+				By(t.EntryLog)
+				t.Execute()
+				By(t.ExitLog)
+				jig.WaitForIngress(true)
+				usingNeg, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+				Expect(err).NotTo(HaveOccurred())
+				Expect(usingNeg).To(BeTrue())
+			}
+		})
+
+		It("should be able to switch between IG and NEG modes", func() {
+			var err error
+			By("Create a basic HTTP ingress using NEG")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
+			jig.WaitForIngress(true)
+			usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(usingNEG).To(BeTrue())
+
+			By("Switch backend service to use IG")
+			svcList, err := f.ClientSet.CoreV1().Services(ns).List(metav1.ListOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			for _, svc := range svcList.Items {
+				svc.Annotations[NEGAnnotation] = "false"
+				_, err = f.ClientSet.CoreV1().Services(ns).Update(&svc)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) {
+				return gceController.BackendServiceUsingIG(jig.GetIngressNodePorts(true))
+			})
+			jig.WaitForIngress(true)
+
+			By("Switch backend service to use NEG")
+			svcList, err = f.ClientSet.CoreV1().Services(ns).List(metav1.ListOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			for _, svc := range svcList.Items {
+				svc.Annotations[NEGAnnotation] = "true"
+				_, err = f.ClientSet.CoreV1().Services(ns).Update(&svc)
+				Expect(err).NotTo(HaveOccurred())
+			}
+			wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) {
+				return gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+			})
+			jig.WaitForIngress(true)
+		})
+
+		It("should sync endpoints to NEG", func() {
+			name := "hostname"
+			scaleAndValidateNEG := func(num int) {
+				scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				if scale.Spec.Replicas != int32(num) {
+					scale.Spec.Replicas = int32(num)
+					_, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale)
+					Expect(err).NotTo(HaveOccurred())
+				}
+				wait.Poll(5*time.Second, NEGUpdateTimeout, func() (bool, error) {
+					res, err := jig.GetDistinctResponseFromIngress()
+					if err != nil {
+						return false, err
+					}
+					return res.Len() == num, err
+				})
+			}
+
+			By("Create a basic HTTP ingress using NEG")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
+			jig.WaitForIngress(true)
+			usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(usingNEG).To(BeTrue())
+			// initial replicas number is 1
+			scaleAndValidateNEG(1)
+
+			By("Scale up number of backends to 5")
+			scaleAndValidateNEG(5)
+
+			By("Scale down number of backends to 3")
+			scaleAndValidateNEG(3)
+
+			By("Scale up number of backends to 6")
+			scaleAndValidateNEG(6)
+
+			By("Scale down number of backends to 2")
+			scaleAndValidateNEG(3)
+		})
+
+		It("rolling update backend pods should not cause service disruption", func() {
+			name := "hostname"
+			replicas := 8
+			By("Create a basic HTTP ingress using NEG")
+			jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "neg"), ns, map[string]string{}, map[string]string{})
+			jig.WaitForIngress(true)
+			usingNEG, err := gceController.BackendServiceUsingNEG(jig.GetIngressNodePorts(false))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(usingNEG).To(BeTrue())
+
+			By(fmt.Sprintf("Scale backend replicas to %d", replicas))
+			scale, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).GetScale(name, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			scale.Spec.Replicas = int32(replicas)
+			_, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).UpdateScale(name, scale)
+			Expect(err).NotTo(HaveOccurred())
+			wait.Poll(5*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) {
+				res, err := jig.GetDistinctResponseFromIngress()
+				if err != nil {
+					return false, err
+				}
+				return res.Len() == replicas, err
+			})
+
+			By("Trigger rolling update and observe service disruption")
+			deploy, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			// trigger by changing graceful termination period to 60 seconds
+			gracePeriod := int64(60)
+			deploy.Spec.Template.Spec.TerminationGracePeriodSeconds = &gracePeriod
+			_, err = f.ClientSet.ExtensionsV1beta1().Deployments(ns).Update(deploy)
+			Expect(err).NotTo(HaveOccurred())
+			wait.Poll(30*time.Second, framework.LoadBalancerPollTimeout, func() (bool, error) {
+				res, err := jig.GetDistinctResponseFromIngress()
+				Expect(err).NotTo(HaveOccurred())
+				deploy, err := f.ClientSet.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				if int(deploy.Status.UpdatedReplicas) == replicas {
+					if res.Len() == replicas {
+						return true, nil
+					} else {
+						framework.Logf("Expecting %d different responses, but got %d.", replicas, res.Len())
+						return false, nil
+					}
+
+				} else {
+					framework.Logf("Waiting for rolling update to finished. Keep sending traffic.")
+					return false, nil
+				}
+			})
+		})
+	})

 	// Time: borderline 5m, slow by design
 	Describe("[Slow] Nginx", func() {
@@ -191,7 +374,7 @@ var _ = SIGDescribe("Loadbalancing: L7", func() {
 			// Poll more frequently to reduce e2e completion time.
 			// This test runs in presubmit.
 			jig.PollInterval = 5 * time.Second
-			conformanceTests = framework.CreateIngressComformanceTests(jig, ns)
+			conformanceTests = framework.CreateIngressComformanceTests(jig, ns, map[string]string{})
 			for _, t := range conformanceTests {
 				By(t.EntryLog)
 				t.Execute()

--- a/test/e2e/testing-manifests/ingress/neg/ing.yaml
+++ b/test/e2e/testing-manifests/ingress/neg/ing.yaml
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: hostname
+spec:
+  backend:
+    serviceName: hostname
+    servicePort: 80
\ No newline at end of file
--- a/test/e2e/testing-manifests/ingress/neg/rc.yaml
+++ b/test/e2e/testing-manifests/ingress/neg/rc.yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    run: hostname
+  name: hostname
+spec:
+  template:
+    metadata:
+      labels:
+        run: hostname
+    spec:
+      containers:
+      - image: gcr.io/kubernetes-e2e-test-images/serve-hostname-amd64:1.1
+        imagePullPolicy: IfNotPresent
+        name: hostname
+      terminationGracePeriodSeconds: 120
\ No newline at end of file
--- a/test/e2e/testing-manifests/ingress/neg/svc.yaml
+++ b/test/e2e/testing-manifests/ingress/neg/svc.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: hostname
+  annotations:
+    alpha.cloud.google.com/load-balancer-neg: "true"
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 9376
+  selector:
+    run: hostname
+  sessionAffinity: None
+  type: NodePort
\ No newline at end of file
--- a/test/e2e/upgrades/ingress.go
+++ b/test/e2e/upgrades/ingress.go
@@ -69,7 +69,7 @@ func (t *IngressUpgradeTest) Setup(f *framework.Framework) {
 	jig.CreateIngress(filepath.Join(framework.IngressManifestPath, "static-ip"), ns.Name, map[string]string{
 		"kubernetes.io/ingress.global-static-ip-name": t.ipName,
 		"kubernetes.io/ingress.allow-http":            "false",
-	})
+	}, map[string]string{})

 	By("waiting for Ingress to come up with ip: " + t.ip)
 	framework.ExpectNoError(framework.PollURL(fmt.Sprintf("https://%v/", t.ip), "", framework.LoadBalancerPollTimeout, jig.PollInterval, t.httpClient, false))