Merge pull request #41794 from shashidharatd/federation-upgrade-tests-1

cluster/gce/gci/mounter/mounter.go

@@ -63,7 +63,7 @@ func main() {
 // MountInChroot is to run mount within chroot with the passing root directory
 func mountInChroot(rootfsPath string, args []string) error {
 	if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
-		return fmt.Errorf("Path <%s> does not exist.\n", rootfsPath)
+		return fmt.Errorf("path <%s> does not exist", rootfsPath)
 	}
 	args = append([]string{rootfsPath, mountCmd}, args...)
 	output, err := exec.Command(chrootCmd, args...).CombinedOutput()
@@ -73,7 +73,7 @@ func mountInChroot(rootfsPath string, args []string) error {
 
 	if !strings.EqualFold(string(output), nfsRPCBindErrMsg) {
 		// Mount failed but not because of RPC bind error
-		return fmt.Errorf("Mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s\n", err, chrootCmd, args, string(output))
+		return fmt.Errorf("mount failed: %v\nMounting command: %s\nMounting arguments: %v\nOutput: %s", err, chrootCmd, args, string(output))
 	}
 
 	// Mount failed because it is NFS V3 and we need to run rpcBind

federation/cluster/upgrade.sh

+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../..
+# For `kube::log::status` function since it already sources
+# "${KUBE_ROOT}/cluster/lib/logging.sh"
+source "${KUBE_ROOT}/cluster/common.sh"
+# For $FEDERATION_NAME, $FEDERATION_NAMESPACE, $HOST_CLUSTER_CONTEXT,
+source "${KUBE_ROOT}/federation/cluster/common.sh"
+
+KUBE_VERSION="${1}"
+
+host_kubectl="${KUBE_ROOT}/cluster/kubectl.sh --context=${HOST_CLUSTER_CONTEXT} --namespace=${FEDERATION_NAMESPACE}"
+
+function upgrade() {
+  local -r project="${KUBE_PROJECT:-${PROJECT:-}}"
+  local -r kube_registry="${KUBE_REGISTRY:-gcr.io/${project}}"
+  local -r image_version="${kube_registry}/hyperkube-amd64:${KUBE_VERSION}"
+
+  kube::log::status "Upgrading federation control plane ${FEDERATION_NAME} with image ${image_version}"
+
+  # Upgrade apiserver image
+  ${host_kubectl} set image deployment/federation-apiserver apiserver=${image_version}
+
+  # Upgrade controller-manager image
+  ${host_kubectl} set image deployment/federation-controller-manager controller-manager=${image_version}
+}
+
+upgrade

hack/.linted_packages

 cluster/addons/fluentd-elasticsearch/es-image
+cluster/gce/gci/mounter
 cluster/images/etcd/attachlease
 cluster/images/etcd/rollback
 cmd/clicheck
@@ -385,6 +386,7 @@ staging/src/k8s.io/kube-aggregator/pkg/client/listers/apiregistration/v1alpha1
 staging/src/k8s.io/sample-apiserver
 staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install
 test/e2e/perftype
+test/e2e_federation/upgrades
 test/e2e_node/runner/local
 test/images/clusterapi-tester
 test/images/entrypoint-tester

hack/verify-flags/known-flags.txt

@@ -262,6 +262,7 @@ federated-api-qps
 federated-kube-context
 federation-name
 federation-system-namespace
+federation-upgrade-target
 file-check-frequency
 file_content_in_loop
 file-suffix

test/e2e/cluster_upgrade.go

@@ -17,11 +17,6 @@ limitations under the License.
 package e2e
 
 import (
-	"fmt"
-	"path"
-	"strings"
-
-	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
 	"k8s.io/kubernetes/test/e2e/chaosmonkey"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/upgrades"
@@ -55,10 +50,10 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	framework.KubeDescribe("master upgrade", func() {
 		It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() {
 			cm := chaosmonkey.New(func() {
-				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(checkMasterVersion(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
 			})
 			for _, t := range upgradeTests {
 				cm.RegisterInterface(&chaosMonkeyAdapter{
@@ -75,10 +70,10 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	framework.KubeDescribe("node upgrade", func() {
 		It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() {
 			cm := chaosmonkey.New(func() {
-				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(checkNodesVersions(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
 			})
 			for _, t := range upgradeTests {
 				cm.RegisterInterface(&chaosMonkeyAdapter{
@@ -94,12 +89,12 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 	framework.KubeDescribe("cluster upgrade", func() {
 		It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() {
 			cm := chaosmonkey.New(func() {
-				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(checkMasterVersion(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
 				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(checkNodesVersions(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
 			})
 			for _, t := range upgradeTests {
 				cm.RegisterInterface(&chaosMonkeyAdapter{
@@ -127,12 +122,12 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
 		It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() {
 			cm := chaosmonkey.New(func() {
 				// Yes this really is a downgrade. And nodes must downgrade first.
-				v, err := realVersion(framework.TestContext.UpgradeTarget)
+				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
 				framework.ExpectNoError(err)
 				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(checkNodesVersions(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
 				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(checkMasterVersion(f.ClientSet, v))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
 			})
 			for _, t := range upgradeTests {
 				cm.RegisterInterface(&chaosMonkeyAdapter{
@@ -190,53 +185,3 @@ func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
 func (cma *chaosMonkeyAdapter) Teardown() {
 	cma.test.Teardown(cma.framework)
 }
-
-// realVersion turns a version constant s into a version string deployable on
-// GKE.  See hack/get-build.sh for more information.
-func realVersion(s string) (string, error) {
-	framework.Logf(fmt.Sprintf("Getting real version for %q", s))
-	v, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "hack/get-build.sh"), "-v", s)
-	if err != nil {
-		return v, err
-	}
-	framework.Logf("Version for %q is %q", s, v)
-	return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil
-}
-
-func checkMasterVersion(c clientset.Interface, want string) error {
-	framework.Logf("Checking master version")
-	v, err := c.Discovery().ServerVersion()
-	if err != nil {
-		return fmt.Errorf("checkMasterVersion() couldn't get the master version: %v", err)
-	}
-	// We do prefix trimming and then matching because:
-	// want looks like:  0.19.3-815-g50e67d4
-	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
-	got := strings.TrimPrefix(v.GitVersion, "v")
-	if !strings.HasPrefix(got, want) {
-		return fmt.Errorf("master had kube-apiserver version %s which does not start with %s",
-			got, want)
-	}
-	framework.Logf("Master is at version %s", want)
-	return nil
-}
-
-func checkNodesVersions(cs clientset.Interface, want string) error {
-	l := framework.GetReadySchedulableNodesOrDie(cs)
-	for _, n := range l.Items {
-		// We do prefix trimming and then matching because:
-		// want   looks like:  0.19.3-815-g50e67d4
-		// kv/kvp look  like: v0.19.3-815-g50e67d4034e858-dirty
-		kv, kpv := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v"),
-			strings.TrimPrefix(n.Status.NodeInfo.KubeProxyVersion, "v")
-		if !strings.HasPrefix(kv, want) {
-			return fmt.Errorf("node %s had kubelet version %s which does not start with %s",
-				n.ObjectMeta.Name, kv, want)
-		}
-		if !strings.HasPrefix(kpv, want) {
-			return fmt.Errorf("node %s had kube-proxy version %s which does not start with %s",
-				n.ObjectMeta.Name, kpv, want)
-		}
-	}
-	return nil
-}

test/e2e/framework/BUILD

@@ -32,6 +32,7 @@ go_library(
         "service_util.go",
         "statefulset_utils.go",
         "test_context.go",
+        "upgrade_util.go",
         "util.go",
     ],
     tags = ["automanaged"],

test/e2e/framework/test_context.go

@@ -88,6 +88,8 @@ type TestContextType struct {
 	NodeTestContextType
 	// Federation e2e context
 	FederatedKubeContext string
+	// Federation control plane version to upgrade to while doing upgrade tests
+	FederationUpgradeTarget string
 
 	// Viper-only parameters.  These will in time replace all flags.
 
@@ -202,6 +204,7 @@ func RegisterClusterFlags() {
 	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
 	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
 	flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.")
+	flag.StringVar(&TestContext.FederationUpgradeTarget, "federation-upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an federation upgrade test.")
 	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
 	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
 	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")

test/e2e/framework/upgrade_util.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+)
+
+// RealVersion turns a version constants into a version string deployable on
+// GKE.  See hack/get-build.sh for more information.
+func RealVersion(s string) (string, error) {
+	Logf("Getting real version for %q", s)
+	v, _, err := RunCmd(path.Join(TestContext.RepoRoot, "hack/get-build.sh"), "-v", s)
+	if err != nil {
+		return v, err
+	}
+	Logf("Version for %q is %q", s, v)
+	return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil
+}
+
+func CheckMasterVersion(c clientset.Interface, want string) error {
+	Logf("Checking master version")
+	v, err := c.Discovery().ServerVersion()
+	if err != nil {
+		return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err)
+	}
+	// We do prefix trimming and then matching because:
+	// want looks like:  0.19.3-815-g50e67d4
+	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
+	got := strings.TrimPrefix(v.GitVersion, "v")
+	if !strings.HasPrefix(got, want) {
+		return fmt.Errorf("master had kube-apiserver version %s which does not start with %s",
+			got, want)
+	}
+	Logf("Master is at version %s", want)
+	return nil
+}
+
+func CheckNodesVersions(cs clientset.Interface, want string) error {
+	l := GetReadySchedulableNodesOrDie(cs)
+	for _, n := range l.Items {
+		// We do prefix trimming and then matching because:
+		// want   looks like:  0.19.3-815-g50e67d4
+		// kv/kvp look  like: v0.19.3-815-g50e67d4034e858-dirty
+		kv, kpv := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v"),
+			strings.TrimPrefix(n.Status.NodeInfo.KubeProxyVersion, "v")
+		if !strings.HasPrefix(kv, want) {
+			return fmt.Errorf("node %s had kubelet version %s which does not start with %s",
+				n.ObjectMeta.Name, kv, want)
+		}
+		if !strings.HasPrefix(kpv, want) {
+			return fmt.Errorf("node %s had kube-proxy version %s which does not start with %s",
+				n.ObjectMeta.Name, kpv, want)
+		}
+	}
+	return nil
+}

test/e2e_federation/BUILD

@@ -20,6 +20,7 @@ go_library(
         "replicaset.go",
         "secret.go",
         "service.go",
+        "upgrade.go",
         "util.go",
     ],
     tags = ["automanaged"],
@@ -34,9 +35,11 @@ go_library(
         "//pkg/api/v1:go_default_library",
         "//pkg/apis/extensions/v1beta1:go_default_library",
         "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//test/e2e/chaosmonkey:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e_federation/framework:go_default_library",
+        "//test/e2e_federation/upgrades:go_default_library",
         "//vendor:github.com/onsi/ginkgo",
         "//vendor:github.com/onsi/gomega",
         "//vendor:k8s.io/apimachinery/pkg/api/errors",
@@ -63,6 +66,7 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//test/e2e_federation/framework:all-srcs",
+        "//test/e2e_federation/upgrades:all-srcs",
     ],
     tags = ["automanaged"],
 )

test/e2e_federation/framework/util.go

@@ -19,7 +19,9 @@ package framework
 import (
 	"fmt"
 	"os"
+	"path"
 	"regexp"
+	"strings"
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -110,3 +112,46 @@ func GetValidDNSSubdomainName(name string) (string, error) {
 	}
 	return name, nil
 }
+
+func FederationControlPlaneUpgrade(version string) error {
+	version = "v" + version
+	_, _, err := framework.RunCmd(path.Join(framework.TestContext.RepoRoot, "federation/cluster/upgrade.sh"), version)
+	return err
+}
+
+func CheckFederationVersion(c federation_clientset.Interface, want string) error {
+	framework.Logf("Checking federation version")
+	v, err := c.Discovery().ServerVersion()
+	if err != nil {
+		return fmt.Errorf("CheckFederationVersion() couldn't get the master version: %v", err)
+	}
+	// We do prefix trimming and then matching because:
+	// want looks like:  0.19.3-815-g50e67d4
+	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
+	got := strings.TrimPrefix(v.GitVersion, "v")
+	if !strings.HasPrefix(got, want) {
+		return fmt.Errorf("federation had apiserver version %s which does not start with %s",
+			got, want)
+	}
+	framework.Logf("Federation is at version %s", want)
+	return nil
+}
+
+func MasterUpgrade(context, version string) error {
+	switch framework.TestContext.Provider {
+	case "gce":
+		return masterUpgradeGCE(context, version)
+	default:
+		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", framework.TestContext.Provider)
+	}
+}
+
+func masterUpgradeGCE(context, rawVersion string) error {
+	version := "v" + rawVersion
+	// TODO: this breaks if we want to upgrade 2 clusters in same zone. use alternate methods in future to get zone of a cluster
+	zone := strings.TrimPrefix(context, "federation-e2e-"+framework.TestContext.Provider+"-")
+
+	env := append(os.Environ(), "KUBE_CONTEXT="+context, "ZONE="+zone)
+	_, _, err := framework.RunCmdEnv(env, path.Join(framework.TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", version)
+	return err
+}

test/e2e_federation/upgrade.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_federation
+
+import (
+	"k8s.io/kubernetes/test/e2e/chaosmonkey"
+	"k8s.io/kubernetes/test/e2e/framework"
+	fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+	"k8s.io/kubernetes/test/e2e_federation/upgrades"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var upgradeTests = []upgrades.Test{}
+
+var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
+	f := fedframework.NewDefaultFederatedFramework("federation-upgrade")
+
+	framework.KubeDescribe("Federation Control Plane upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("FCP upgrade followed by federated clusters upgrade", func() {
+		It("should maintain a functioning federation [Feature:FCPUpgradeFollowedByFederatedClustersUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federationControlPlaneUpgrade(f)
+				federatedClustersUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FCPUpgradeFollowedByFederatedClustersUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+
+	framework.KubeDescribe("Federated clusters upgrade followed by FCP upgrade", func() {
+		It("should maintain a functioning federation [Feature:FederatedClustersUpgradeFollowedByFCPUpgrade]", func() {
+			fedframework.SkipUnlessFederated(f.ClientSet)
+			cm := chaosmonkey.New(func() {
+				federatedClustersUpgrade(f)
+				federationControlPlaneUpgrade(f)
+			})
+			for _, t := range upgradeTests {
+				cm.RegisterInterface(&chaosMonkeyAdapter{
+					test:        t,
+					framework:   f,
+					upgradeType: upgrades.FederatedClustersUpgradeFollowedByFCPUpgrade,
+				})
+			}
+			cm.Do()
+		})
+	})
+})
+
+type chaosMonkeyAdapter struct {
+	test        upgrades.Test
+	framework   *fedframework.Framework
+	upgradeType upgrades.FederationUpgradeType
+}
+
+func (cma *chaosMonkeyAdapter) Setup() {
+	cma.test.Setup(cma.framework)
+}
+
+func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
+	cma.test.Test(cma.framework, stopCh, cma.upgradeType)
+}
+
+func (cma *chaosMonkeyAdapter) Teardown() {
+	cma.test.Teardown(cma.framework)
+}
+
+func federationControlPlaneUpgrade(f *fedframework.Framework) {
+	federationVersion, err := framework.RealVersion(framework.TestContext.FederationUpgradeTarget)
+	framework.ExpectNoError(err)
+	framework.ExpectNoError(fedframework.FederationControlPlaneUpgrade(federationVersion))
+	framework.ExpectNoError(fedframework.CheckFederationVersion(f.FederationClientset, federationVersion))
+}
+
+func federatedClustersUpgrade(f *fedframework.Framework) {
+	k8sVersion, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
+	framework.ExpectNoError(err)
+	clusters, _ := getRegisteredClusters(UserAgentName, f)
+	for _, cluster := range clusters {
+		framework.ExpectNoError(fedframework.MasterUpgrade(cluster.name, k8sVersion))
+		framework.ExpectNoError(framework.CheckMasterVersion(cluster.Clientset, k8sVersion))
+
+		// TODO: Need to add Node upgrade. Add once this framework is stable
+	}
+}

test/e2e_federation/upgrades/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["upgrade.go"],
+    tags = ["automanaged"],
+    deps = ["//test/e2e_federation/framework:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

test/e2e_federation/upgrades/upgrade.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package upgrades provides a framework for testing Kubernetes federation
+// features before, during, and after different types of upgrades.
+package upgrades
+
+import fedframework "k8s.io/kubernetes/test/e2e_federation/framework"
+
+// FederationUpgradeType represents different types of federation upgrades.
+type FederationUpgradeType int
+
+const (
+	// FCPUpgrade indicates that federation control plane is being upgraded.
+	FCPUpgrade FederationUpgradeType = iota
+
+	// FederatedClustersUpgrade indicates that federated clusters are being upgraded.
+	FederatedClustersUpgrade
+
+	// FCPUpgradeFollowedByFederatedClustersUpgrade indicates that federation control plane is upgraded
+	// followed by federated clusters upgrade.
+	FCPUpgradeFollowedByFederatedClustersUpgrade
+
+	// FederatedClustersUpgradeFollowedByFCPUpgrade indicates that federated clusters are upgraded
+	// followed by federation control plane upgrade.
+	FederatedClustersUpgradeFollowedByFCPUpgrade
+)
+
+// Test is an interface for federation upgrade tests.
+type Test interface {
+	// Setup should create and verify whatever objects need to
+	// exist before the upgrade disruption starts.
+	Setup(f *fedframework.Framework)
+
+	// Test will run during the upgrade. When the upgrade is
+	// complete, done will be closed and final validation can
+	// begin.
+	Test(f *fedframework.Framework, done <-chan struct{}, upgrade FederationUpgradeType)
+
+	// TearDown should clean up any objects that are created that
+	// aren't already cleaned up by the framework.
+	Teardown(f *fedframework.Framework)
+}