Merge pull request #43089 from krousey/upgrades

test/e2e/BUILD

@@ -196,6 +196,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/watch",
         "//vendor:k8s.io/apiserver/pkg/authentication/serviceaccount",
         "//vendor:k8s.io/apiserver/pkg/registry/generic/registry",
+        "//vendor:k8s.io/client-go/discovery",
         "//vendor:k8s.io/client-go/kubernetes",
         "//vendor:k8s.io/client-go/pkg/api/v1",
         "//vendor:k8s.io/client-go/pkg/apis/extensions/v1beta1",

test/e2e/cluster_upgrade.go

@@ -17,6 +17,8 @@ limitations under the License.
 package e2e
 
 import (
+	"k8s.io/client-go/discovery"
+	"k8s.io/kubernetes/pkg/util/version"
 	"k8s.io/kubernetes/test/e2e/chaosmonkey"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/upgrades"
@@ -50,18 +52,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 
 	framework.KubeDescribe("master upgrade", func() {
 		It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
 			cm := chaosmonkey.New(func() {
-				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
-				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.MasterUpgrade(target))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
 			})
 			for _, t := range upgradeTests {
-				cm.RegisterInterface(&chaosMonkeyAdapter{
+				cma := chaosMonkeyAdapter{
 					test:        t,
 					framework:   testFrameworks[t.Name()],
 					upgradeType: upgrades.MasterUpgrade,
-				})
+					upgCtx:      *upgCtx,
+				}
+				cm.Register(cma.Test)
 			}
 
 			cm.Do()
@@ -70,18 +76,22 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 
 	framework.KubeDescribe("node upgrade", func() {
 		It("should maintain a functioning cluster [Feature:NodeUpgrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
 			cm := chaosmonkey.New(func() {
-				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
-				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
 			})
 			for _, t := range upgradeTests {
-				cm.RegisterInterface(&chaosMonkeyAdapter{
+				cma := chaosMonkeyAdapter{
 					test:        t,
 					framework:   testFrameworks[t.Name()],
 					upgradeType: upgrades.NodeUpgrade,
-				})
+					upgCtx:      *upgCtx,
+				}
+				cm.Register(cma.Test)
 			}
 			cm.Do()
 		})
@@ -89,20 +99,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 
 	framework.KubeDescribe("cluster upgrade", func() {
 		It("should maintain a functioning cluster [Feature:ClusterUpgrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
 			cm := chaosmonkey.New(func() {
-				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
-				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
-				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.MasterUpgrade(target))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
+				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
 			})
 			for _, t := range upgradeTests {
-				cm.RegisterInterface(&chaosMonkeyAdapter{
+				cma := chaosMonkeyAdapter{
 					test:        t,
 					framework:   testFrameworks[t.Name()],
 					upgradeType: upgrades.ClusterUpgrade,
-				})
+					upgCtx:      *upgCtx,
+				}
+				cm.Register(cma.Test)
 			}
 			cm.Do()
 		})
@@ -121,21 +135,25 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
 
 	framework.KubeDescribe("cluster downgrade", func() {
 		It("should maintain a functioning cluster [Feature:ClusterDowngrade]", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+			framework.ExpectNoError(err)
+
 			cm := chaosmonkey.New(func() {
 				// Yes this really is a downgrade. And nodes must downgrade first.
-				v, err := framework.RealVersion(framework.TestContext.UpgradeTarget)
-				framework.ExpectNoError(err)
-				framework.ExpectNoError(framework.NodeUpgrade(f, v, framework.TestContext.UpgradeImage))
-				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, v))
-				framework.ExpectNoError(framework.MasterUpgrade(v))
-				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, v))
+				target := upgCtx.Versions[1].Version.String()
+				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
+				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
+				framework.ExpectNoError(framework.MasterUpgrade(target))
+				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
 			})
 			for _, t := range upgradeTests {
-				cm.RegisterInterface(&chaosMonkeyAdapter{
+				cma := chaosMonkeyAdapter{
 					test:        t,
 					framework:   testFrameworks[t.Name()],
 					upgradeType: upgrades.ClusterUpgrade,
-				})
+					upgCtx:      *upgCtx,
+				}
+				cm.Register(cma.Test)
 			}
 			cm.Do()
 		})
@@ -143,6 +161,8 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
 })
 
 var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
+	f := framework.NewDefaultFramework("etc-upgrade")
+
 	// Create the frameworks here because we can only create them
 	// in a "Describe".
 	testFrameworks := map[string]*framework.Framework{}
@@ -152,16 +172,21 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
 
 	framework.KubeDescribe("etcd upgrade", func() {
 		It("should maintain a functioning cluster", func() {
+			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
+			framework.ExpectNoError(err)
+
 			cm := chaosmonkey.New(func() {
 				framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
 				// TODO(mml): verify the etcd version
 			})
 			for _, t := range upgradeTests {
-				cm.RegisterInterface(&chaosMonkeyAdapter{
+				cma := chaosMonkeyAdapter{
 					test:        t,
 					framework:   testFrameworks[t.Name()],
 					upgradeType: upgrades.EtcdUpgrade,
-				})
+					upgCtx:      *upgCtx,
+				}
+				cm.Register(cma.Test)
 			}
 
 			cm.Do()
@@ -173,16 +198,60 @@ type chaosMonkeyAdapter struct {
 	test        upgrades.Test
 	framework   *framework.Framework
 	upgradeType upgrades.UpgradeType
+	upgCtx      upgrades.UpgradeContext
 }
 
-func (cma *chaosMonkeyAdapter) Setup() {
+func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
+	if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) {
+		By("skipping test " + cma.test.Name())
+		sem.Ready()
+		return
+	}
+
 	cma.test.Setup(cma.framework)
+	defer cma.test.Teardown(cma.framework)
+	sem.Ready()
+	cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType)
 }
 
-func (cma *chaosMonkeyAdapter) Test(stopCh <-chan struct{}) {
-	cma.test.Test(cma.framework, stopCh, cma.upgradeType)
-}
+func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) {
+	current, err := c.ServerVersion()
+	if err != nil {
+		return nil, err
+	}
+
+	curVer, err := version.ParseSemantic(current.String())
+	if err != nil {
+		return nil, err
+	}
+
+	upgCtx := &upgrades.UpgradeContext{
+		Versions: []upgrades.VersionContext{
+			{
+				Version:   *curVer,
+				NodeImage: framework.TestContext.NodeOSDistro,
+			},
+		},
+	}
+
+	if len(upgradeTarget) == 0 {
+		return upgCtx, nil
+	}
+
+	next, err := framework.RealVersion(upgradeTarget)
+	if err != nil {
+		return nil, err
+	}
+
+	nextVer, err := version.ParseSemantic(next)
+	if err != nil {
+		return nil, err
+	}
+
+	upgCtx.Versions = append(upgCtx.Versions, upgrades.VersionContext{
+		Version:   *nextVer,
+		NodeImage: framework.TestContext.UpgradeImage,
+	})
 
-func (cma *chaosMonkeyAdapter) Teardown() {
-	cma.test.Teardown(cma.framework)
+	return upgCtx, nil
 }

test/e2e/common/apparmor.go

@@ -32,8 +32,11 @@ const (
 	appArmorDeniedPath    = "/expect_permission_denied"
 )
 
+// AppArmorDistros are distros with AppArmor support
+var AppArmorDistros = []string{"gci", "ubuntu"}
+
 func SkipIfAppArmorNotSupported() {
-	framework.SkipUnlessNodeOSDistroIs("gci", "ubuntu")
+	framework.SkipUnlessNodeOSDistroIs(AppArmorDistros...)
 }
 
 func LoadAppArmorProfiles(f *framework.Framework) {

test/e2e/upgrades/BUILD

@@ -33,6 +33,7 @@ go_library(
         "//pkg/controller:go_default_library",
         "//pkg/controller/deployment/util:go_default_library",
         "//pkg/kubelet/sysctl:go_default_library",
+        "//pkg/util/version:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//vendor:github.com/onsi/ginkgo",

test/e2e/upgrades/apparmor.go

@@ -34,9 +34,22 @@ type AppArmorUpgradeTest struct {
 
 func (AppArmorUpgradeTest) Name() string { return "apparmor-upgrade" }
 
+func (AppArmorUpgradeTest) Skip(upgCtx UpgradeContext) bool {
+	var supportedImages map[string]bool
+	for _, d := range common.AppArmorDistros {
+		supportedImages[d] = true
+	}
+
+	for _, vCtx := range upgCtx.Versions {
+		if !supportedImages[vCtx.NodeImage] {
+			return true
+		}
+	}
+	return false
+}
+
 // Setup creates a secret and then verifies that a pod can consume it.
 func (t *AppArmorUpgradeTest) Setup(f *framework.Framework) {
-	common.SkipIfAppArmorNotSupported()
 	By("Loading AppArmor profiles to nodes")
 	common.LoadAppArmorProfiles(f)
 

test/e2e/upgrades/statefulset.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api/v1"
 	apps "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
+	"k8s.io/kubernetes/pkg/util/version"
 
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -35,6 +36,17 @@ type StatefulSetUpgradeTest struct {
 
 func (StatefulSetUpgradeTest) Name() string { return "statefulset-upgrade" }
 
+func (StatefulSetUpgradeTest) Skip(upgCtx UpgradeContext) bool {
+	minVersion := version.MustParseSemantic("1.5.0")
+
+	for _, vCtx := range upgCtx.Versions {
+		if vCtx.Version.LessThan(minVersion) {
+			return true
+		}
+	}
+	return false
+}
+
 // Setup creates a StatefulSet and a HeadlessService. It verifies the basic SatefulSet properties
 func (t *StatefulSetUpgradeTest) Setup(f *framework.Framework) {
 	ssName := "ss"

test/e2e/upgrades/upgrade.go

@@ -18,7 +18,10 @@ limitations under the License.
 // features before, during, and after different types of upgrades.
 package upgrades
 
-import "k8s.io/kubernetes/test/e2e/framework"
+import (
+	"k8s.io/kubernetes/pkg/util/version"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
 
 // UpgradeType represents different types of upgrades.
 type UpgradeType int
@@ -57,3 +60,24 @@ type Test interface {
 	// aren't already cleaned up by the framework.
 	Teardown(f *framework.Framework)
 }
+
+// Skippable is an interface that an upgrade test can implement to be
+// able to indicate that it should be skipped.
+type Skippable interface {
+	// Skip should return true if test should be skipped. upgCtx
+	// provides information about the upgrade that is going to
+	// occur.
+	Skip(upgCtx UpgradeContext) bool
+}
+
+// UpgradeContext contains information about all the stages of the
+// upgrade that is going to occur.
+type UpgradeContext struct {
+	Versions []VersionContext
+}
+
+// VersionContext represents a stage of the upgrade.
+type VersionContext struct {
+	Version   version.Version
+	NodeImage string
+}