Merge pull request #41373 from msau42/e2e-pvutil

test/e2e/BUILD

@@ -79,7 +79,6 @@ go_library(
         "proxy.go",
         "pv_reclaimpolicy.go",
         "pvc_label_selector.go",
-        "pvutil.go",
         "rc.go",
         "reboot.go",
         "replica_set.go",

test/e2e/framework/BUILD

@@ -25,6 +25,7 @@ go_library(
         "nodes_util.go",
         "perf_util.go",
         "pods.go",
+        "pv_util.go",
         "resource_usage_gatherer.go",
         "service_util.go",
         "statefulset_utils.go",
@@ -69,6 +70,7 @@ go_library(
         "//pkg/util/labels:go_default_library",
         "//pkg/util/system:go_default_library",
         "//pkg/util/version:go_default_library",
+        "//pkg/volume/util/volumehelper:go_default_library",
         "//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
         "//plugin/pkg/scheduler/schedulercache:go_default_library",
         "//test/e2e/perftype:go_default_library",

test/e2e/framework/util.go

@@ -4594,6 +4594,14 @@ func getPodLogsInternal(c clientset.Interface, namespace, podName, containerName
 	return string(logs), err
 }
 
+func GetGCECloud() (*gcecloud.GCECloud, error) {
+	gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider)
+	}
+	return gceCloud, nil
+}
+
 // EnsureLoadBalancerResourcesDeleted ensures that cloud load balancer resources that were created
 // are actually cleaned up.  Currently only implemented for GCE/GKE.
 func EnsureLoadBalancerResourcesDeleted(ip, portRange string) error {
@@ -4604,9 +4612,9 @@ func EnsureLoadBalancerResourcesDeleted(ip, portRange string) error {
 }
 
 func ensureGCELoadBalancerResourcesDeleted(ip, portRange string) error {
-	gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud)
-	if !ok {
-		return fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider)
+	gceCloud, err := GetGCECloud()
+	if err != nil {
+		return err
 	}
 	project := TestContext.CloudConfig.ProjectID
 	region, err := gcecloud.GetGCERegion(TestContext.CloudConfig.Zone)
@@ -5094,9 +5102,9 @@ func (p *E2ETestNodePreparer) CleanupNodes() error {
 // the given name. The name is usually the UUID of the Service prefixed with an
 // alpha-numeric character ('a') to work around cloudprovider rules.
 func CleanupGCEResources(loadBalancerName string) (retErr error) {
-	gceCloud, ok := TestContext.CloudConfig.Provider.(*gcecloud.GCECloud)
-	if !ok {
-		return fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", TestContext.CloudConfig.Provider)
+	gceCloud, err := GetGCECloud()
+	if err != nil {
+		return err
 	}
 	if err := gceCloud.DeleteFirewall(loadBalancerName); err != nil &&
 		!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) {

test/e2e/kubelet.go

@@ -415,8 +415,8 @@ var _ = framework.KubeDescribe("kubelet", func() {
 			})
 
 			AfterEach(func() {
-				deletePodWithWait(f, c, pod)
-				deletePodWithWait(f, c, nfsServerPod)
+				framework.DeletePodWithWait(f, c, pod)
+				framework.DeletePodWithWait(f, c, nfsServerPod)
 			})
 
 			// execute It blocks from above table of tests
@@ -427,11 +427,11 @@ var _ = framework.KubeDescribe("kubelet", func() {
 					pod = createPodUsingNfs(f, c, ns, nfsIP, t.podCmd)
 
 					By("Delete the NFS server pod")
-					deletePodWithWait(f, c, nfsServerPod)
+					framework.DeletePodWithWait(f, c, nfsServerPod)
 					nfsServerPod = nil
 
 					By("Delete the pod mounted to the NFS volume")
-					deletePodWithWait(f, c, pod)
+					framework.DeletePodWithWait(f, c, pod)
 					// pod object is now stale, but is intentionally not nil
 
 					By("Check if host running deleted pod has been cleaned up -- expect not")

test/e2e/pd.go

@@ -589,7 +589,7 @@ func createPD() (string, error) {
 	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
 		pdName := fmt.Sprintf("%s-%s", framework.TestContext.Prefix, string(uuid.NewUUID()))
 
-		gceCloud, err := getGCECloud()
+		gceCloud, err := framework.GetGCECloud()
 		if err != nil {
 			return "", err
 		}
@@ -624,7 +624,7 @@ func createPD() (string, error) {
 
 func deletePD(pdName string) error {
 	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
-		gceCloud, err := getGCECloud()
+		gceCloud, err := framework.GetGCECloud()
 		if err != nil {
 			return err
 		}
@@ -663,7 +663,7 @@ func deletePD(pdName string) error {
 
 func detachPD(nodeName types.NodeName, pdName string) error {
 	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
-		gceCloud, err := getGCECloud()
+		gceCloud, err := framework.GetGCECloud()
 		if err != nil {
 			return err
 		}
@@ -771,7 +771,7 @@ func testPDPod(diskNames []string, targetNode types.NodeName, readOnly bool, num
 func waitForPDDetach(diskName string, nodeName types.NodeName) error {
 	if framework.TestContext.Provider == "gce" || framework.TestContext.Provider == "gke" {
 		framework.Logf("Waiting for GCE PD %q to detach from node %q.", diskName, nodeName)
-		gceCloud, err := getGCECloud()
+		gceCloud, err := framework.GetGCECloud()
 		if err != nil {
 			return err
 		}
@@ -798,16 +798,6 @@ func waitForPDDetach(diskName string, nodeName types.NodeName) error {
 	return nil
 }
 
-func getGCECloud() (*gcecloud.GCECloud, error) {
-	gceCloud, ok := framework.TestContext.CloudConfig.Provider.(*gcecloud.GCECloud)
-
-	if !ok {
-		return nil, fmt.Errorf("failed to convert CloudConfig.Provider to GCECloud: %#v", framework.TestContext.CloudConfig.Provider)
-	}
-
-	return gceCloud, nil
-}
-
 func detachAndDeletePDs(diskName string, hosts []types.NodeName) {
 	for _, host := range hosts {
 		framework.Logf("Detaching GCE PD %q from node %q.", diskName, host)

test/e2e/persistent_volumes-disruptive.go

@@ -54,7 +54,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
 		c                         clientset.Interface
 		ns                        string
 		nfsServerPod              *v1.Pod
-		nfsPVconfig               persistentVolumeConfig
+		nfsPVconfig               framework.PersistentVolumeConfig
 		nfsServerIP, clientNodeIP string
 		clientNode                *v1.Node
 	)
@@ -72,9 +72,9 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
 		framework.Logf("[BeforeEach] Configuring PersistentVolume")
 		nfsServerIP = nfsServerPod.Status.PodIP
 		Expect(nfsServerIP).NotTo(BeEmpty())
-		nfsPVconfig = persistentVolumeConfig{
-			namePrefix: "nfs-",
-			pvSource: v1.PersistentVolumeSource{
+		nfsPVconfig = framework.PersistentVolumeConfig{
+			NamePrefix: "nfs-",
+			PVSource: v1.PersistentVolumeSource{
 				NFS: &v1.NFSVolumeSource{
 					Server:   nfsServerIP,
 					Path:     "/exports",
@@ -98,7 +98,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Volume][Disruptive][Flaky]",
 	})
 
 	AfterEach(func() {
-		deletePodWithWait(f, c, nfsServerPod)
+		framework.DeletePodWithWait(f, c, nfsServerPod)
 	})
 
 	Context("when kubelet restarts", func() {
@@ -175,7 +175,7 @@ func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framew
 
 	By("Restarting the kubelet.")
 	kubeletCommand(kStop, c, clientPod)
-	deletePodWithWait(f, c, clientPod)
+	framework.DeletePodWithWait(f, c, clientPod)
 	kubeletCommand(kStart, c, clientPod)
 
 	By("Expecting the volume mount not to be found.")
@@ -187,9 +187,9 @@ func testVolumeUnmountsFromDeletedPod(c clientset.Interface, f *framework.Framew
 }
 
 // initTestCase initializes spec resources (pv, pvc, and pod) and returns pointers to be consumed by the test
-func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig persistentVolumeConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
-	pv, pvc := createPVPVC(c, pvConfig, ns, false)
-	pod := makePod(ns, pvc.Name)
+func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig framework.PersistentVolumeConfig, ns, nodeName string) (*v1.Pod, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
+	pv, pvc := framework.CreatePVPVC(c, pvConfig, ns, false)
+	pod := framework.MakePod(ns, pvc.Name)
 	pod.Spec.NodeName = nodeName
 	framework.Logf("Creating nfs client Pod %s on node %s", pod.Name, nodeName)
 	pod, err := c.Core().Pods(ns).Create(pod)
@@ -208,9 +208,9 @@ func initTestCase(f *framework.Framework, c clientset.Interface, pvConfig persis
 
 // tearDownTestCase destroy resources created by initTestCase.
 func tearDownTestCase(c clientset.Interface, f *framework.Framework, ns string, pod *v1.Pod, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) {
-	deletePodWithWait(f, c, pod)
-	deletePersistentVolumeClaim(c, pvc.Name, ns)
-	deletePersistentVolume(c, pv.Name)
+	framework.DeletePodWithWait(f, c, pod)
+	framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
+	framework.DeletePersistentVolume(c, pv.Name)
 }
 
 // kubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod.

test/e2e/persistent_volumes-vsphere.go

@@ -38,7 +38,7 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 		pv         *v1.PersistentVolume
 		pvc        *v1.PersistentVolumeClaim
 		clientPod  *v1.Pod
-		pvConfig   persistentVolumeConfig
+		pvConfig   framework.PersistentVolumeConfig
 		vsp        *vsphere.VSphere
 		err        error
 		node       types.NodeName
@@ -69,23 +69,23 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 		if volumePath == "" {
 			volumePath, err = createVSphereVolume(vsp, nil)
 			Expect(err).NotTo(HaveOccurred())
-			pvConfig = persistentVolumeConfig{
-				namePrefix: "vspherepv-",
-				pvSource: v1.PersistentVolumeSource{
+			pvConfig = framework.PersistentVolumeConfig{
+				NamePrefix: "vspherepv-",
+				PVSource: v1.PersistentVolumeSource{
 					VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
 						VolumePath: volumePath,
 						FSType:     "ext4",
 					},
 				},
-				prebind: nil,
+				Prebind: nil,
 			}
 		}
 		By("Creating the PV and PVC")
-		pv, pvc = createPVPVC(c, pvConfig, ns, false)
-		waitOnPVandPVC(c, ns, pv, pvc)
+		pv, pvc = framework.CreatePVPVC(c, pvConfig, ns, false)
+		framework.WaitOnPVandPVC(c, ns, pv, pvc)
 
 		By("Creating the Client Pod")
-		clientPod = createClientPod(c, ns, pvc)
+		clientPod = framework.CreateClientPod(c, ns, pvc)
 		node := types.NodeName(clientPod.Spec.NodeName)
 
 		By("Verify disk should be attached to the node")
@@ -100,15 +100,15 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 			if clientPod != nil {
 				clientPod, err = c.CoreV1().Pods(ns).Get(clientPod.Name, metav1.GetOptions{})
 				if !apierrs.IsNotFound(err) {
-					deletePodWithWait(f, c, clientPod)
+					framework.DeletePodWithWait(f, c, clientPod)
 				}
 			}
 
 			if pv != nil {
-				deletePersistentVolume(c, pv.Name)
+				framework.DeletePersistentVolume(c, pv.Name)
 			}
 			if pvc != nil {
-				deletePersistentVolumeClaim(c, pvc.Name, ns)
+				framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 			}
 		}
 	})
@@ -136,7 +136,7 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 
 	It("should test that deleting a PVC before the pod does not cause pod deletion to fail on PD detach", func() {
 		By("Deleting the Claim")
-		deletePersistentVolumeClaim(c, pvc.Name, ns)
+		framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 
 		pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
 		if !apierrs.IsNotFound(err) {
@@ -144,7 +144,7 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 		}
 		pvc = nil
 		By("Deleting the Pod")
-		deletePodWithWait(f, c, clientPod)
+		framework.DeletePodWithWait(f, c, clientPod)
 
 	})
 
@@ -157,13 +157,13 @@ var _ = framework.KubeDescribe("PersistentVolumes:vsphere", func() {
 	*/
 	It("should test that deleting the PV before the pod does not cause pod deletion to fail on PD detach", func() {
 		By("Deleting the Persistent Volume")
-		deletePersistentVolume(c, pv.Name)
+		framework.DeletePersistentVolume(c, pv.Name)
 		pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
 		if !apierrs.IsNotFound(err) {
 			Expect(err).NotTo(HaveOccurred())
 		}
 		pv = nil
 		By("Deleting the pod")
-		deletePodWithWait(f, c, clientPod)
+		framework.DeletePodWithWait(f, c, clientPod)
 	})
 })

test/e2e/pv_reclaimpolicy.go

@@ -119,14 +119,14 @@ var _ = framework.KubeDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func
 			writeContentToVSpherePV(c, pvc, volumeFileContent)
 
 			By("Delete PVC")
-			deletePersistentVolumeClaim(c, pvc.Name, ns)
+			framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 			pvc = nil
 
 			By("Verify PV is retained")
 			framework.Logf("Waiting for PV %v to become Released", pv.Name)
 			err = framework.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 3*time.Second, 300*time.Second)
 			Expect(err).NotTo(HaveOccurred())
-			deletePersistentVolume(c, pv.Name)
+			framework.DeletePersistentVolume(c, pv.Name)
 
 			By("Creating the PV for same volume path")
 			pv = getVSpherePersistentVolumeSpec(volumePath, v1.PersistentVolumeReclaimRetain, nil)
@@ -139,7 +139,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Feature:ReclaimPolicy]", func
 			Expect(err).NotTo(HaveOccurred())
 
 			By("wait for the pv and pvc to bind")
-			waitOnPVandPVC(c, ns, pv, pvc)
+			framework.WaitOnPVandPVC(c, ns, pv, pvc)
 			verifyContentOfVSpherePV(c, pvc, volumeFileContent)
 
 		})
@@ -173,10 +173,10 @@ func testCleanupVSpherePersistentVolumeReclaim(vsp *vsphere.VSphere, c clientset
 		vsp.DeleteVolume(volumePath)
 	}
 	if pv != nil {
-		deletePersistentVolume(c, pv.Name)
+		framework.DeletePersistentVolume(c, pv.Name)
 	}
 	if pvc != nil {
-		deletePersistentVolumeClaim(c, pvc.Name, ns)
+		framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 	}
 }
 
@@ -185,10 +185,10 @@ func deletePVCAfterBind(c clientset.Interface, ns string, pvc *v1.PersistentVolu
 	var err error
 
 	By("wait for the pv and pvc to bind")
-	waitOnPVandPVC(c, ns, pv, pvc)
+	framework.WaitOnPVandPVC(c, ns, pv, pvc)
 
 	By("delete pvc")
-	deletePersistentVolumeClaim(c, pvc.Name, ns)
+	framework.DeletePersistentVolumeClaim(c, pvc.Name, ns)
 	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
 	if !apierrs.IsNotFound(err) {
 		Expect(err).NotTo(HaveOccurred())

test/e2e/pvc_label_selector.go

@@ -81,14 +81,14 @@ var _ = framework.KubeDescribe("PersistentVolumes [Feature:LabelSelector]", func
 			Expect(err).NotTo(HaveOccurred())
 
 			By("wait for the pvc_ssd to bind with pv_ssd")
-			waitOnPVandPVC(c, ns, pv_ssd, pvc_ssd)
+			framework.WaitOnPVandPVC(c, ns, pv_ssd, pvc_ssd)
 
 			By("Verify status of pvc_vvol is pending")
 			err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimPending, c, ns, pvc_vvol.Name, 3*time.Second, 300*time.Second)
 			Expect(err).NotTo(HaveOccurred())
 
 			By("delete pvc_ssd")
-			deletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
+			framework.DeletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
 
 			By("verify pv_ssd is deleted")
 			err = framework.WaitForPersistentVolumeDeleted(c, pv_ssd.Name, 3*time.Second, 300*time.Second)
@@ -96,7 +96,7 @@ var _ = framework.KubeDescribe("PersistentVolumes [Feature:LabelSelector]", func
 			volumePath = ""
 
 			By("delete pvc_vvol")
-			deletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
+			framework.DeletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
 		})
 	})
 })
@@ -139,12 +139,12 @@ func testCleanupVSpherePVClabelselector(c clientset.Interface, ns string, volume
 		vsp.DeleteVolume(volumePath)
 	}
 	if pvc_ssd != nil {
-		deletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
+		framework.DeletePersistentVolumeClaim(c, pvc_ssd.Name, ns)
 	}
 	if pvc_vvol != nil {
-		deletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
+		framework.DeletePersistentVolumeClaim(c, pvc_vvol.Name, ns)
 	}
 	if pv_ssd != nil {
-		deletePersistentVolume(c, pv_ssd.Name)
+		framework.DeletePersistentVolume(c, pv_ssd.Name)
 	}
 }

test/e2e/volume_provisioning.go

@@ -143,7 +143,7 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 			allZones := sets.NewString()     // all zones in the project
 			managedZones := sets.NewString() // subset of allZones
 
-			gceCloud, err := getGCECloud()
+			gceCloud, err := framework.GetGCECloud()
 			Expect(err).NotTo(HaveOccurred())
 
 			// Get all k8s managed zones

test/e2e/vsphere_utils.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"fmt"
+
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -84,24 +85,24 @@ func waitForVSphereDiskToDetach(vsp *vsphere.VSphere, volumePath string, nodeNam
 // function to create vsphere volume spec with given VMDK volume path, Reclaim Policy and labels
 func getVSpherePersistentVolumeSpec(volumePath string, persistentVolumeReclaimPolicy v1.PersistentVolumeReclaimPolicy, labels map[string]string) *v1.PersistentVolume {
 	var (
-		pvConfig persistentVolumeConfig
+		pvConfig framework.PersistentVolumeConfig
 		pv       *v1.PersistentVolume
 		claimRef *v1.ObjectReference
 	)
-	pvConfig = persistentVolumeConfig{
-		namePrefix: "vspherepv-",
-		pvSource: v1.PersistentVolumeSource{
+	pvConfig = framework.PersistentVolumeConfig{
+		NamePrefix: "vspherepv-",
+		PVSource: v1.PersistentVolumeSource{
 			VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
 				VolumePath: volumePath,
 				FSType:     "ext4",
 			},
 		},
-		prebind: nil,
+		Prebind: nil,
 	}
 
 	pv = &v1.PersistentVolume{
 		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: pvConfig.namePrefix,
+			GenerateName: pvConfig.NamePrefix,
 			Annotations: map[string]string{
 				volumehelper.VolumeGidAnnotationKey: "777",
 			},
@@ -111,7 +112,7 @@ func getVSpherePersistentVolumeSpec(volumePath string, persistentVolumeReclaimPo
 			Capacity: v1.ResourceList{
 				v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
 			},
-			PersistentVolumeSource: pvConfig.pvSource,
+			PersistentVolumeSource: pvConfig.PVSource,
 			AccessModes: []v1.PersistentVolumeAccessMode{
 				v1.ReadWriteOnce,
 			},