Revert "Remove alpha provisioning"

cmd/kube-controller-manager/app/BUILD

@@ -69,6 +69,7 @@ go_library(
         "//pkg/controller/ttl:go_default_library",
         "//pkg/controller/volume/attachdetach:go_default_library",
         "//pkg/controller/volume/persistentvolume:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/quota/install:go_default_library",
         "//pkg/serviceaccount:go_default_library",
         "//pkg/util/configz:go_default_library",
@@ -97,6 +98,7 @@ go_library(
         "//vendor:k8s.io/apimachinery/pkg/util/sets",
         "//vendor:k8s.io/apimachinery/pkg/util/wait",
         "//vendor:k8s.io/apiserver/pkg/server/healthz",
+        "//vendor:k8s.io/apiserver/pkg/util/feature",
         "//vendor:k8s.io/client-go/discovery",
         "//vendor:k8s.io/client-go/dynamic",
         "//vendor:k8s.io/client-go/kubernetes/typed/core/v1",

cmd/kube-controller-manager/app/controllermanager.go

@@ -472,9 +472,14 @@ func StartControllers(controllers map[string]InitFunc, s *options.CMServer, root
 		glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", s.AllocateNodeCIDRs, s.ConfigureCloudRoutes)
 	}
 
+	alphaProvisioner, err := NewAlphaVolumeProvisioner(cloud, s.VolumeConfiguration)
+	if err != nil {
+		return fmt.Errorf("an backward-compatible provisioner could not be created: %v, but one was expected. Provisioning will not work. This functionality is considered an early Alpha version.", err)
+	}
 	params := persistentvolumecontroller.ControllerParameters{
 		KubeClient:                clientBuilder.ClientOrDie("persistent-volume-binder"),
 		SyncPeriod:                s.PVClaimBinderSyncPeriod.Duration,
+		AlphaProvisioner:          alphaProvisioner,
 		VolumePlugins:             ProbeControllerVolumePlugins(cloud, s.VolumeConfiguration),
 		Cloud:                     cloud,
 		ClusterName:               s.ClusterName,

cmd/kube-controller-manager/app/plugins.go

@@ -29,6 +29,7 @@ import (
 
 	// Volume plugins
 	"github.com/golang/glog"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/aws"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
@@ -36,6 +37,7 @@ import (
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/openstack"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/photon"
 	"k8s.io/kubernetes/pkg/cloudprovider/providers/vsphere"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/util/io"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/aws_ebs"
@@ -134,6 +136,36 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
 	return allPlugins
 }
 
+// NewAlphaVolumeProvisioner returns a volume provisioner to use when running in
+// a cloud or development environment. The alpha implementation of provisioning
+// allows 1 implied provisioner per cloud and is here only for compatibility
+// with Kubernetes 1.3
+// TODO: remove in Kubernetes 1.5
+func NewAlphaVolumeProvisioner(cloud cloudprovider.Interface, config componentconfig.VolumeConfiguration) (volume.ProvisionableVolumePlugin, error) {
+	switch {
+	case !utilfeature.DefaultFeatureGate.Enabled(features.DynamicVolumeProvisioning):
+		return nil, nil
+	case cloud == nil && config.EnableHostPathProvisioning:
+		return getProvisionablePluginFromVolumePlugins(host_path.ProbeVolumePlugins(
+			volume.VolumeConfig{
+				ProvisioningEnabled: true,
+			}))
+	case cloud != nil && aws.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(aws_ebs.ProbeVolumePlugins())
+	case cloud != nil && gce.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(gce_pd.ProbeVolumePlugins())
+	case cloud != nil && openstack.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(cinder.ProbeVolumePlugins())
+	case cloud != nil && vsphere.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(vsphere_volume.ProbeVolumePlugins())
+	case cloud != nil && azure.CloudProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(azure_dd.ProbeVolumePlugins())
+	case cloud != nil && photon.ProviderName == cloud.ProviderName():
+		return getProvisionablePluginFromVolumePlugins(photon_pd.ProbeVolumePlugins())
+	}
+	return nil, nil
+}
+
 func getProvisionablePluginFromVolumePlugins(plugins []volume.VolumePlugin) (volume.ProvisionableVolumePlugin, error) {
 	for _, plugin := range plugins {
 		if provisonablePlugin, ok := plugin.(volume.ProvisionableVolumePlugin); ok {

examples/cockroachdb/cockroachdb-statefulset.yaml

@@ -183,6 +183,8 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: datadir
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
     spec:
       accessModes:
         - "ReadWriteOnce"

examples/storage/cassandra/README.md

@@ -254,6 +254,8 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: cassandra-data
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
     spec:
       accessModes: [ "ReadWriteOnce" ]
       resources:

examples/storage/cassandra/cassandra-statefulset.yaml

@@ -77,6 +77,8 @@ spec:
   volumeClaimTemplates:
   - metadata:
       name: cassandra-data
+      annotations:
+        volume.alpha.kubernetes.io/storage-class: anything
     spec:
       accessModes: [ "ReadWriteOnce" ]
       resources:

examples/volumes/nfs/provisioner/nfs-server-gce-pv.yaml

@@ -4,6 +4,8 @@ metadata:
   name: nfs-pv-provisioning-demo
   labels:
     demo: nfs-pv-provisioning
+  annotations:
+    volume.alpha.kubernetes.io/storage-class: any
 spec:
   accessModes: [ "ReadWriteOnce" ]
   resources:

pkg/apis/storage/util/helpers.go

@@ -27,6 +27,10 @@ import (
 const IsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
 const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
 
+// AlphaStorageClassAnnotation represents the previous alpha storage class
+// annotation.  it's no longer used and held here for posterity.
+const AlphaStorageClassAnnotation = "volume.alpha.kubernetes.io/storage-class"
+
 // BetaStorageClassAnnotation represents the beta/previous StorageClass annotation.
 // It's currently still used and will be held for backwards compatibility
 const BetaStorageClassAnnotation = "volume.beta.kubernetes.io/storage-class"
@@ -71,7 +75,7 @@ func GetClaimStorageClass(claim *api.PersistentVolumeClaim) string {
 
 // GetStorageClassAnnotation returns the StorageClass value
 // if the annotation is set, empty string if not
-// TODO: remove Beta when no longer used or needed
+// TODO: remove Alpha and Beta when no longer used or needed
 func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
 	if class, ok := obj.Annotations[StorageClassAnnotation]; ok {
 		return class
@@ -79,13 +83,16 @@ func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
 	if class, ok := obj.Annotations[BetaStorageClassAnnotation]; ok {
 		return class
 	}
+	if class, ok := obj.Annotations[AlphaStorageClassAnnotation]; ok {
+		return class
+	}
 
 	return ""
 }
 
 // HasStorageClassAnnotation returns a boolean
 // if the annotation is set
-// TODO: remove Beta when no longer used or needed
+// TODO: remove Alpha and Beta when no longer used or needed
 func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
 	if _, found := obj.Annotations[StorageClassAnnotation]; found {
 		return found
@@ -93,6 +100,9 @@ func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
 	if _, found := obj.Annotations[BetaStorageClassAnnotation]; found {
 		return found
 	}
+	if _, found := obj.Annotations[AlphaStorageClassAnnotation]; found {
+		return found
+	}
 
 	return false
 

pkg/apis/storage/v1beta1/util/helpers.go

@@ -27,6 +27,10 @@ import (
 const IsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
 const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
 
+// AlphaStorageClassAnnotation represents the previous alpha storage class
+// annotation.  it's no longer used and held here for posterity.
+const AlphaStorageClassAnnotation = "volume.alpha.kubernetes.io/storage-class"
+
 // BetaStorageClassAnnotation represents the beta/previous StorageClass annotation.
 // It's currently still used and will be held for backwards compatibility
 const BetaStorageClassAnnotation = "volume.beta.kubernetes.io/storage-class"
@@ -71,7 +75,7 @@ func GetClaimStorageClass(claim *v1.PersistentVolumeClaim) string {
 
 // GetStorageClassAnnotation returns the StorageClass value
 // if the annotation is set, empty string if not
-// TODO: remove Beta when no longer used or needed
+// TODO: remove Alpha and Beta when no longer used or needed
 func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
 	if class, ok := obj.Annotations[StorageClassAnnotation]; ok {
 		return class
@@ -79,12 +83,16 @@ func GetStorageClassAnnotation(obj metav1.ObjectMeta) string {
 	if class, ok := obj.Annotations[BetaStorageClassAnnotation]; ok {
 		return class
 	}
+	if class, ok := obj.Annotations[AlphaStorageClassAnnotation]; ok {
+		return class
+	}
+
 	return ""
 }
 
 // HasStorageClassAnnotation returns a boolean
 // if the annotation is set
-// TODO: remove Beta when no longer used or needed
+// TODO: remove Alpha and Beta when no longer used or needed
 func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
 	if _, found := obj.Annotations[StorageClassAnnotation]; found {
 		return found
@@ -92,7 +100,12 @@ func HasStorageClassAnnotation(obj metav1.ObjectMeta) bool {
 	if _, found := obj.Annotations[BetaStorageClassAnnotation]; found {
 		return found
 	}
+	if _, found := obj.Annotations[AlphaStorageClassAnnotation]; found {
+		return found
+	}
+
 	return false
+
 }
 
 // IsDefaultAnnotationText returns a pretty Yes/No String if

pkg/controller/volume/persistentvolume/framework_test.go

@@ -834,6 +834,9 @@ func wrapTestWithPluginCalls(expectedRecycleCalls, expectedDeleteCalls []error,
 			provisionCalls: expectedProvisionCalls,
 		}
 		ctrl.volumePluginMgr.InitPlugins([]vol.VolumePlugin{plugin}, ctrl)
+		if expectedProvisionCalls != nil {
+			ctrl.alphaProvisioner = plugin
+		}
 		return toWrap(ctrl, reactor, test)
 	}
 }

pkg/controller/volume/persistentvolume/index.go

@@ -130,6 +130,14 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVol
 				return volume, nil
 			}
 
+			// In Alpha dynamic provisioning, we do now want not match claims
+			// with existing PVs, findByClaim must find only PVs that are
+			// pre-bound to the claim (by dynamic provisioning). TODO: remove in
+			// 1.5
+			if metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation) {
+				continue
+			}
+
 			// filter out:
 			// - volumes bound to another claim
 			// - volumes whose labels don't match the claim's selector, if specified

pkg/controller/volume/persistentvolume/provision_test.go

@@ -95,6 +95,10 @@ var provision2Success = provisionCall{
 	expectedParameters: class2Parameters,
 }
 
+var provisionAlphaSuccess = provisionCall{
+	ret: nil,
+}
+
 // Test single call to syncVolume, expecting provisioning to happen.
 // 1. Fill in the controller with initial data
 // 2. Call the syncVolume *once*.
@@ -356,6 +360,36 @@ func TestProvisionSync(t *testing.T) {
 	runSyncTests(t, tests, storageClasses)
 }
 
+func TestAlphaProvisionSync(t *testing.T) {
+	tests := []controllerTest{
+		{
+			// Provision a volume with alpha annotation
+			"14-1 - successful alpha provisioning",
+			novolumes,
+			newVolumeArray("pvc-uid14-1", "1Gi", "uid14-1", "claim14-1", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
+			newClaimArray("claim14-1", "uid14-1", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation),
+			// Binding will be completed in the next syncClaim
+			newClaimArray("claim14-1", "uid14-1", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
+			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
+		},
+		{
+			// Provision success - there is already a volume available, still
+			// we provision a new one when requested.
+			"14-2 - no alpha provisioning when there is a volume available",
+			newVolumeArray("volume14-2", "1Gi", "", "", v1.VolumePending, v1.PersistentVolumeReclaimRetain),
+			[]*v1.PersistentVolume{
+				newVolume("volume14-2", "1Gi", "", "", v1.VolumePending, v1.PersistentVolumeReclaimRetain),
+				newVolume("pvc-uid14-2", "1Gi", "uid14-2", "claim14-2", v1.VolumeBound, v1.PersistentVolumeReclaimDelete, annBoundByController, annDynamicallyProvisioned),
+			},
+			newClaimArray("claim14-2", "uid14-2", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation),
+			// Binding will be completed in the next syncClaim
+			newClaimArray("claim14-2", "uid14-2", "1Gi", "", v1.ClaimPending, storageutil.AlphaStorageClassAnnotation, annStorageProvisioner),
+			noevents, noerrors, wrapTestWithProvisionCalls([]provisionCall{provisionAlphaSuccess}, testSyncClaim),
+		},
+	}
+	runSyncTests(t, tests, []*storage.StorageClass{})
+}
+
 // Test multiple calls to syncClaim/syncVolume and periodic sync of all
 // volume/claims. The test follows this pattern:
 // 0. Load the controller with initial data.

pkg/controller/volume/persistentvolume/pv_controller.go

@@ -203,6 +203,10 @@ type PersistentVolumeController struct {
 
 	createProvisionedPVRetryCount int
 	createProvisionedPVInterval   time.Duration
+
+	// Provisioner for annAlphaClass.
+	// TODO: remove in 1.5
+	alphaProvisioner vol.ProvisionableVolumePlugin
 }
 
 // syncClaim is the main controller method to decide what to do with a claim.
@@ -239,7 +243,7 @@ func (ctrl *PersistentVolumeController) syncUnboundClaim(claim *v1.PersistentVol
 			glog.V(4).Infof("synchronizing unbound PersistentVolumeClaim[%s]: no volume found", claimToClaimKey(claim))
 			// No PV could be found
 			// OBSERVATION: pvc is "Pending", will retry
-			if storageutil.GetClaimStorageClass(claim) != "" {
+			if storageutil.GetClaimStorageClass(claim) != "" || metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation) {
 				if err = ctrl.provisionClaim(claim); err != nil {
 					return err
 				}
@@ -1336,7 +1340,12 @@ func (ctrl *PersistentVolumeController) provisionClaimOperation(claimObj interfa
 	// Add annBoundByController (used in deleting the volume)
 	metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annBoundByController, "yes")
 	metav1.SetMetaDataAnnotation(&volume.ObjectMeta, annDynamicallyProvisioned, plugin.GetPluginName())
-	metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storageutil.StorageClassAnnotation, claimClass)
+	// For Alpha provisioning behavior, do not add storage.BetaStorageClassAnnotations for volumes created
+	// by storage.AlphaStorageClassAnnotation
+	// TODO: remove this check in 1.5, storage.StorageClassAnnotation will be always non-empty there.
+	if claimClass != "" {
+		metav1.SetMetaDataAnnotation(&volume.ObjectMeta, storageutil.StorageClassAnnotation, claimClass)
+	}
 
 	// Try to create the PV object several times
 	for i := 0; i < ctrl.createProvisionedPVRetryCount; i++ {
@@ -1438,6 +1447,20 @@ func (ctrl *PersistentVolumeController) newRecyclerEventRecorder(volume *v1.Pers
 // It returns either the provisioning plugin or nil when an external
 // provisioner is requested.
 func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.PersistentVolumeClaim) (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) {
+	// TODO: remove this alpha behavior in 1.5
+	alpha := metav1.HasAnnotation(claim.ObjectMeta, storageutil.AlphaStorageClassAnnotation)
+	beta := metav1.HasAnnotation(claim.ObjectMeta, storageutil.BetaStorageClassAnnotation)
+	if alpha && beta {
+		// Both Alpha and Beta annotations are set. Do beta.
+		alpha = false
+		msg := fmt.Sprintf("both %q and %q annotations are present, using %q", storageutil.AlphaStorageClassAnnotation, storageutil.BetaStorageClassAnnotation, storageutil.BetaStorageClassAnnotation)
+		ctrl.eventRecorder.Event(claim, v1.EventTypeNormal, "ProvisioningIgnoreAlpha", msg)
+	}
+	if alpha {
+		// Fall back to fixed list of provisioner plugins
+		return ctrl.findAlphaProvisionablePlugin()
+	}
+
 	// provisionClaim() which leads here is never called with claimClass=="", we
 	// can save some checks.
 	claimClass := storageutil.GetClaimStorageClass(claim)
@@ -1465,6 +1488,28 @@ func (ctrl *PersistentVolumeController) findProvisionablePlugin(claim *v1.Persis
 	return plugin, class, nil
 }
 
+// findAlphaProvisionablePlugin returns a volume plugin compatible with
+// Kubernetes 1.3.
+// TODO: remove in Kubernetes 1.5
+func (ctrl *PersistentVolumeController) findAlphaProvisionablePlugin() (vol.ProvisionableVolumePlugin, *storage.StorageClass, error) {
+	if ctrl.alphaProvisioner == nil {
+		return nil, nil, fmt.Errorf("cannot find volume plugin for alpha provisioning")
+	}
+
+	// Return a dummy StorageClass instance with no parameters
+	storageClass := &storage.StorageClass{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "StorageClass",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "",
+		},
+		Provisioner: ctrl.alphaProvisioner.GetPluginName(),
+	}
+	glog.V(4).Infof("using alpha provisioner %s", ctrl.alphaProvisioner.GetPluginName())
+	return ctrl.alphaProvisioner, storageClass, nil
+}
+
 // findDeletablePlugin finds a deleter plugin for a given volume. It returns
 // either the deleter plugin or nil when an external deleter is requested.
 func (ctrl *PersistentVolumeController) findDeletablePlugin(volume *v1.PersistentVolume) (vol.DeletableVolumePlugin, error) {

pkg/controller/volume/persistentvolume/pv_controller_base.go

@@ -53,6 +53,7 @@ import (
 type ControllerParameters struct {
 	KubeClient                             clientset.Interface
 	SyncPeriod                             time.Duration
+	AlphaProvisioner                       vol.ProvisionableVolumePlugin
 	VolumePlugins                          []vol.VolumePlugin
 	Cloud                                  cloudprovider.Interface
 	ClusterName                            string
@@ -81,11 +82,17 @@ func NewController(p ControllerParameters) *PersistentVolumeController {
 		clusterName:                   p.ClusterName,
 		createProvisionedPVRetryCount: createProvisionedPVRetryCount,
 		createProvisionedPVInterval:   createProvisionedPVInterval,
+		alphaProvisioner:              p.AlphaProvisioner,
 		claimQueue:                    workqueue.NewNamed("claims"),
 		volumeQueue:                   workqueue.NewNamed("volumes"),
 	}
 
 	controller.volumePluginMgr.InitPlugins(p.VolumePlugins, controller)
+	if controller.alphaProvisioner != nil {
+		if err := controller.alphaProvisioner.Init(controller); err != nil {
+			glog.Errorf("PersistentVolumeController: error initializing alpha provisioner plugin: %v", err)
+		}
+	}
 
 	volumeSource := p.VolumeSource
 	if volumeSource == nil {

test/e2e/volume_provisioning.go

@@ -190,26 +190,12 @@ var _ = framework.KubeDescribe("Dynamic provisioning", func() {
 		It("should create and delete alpha persistent volumes [Slow] [Volume]", func() {
 			framework.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere")
 
-			// Check there is a default storage class
-			classes, err := c.Storage().StorageClasses().List(metav1.ListOptions{})
-			Expect(err).NotTo(HaveOccurred())
-			found := false
-			for _, class := range classes.Items {
-				if storageutil.IsDefaultAnnotation(class.ObjectMeta) {
-					found = true
-					break
-				}
-			}
-			if !found {
-				Skip("No default class found")
-			}
-
 			By("creating a claim with an alpha dynamic provisioning annotation")
 			claim := newClaim(ns, "", true)
 			defer func() {
 				c.Core().PersistentVolumeClaims(ns).Delete(claim.Name, nil)
 			}()
-			claim, err = c.Core().PersistentVolumeClaims(ns).Create(claim)
+			claim, err := c.Core().PersistentVolumeClaims(ns).Create(claim)
 			Expect(err).NotTo(HaveOccurred())
 
 			if framework.ProviderIs("vsphere") {
@@ -277,7 +263,7 @@ func newClaim(ns, suffix string, alpha bool) *v1.PersistentVolumeClaim {
 
 	if alpha {
 		claim.Annotations = map[string]string{
-			"volume.alpha.kubernetes.io/storage-class": "anything",
+			storageutil.AlphaStorageClassAnnotation: "",
 		}
 	} else {
 		claim.Annotations = map[string]string{