Refactor flex pv to allow secret namespace

pkg/api/persistentvolume/util.go

@@ -62,8 +62,16 @@ func VisitPVSecretNames(pv *api.PersistentVolume, visitor Visitor) bool {
 			}
 		}
 	case source.FlexVolume != nil:
-		if source.FlexVolume.SecretRef != nil && !visitor(getClaimRefNamespace(pv), source.FlexVolume.SecretRef.Name) {
-			return false
+		if source.FlexVolume.SecretRef != nil {
+			// previously persisted PV objects use claimRef namespace
+			ns := getClaimRefNamespace(pv)
+			if len(source.FlexVolume.SecretRef.Namespace) > 0 {
+				// use the secret namespace if namespace is set
+				ns = source.FlexVolume.SecretRef.Namespace
+			}
+			if !visitor(ns, source.FlexVolume.SecretRef.Name) {
+				return false
+			}
 		}
 	case source.RBD != nil:
 		if source.RBD.SecretRef != nil {

pkg/api/persistentvolume/util_test.go

@@ -61,8 +61,15 @@ func TestPVSecrets(t *testing.T) {
 		{Spec: api.PersistentVolumeSpec{
 			ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"},
 			PersistentVolumeSource: api.PersistentVolumeSource{
-				FlexVolume: &api.FlexVolumeSource{
-					SecretRef: &api.LocalObjectReference{
+				FlexVolume: &api.FlexPersistentVolumeSource{
+					SecretRef: &api.SecretReference{
+						Name:      "Spec.PersistentVolumeSource.FlexVolume.SecretRef",
+						Namespace: "flexns"}}}}},
+		{Spec: api.PersistentVolumeSpec{
+			ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"},
+			PersistentVolumeSource: api.PersistentVolumeSource{
+				FlexVolume: &api.FlexPersistentVolumeSource{
+					SecretRef: &api.SecretReference{
 						Name: "Spec.PersistentVolumeSource.FlexVolume.SecretRef"}}}}},
 		{Spec: api.PersistentVolumeSpec{
 			ClaimRef: &api.ObjectReference{Namespace: "claimrefns", Name: "claimrefname"},
@@ -160,15 +167,22 @@ func TestPVSecrets(t *testing.T) {
 	expectedNamespacedNames := sets.NewString(
 		"claimrefns/Spec.PersistentVolumeSource.AzureFile.SecretName",
 		"Spec.PersistentVolumeSource.AzureFile.SecretNamespace/Spec.PersistentVolumeSource.AzureFile.SecretName",
+
 		"claimrefns/Spec.PersistentVolumeSource.CephFS.SecretRef",
 		"cephfs/Spec.PersistentVolumeSource.CephFS.SecretRef",
+
 		"claimrefns/Spec.PersistentVolumeSource.FlexVolume.SecretRef",
+		"flexns/Spec.PersistentVolumeSource.FlexVolume.SecretRef",
+
 		"claimrefns/Spec.PersistentVolumeSource.RBD.SecretRef",
 		"rbdns/Spec.PersistentVolumeSource.RBD.SecretRef",
+
 		"claimrefns/Spec.PersistentVolumeSource.ScaleIO.SecretRef",
 		"scaleions/Spec.PersistentVolumeSource.ScaleIO.SecretRef",
+
 		"claimrefns/Spec.PersistentVolumeSource.ISCSI.SecretRef",
 		"iscsi/Spec.PersistentVolumeSource.ISCSI.SecretRef",
+
 		"storageosns/Spec.PersistentVolumeSource.StorageOS.SecretRef",
 	)
 	if missingNames := expectedNamespacedNames.Difference(extractedNamesWithNamespace); len(missingNames) > 0 {

pkg/apis/core/types.go

@@ -354,7 +354,7 @@ type PersistentVolumeSource struct {
 	// FlexVolume represents a generic volume resource that is
 	// provisioned/attached using an exec based plugin.
 	// +optional
-	FlexVolume *FlexVolumeSource
+	FlexVolume *FlexPersistentVolumeSource
 	// Cinder represents a cinder volume attached and mounted on kubelets host machine
 	// +optional
 	Cinder *CinderVolumeSource
@@ -867,6 +867,32 @@ type FCVolumeSource struct {
 	WWIDs []string
 }
 
+// FlexPersistentVolumeSource represents a generic persistent volume resource that is
+// provisioned/attached using an exec based plugin.
+type FlexPersistentVolumeSource struct {
+	// Driver is the name of the driver to use for this volume.
+	Driver string
+	// Filesystem type to mount.
+	// Must be a filesystem type supported by the host operating system.
+	// Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script.
+	// +optional
+	FSType string
+	// Optional: SecretRef is reference to the secret object containing
+	// sensitive information to pass to the plugin scripts. This may be
+	// empty if no secret object is specified. If the secret object
+	// contains more than one secret, all secrets are passed to the plugin
+	// scripts.
+	// +optional
+	SecretRef *SecretReference
+	// Optional: Defaults to false (read/write). ReadOnly here will force
+	// the ReadOnly setting in VolumeMounts.
+	// +optional
+	ReadOnly bool
+	// Optional: Extra driver options if any.
+	// +optional
+	Options map[string]string
+}
+
 // FlexVolume represents a generic volume resource that is
 // provisioned/attached using an exec based plugin.
 type FlexVolumeSource struct {

pkg/apis/core/validation/validation.go

@@ -1257,6 +1257,27 @@ func validateFlexVolumeSource(fv *core.FlexVolumeSource, fldPath *field.Path) fi
 	return allErrs
 }
 
+func validateFlexPersistentVolumeSource(fv *core.FlexPersistentVolumeSource, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if len(fv.Driver) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("driver"), ""))
+	}
+
+	// Make sure user-specified options don't use kubernetes namespaces
+	for k := range fv.Options {
+		namespace := k
+		if parts := strings.SplitN(k, "/", 2); len(parts) == 2 {
+			namespace = parts[0]
+		}
+		normalized := "." + strings.ToLower(namespace)
+		if strings.HasSuffix(normalized, ".kubernetes.io") || strings.HasSuffix(normalized, ".k8s.io") {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("options").Key(k), k, "kubernetes.io and k8s.io namespaces are reserved"))
+		}
+	}
+
+	return allErrs
+}
+
 func validateAzureFile(azure *core.AzureFileVolumeSource, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 	if azure.SecretName == "" {
@@ -1588,7 +1609,7 @@ func ValidatePersistentVolume(pv *core.PersistentVolume) field.ErrorList {
 	}
 	if pv.Spec.FlexVolume != nil {
 		numVolumes++
-		allErrs = append(allErrs, validateFlexVolumeSource(pv.Spec.FlexVolume, specPath.Child("flexVolume"))...)
+		allErrs = append(allErrs, validateFlexPersistentVolumeSource(pv.Spec.FlexVolume, specPath.Child("flexVolume"))...)
 	}
 	if pv.Spec.AzureFile != nil {
 		if numVolumes > 0 {

pkg/apis/core/validation/validation_test.go

@@ -469,7 +469,7 @@ func TestValidatePersistentVolumeSourceUpdate(t *testing.T) {
 	validPvSourceNoUpdate := validVolume.DeepCopy()
 	invalidPvSourceUpdateType := validVolume.DeepCopy()
 	invalidPvSourceUpdateType.Spec.PersistentVolumeSource = core.PersistentVolumeSource{
-		FlexVolume: &core.FlexVolumeSource{
+		FlexVolume: &core.FlexPersistentVolumeSource{
 			Driver: "kubernetes.io/blue",
 			FSType: "ext4",
 		},

pkg/printers/internalversion/describe.go

@@ -1076,6 +1076,16 @@ func printAzureFilePersistentVolumeSource(azureFile *api.AzureFilePersistentVolu
 		azureFile.SecretName, ns, azureFile.ShareName, azureFile.ReadOnly)
 }
 
+func printFlexPersistentVolumeSource(flex *api.FlexPersistentVolumeSource, w PrefixWriter) {
+	w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+
+		"    Driver:\t%v\n"+
+		"    FSType:\t%v\n"+
+		"    SecretRef:\t%v\n"+
+		"    ReadOnly:\t%v\n",
+		"    Options:\t%v\n",
+		flex.Driver, flex.FSType, flex.SecretRef, flex.ReadOnly, flex.Options)
+}
+
 func printFlexVolumeSource(flex *api.FlexVolumeSource, w PrefixWriter) {
 	w.Write(LEVEL_2, "Type:\tFlexVolume (a generic volume resource that is provisioned/attached using an exec based plugin)\n"+
 		"    Driver:\t%v\n"+
@@ -1184,7 +1194,7 @@ func describePersistentVolume(pv *api.PersistentVolume, events *api.EventList) (
 		case pv.Spec.AzureFile != nil:
 			printAzureFilePersistentVolumeSource(pv.Spec.AzureFile, w)
 		case pv.Spec.FlexVolume != nil:
-			printFlexVolumeSource(pv.Spec.FlexVolume, w)
+			printFlexPersistentVolumeSource(pv.Spec.FlexVolume, w)
 		case pv.Spec.Flocker != nil:
 			printFlockerVolumeSource(pv.Spec.Flocker, w)
 		case pv.Spec.CSI != nil:

pkg/volume/flexvolume/attacher-defaults.go

@@ -48,7 +48,16 @@ func (a *attacherDefaults) GetDeviceMountPath(spec *volume.Spec, mountsDir strin
 // MountDevice is part of the volume.Attacher interface
 func (a *attacherDefaults) MountDevice(spec *volume.Spec, devicePath string, deviceMountPath string, mounter mount.Interface) error {
 	glog.Warning(logPrefix(a.plugin.flexVolumePlugin), "using default MountDevice for volume ", spec.Name, ", device ", devicePath, ", deviceMountPath ", deviceMountPath)
-	volSource, readOnly := getVolumeSource(spec)
+
+	volSourceFSType, err := getFSType(spec)
+	if err != nil {
+		return err
+	}
+
+	readOnly, err := getReadOnly(spec)
+	if err != nil {
+		return err
+	}
 
 	options := make([]string, 0)
 
@@ -60,5 +69,5 @@ func (a *attacherDefaults) MountDevice(spec *volume.Spec, devicePath string, dev
 
 	diskMounter := &mount.SafeFormatAndMount{Interface: mounter, Exec: a.plugin.host.GetExec(a.plugin.GetPluginName())}
 
-	return diskMounter.FormatAndMount(devicePath, deviceMountPath, volSource.FSType, options)
+	return diskMounter.FormatAndMount(devicePath, deviceMountPath, volSourceFSType, options)
 }

pkg/volume/flexvolume/common_test.go

@@ -119,7 +119,7 @@ func fakePersistentVolumeSpec() *volume.Spec {
 		},
 		Spec: v1.PersistentVolumeSpec{
 			PersistentVolumeSource: v1.PersistentVolumeSource{
-				FlexVolume: &v1.FlexVolumeSource{
+				FlexVolume: &v1.FlexPersistentVolumeSource{
 					Driver:   "kubernetes.io/fakeAttacher",
 					ReadOnly: false,
 				},

pkg/volume/flexvolume/driver-call.go

@@ -162,10 +162,25 @@ func (dc *DriverCall) Run() (*DriverStatus, error) {
 type OptionsForDriver map[string]string
 
 func NewOptionsForDriver(spec *volume.Spec, host volume.VolumeHost, extraOptions map[string]string) (OptionsForDriver, error) {
-	volSource, readOnly := getVolumeSource(spec)
+
+	volSourceFSType, err := getFSType(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	readOnly, err := getReadOnly(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	volSourceOptions, err := getOptions(spec)
+	if err != nil {
+		return nil, err
+	}
+
 	options := map[string]string{}
 
-	options[optionFSType] = volSource.FSType
+	options[optionFSType] = volSourceFSType
 
 	if readOnly {
 		options[optionReadWrite] = "ro"
@@ -179,7 +194,7 @@ func NewOptionsForDriver(spec *volume.Spec, host volume.VolumeHost, extraOptions
 		options[key] = value
 	}
 
-	for key, value := range volSource.Options {
+	for key, value := range volSourceOptions {
 		options[key] = value
 	}
 

pkg/volume/flexvolume/flexvolume_test.go

@@ -185,7 +185,7 @@ func TestCanSupport(t *testing.T) {
 	if !plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{FlexVolume: &v1.FlexVolumeSource{Driver: "kubernetes.io/fakeAttacher"}}}}) {
 		t.Errorf("Expected true")
 	}
-	if !plugin.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{FlexVolume: &v1.FlexVolumeSource{Driver: "kubernetes.io/fakeAttacher"}}}}}) {
+	if !plugin.CanSupport(&volume.Spec{PersistentVolume: &v1.PersistentVolume{Spec: v1.PersistentVolumeSpec{PersistentVolumeSource: v1.PersistentVolumeSource{FlexVolume: &v1.FlexPersistentVolumeSource{Driver: "kubernetes.io/fakeAttacher"}}}}}) {
 		t.Errorf("Expected true")
 	}
 	if plugin.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) {

pkg/volume/flexvolume/plugin.go

@@ -132,8 +132,11 @@ func (plugin *flexVolumePlugin) GetVolumeName(spec *volume.Spec) (string, error)
 
 // CanSupport is part of the volume.VolumePlugin interface.
 func (plugin *flexVolumePlugin) CanSupport(spec *volume.Spec) bool {
-	source, _ := getVolumeSource(spec)
-	return (source != nil) && (source.Driver == plugin.driverName)
+	sourceDriver, err := getDriver(spec)
+	if err != nil {
+		return false
+	}
+	return sourceDriver == plugin.driverName
 }
 
 // RequiresRemount is part of the volume.VolumePlugin interface.
@@ -156,10 +159,19 @@ func (plugin *flexVolumePlugin) NewMounter(spec *volume.Spec, pod *api.Pod, _ vo
 
 // newMounterInternal is the internal mounter routine to build the volume.
 func (plugin *flexVolumePlugin) newMounterInternal(spec *volume.Spec, pod *api.Pod, mounter mount.Interface, runner exec.Interface) (volume.Mounter, error) {
-	source, readOnly := getVolumeSource(spec)
+	sourceDriver, err := getDriver(spec)
+	if err != nil {
+		return nil, err
+	}
+
+	readOnly, err := getReadOnly(spec)
+	if err != nil {
+		return nil, err
+	}
+
 	return &flexVolumeMounter{
 		flexVolume: &flexVolume{
-			driverName:            source.Driver,
+			driverName:            sourceDriver,
 			execPath:              plugin.getExecutable(),
 			mounter:               mounter,
 			plugin:                plugin,

pkg/volume/flexvolume/util.go

@@ -22,15 +22,18 @@ import (
 	"os"
 
 	"github.com/golang/glog"
-	api "k8s.io/api/core/v1"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util"
 )
 
 func addSecretsToOptions(options map[string]string, spec *volume.Spec, namespace string, driverName string, host volume.VolumeHost) error {
-	fv, _ := getVolumeSource(spec)
-	if fv.SecretRef == nil {
+	secretName, secretNamespace, err := getSecretNameAndNamespace(spec, namespace)
+	if err != nil {
+		return err
+	}
+
+	if len(secretName) == 0 || len(secretNamespace) == 0 {
 		return nil
 	}
 
@@ -39,9 +42,9 @@ func addSecretsToOptions(options map[string]string, spec *volume.Spec, namespace
 		return fmt.Errorf("Cannot get kube client")
 	}
 
-	secrets, err := util.GetSecretForPV(namespace, fv.SecretRef.Name, driverName, host.GetKubeClient())
+	secrets, err := util.GetSecretForPV(secretNamespace, secretName, driverName, host.GetKubeClient())
 	if err != nil {
-		err = fmt.Errorf("Couldn't get secret %v/%v err: %v", namespace, fv.SecretRef.Name, err)
+		err = fmt.Errorf("Couldn't get secret %v/%v err: %v", secretNamespace, secretName, err)
 		return err
 	}
 	for name, data := range secrets {
@@ -52,15 +55,68 @@ func addSecretsToOptions(options map[string]string, spec *volume.Spec, namespace
 	return nil
 }
 
-func getVolumeSource(spec *volume.Spec) (volumeSource *api.FlexVolumeSource, readOnly bool) {
+var notFlexVolume = fmt.Errorf("not a flex volume")
+
+func getDriver(spec *volume.Spec) (string, error) {
+	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
+		return spec.Volume.FlexVolume.Driver, nil
+	}
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
+		return spec.PersistentVolume.Spec.FlexVolume.Driver, nil
+	}
+	return "", notFlexVolume
+}
+
+func getFSType(spec *volume.Spec) (string, error) {
+	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
+		return spec.Volume.FlexVolume.FSType, nil
+	}
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
+		return spec.PersistentVolume.Spec.FlexVolume.FSType, nil
+	}
+	return "", notFlexVolume
+}
+
+func getSecretNameAndNamespace(spec *volume.Spec, podNamespace string) (string, string, error) {
 	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
-		volumeSource = spec.Volume.FlexVolume
-		readOnly = volumeSource.ReadOnly
-	} else if spec.PersistentVolume != nil {
-		volumeSource = spec.PersistentVolume.Spec.FlexVolume
-		readOnly = spec.ReadOnly
+		if spec.Volume.FlexVolume.SecretRef == nil {
+			return "", "", nil
+		}
+		return spec.Volume.FlexVolume.SecretRef.Name, podNamespace, nil
+	}
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
+		if spec.PersistentVolume.Spec.FlexVolume.SecretRef == nil {
+			return "", "", nil
+		}
+		secretName := spec.PersistentVolume.Spec.FlexVolume.SecretRef.Name
+		secretNamespace := spec.PersistentVolume.Spec.FlexVolume.SecretRef.Namespace
+		if len(secretNamespace) == 0 {
+			secretNamespace = podNamespace
+		}
+		return secretName, secretNamespace, nil
+	}
+	return "", "", notFlexVolume
+}
+
+func getReadOnly(spec *volume.Spec) (bool, error) {
+	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
+		return spec.Volume.FlexVolume.ReadOnly, nil
+	}
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
+		// ReadOnly is specified at the PV level
+		return spec.ReadOnly, nil
+	}
+	return false, notFlexVolume
+}
+
+func getOptions(spec *volume.Spec) (map[string]string, error) {
+	if spec.Volume != nil && spec.Volume.FlexVolume != nil {
+		return spec.Volume.FlexVolume.Options, nil
+	}
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.FlexVolume != nil {
+		return spec.PersistentVolume.Spec.FlexVolume.Options, nil
 	}
-	return
+	return nil, notFlexVolume
 }
 
 func prepareForMount(mounter mount.Interface, deviceMountPath string) (bool, error) {

plugin/pkg/auth/authorizer/node/node_authorizer_test.go

@@ -410,7 +410,7 @@ func generate(opts sampleDataOpts) ([]*api.Pod, []*api.PersistentVolume) {
 			for i := 0; i < opts.uniquePVCsPerPod; i++ {
 				pv := &api.PersistentVolume{}
 				pv.Name = fmt.Sprintf("pv%d-%s-%s", i, pod.Name, pod.Namespace)
-				pv.Spec.FlexVolume = &api.FlexVolumeSource{SecretRef: &api.LocalObjectReference{Name: fmt.Sprintf("secret-%s", pv.Name)}}
+				pv.Spec.FlexVolume = &api.FlexPersistentVolumeSource{SecretRef: &api.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}}
 				pv.Spec.ClaimRef = &api.ObjectReference{Name: fmt.Sprintf("pvc%d-%s", i, pod.Name), Namespace: pod.Namespace}
 				pvs = append(pvs, pv)
 
@@ -421,7 +421,7 @@ func generate(opts sampleDataOpts) ([]*api.Pod, []*api.PersistentVolume) {
 			for i := 0; i < opts.sharedPVCsPerPod; i++ {
 				pv := &api.PersistentVolume{}
 				pv.Name = fmt.Sprintf("pv%d-shared-%s", i, pod.Namespace)
-				pv.Spec.FlexVolume = &api.FlexVolumeSource{SecretRef: &api.LocalObjectReference{Name: fmt.Sprintf("secret-%s", pv.Name)}}
+				pv.Spec.FlexVolume = &api.FlexPersistentVolumeSource{SecretRef: &api.SecretReference{Name: fmt.Sprintf("secret-%s", pv.Name)}}
 				pv.Spec.ClaimRef = &api.ObjectReference{Name: fmt.Sprintf("pvc%d-shared", i), Namespace: pod.Namespace}
 				pvs = append(pvs, pv)
 

staging/src/k8s.io/api/core/v1/types.go

@@ -418,7 +418,7 @@ type PersistentVolumeSource struct {
 	// FlexVolume represents a generic volume resource that is
 	// provisioned/attached using an exec based plugin.
 	// +optional
-	FlexVolume *FlexVolumeSource `json:"flexVolume,omitempty" protobuf:"bytes,12,opt,name=flexVolume"`
+	FlexVolume *FlexPersistentVolumeSource `json:"flexVolume,omitempty" protobuf:"bytes,12,opt,name=flexVolume"`
 	// AzureFile represents an Azure File Service mount on the host and bind mount to the pod.
 	// +optional
 	AzureFile *AzureFilePersistentVolumeSource `json:"azureFile,omitempty" protobuf:"bytes,13,opt,name=azureFile"`
@@ -1081,6 +1081,32 @@ type QuobyteVolumeSource struct {
 	Group string `json:"group,omitempty" protobuf:"bytes,5,opt,name=group"`
 }
 
+// FlexPersistentVolumeSource represents a generic persistent volume resource that is
+// provisioned/attached using an exec based plugin.
+type FlexPersistentVolumeSource struct {
+	// Driver is the name of the driver to use for this volume.
+	Driver string `json:"driver" protobuf:"bytes,1,opt,name=driver"`
+	// Filesystem type to mount.
+	// Must be a filesystem type supported by the host operating system.
+	// Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script.
+	// +optional
+	FSType string `json:"fsType,omitempty" protobuf:"bytes,2,opt,name=fsType"`
+	// Optional: SecretRef is reference to the secret object containing
+	// sensitive information to pass to the plugin scripts. This may be
+	// empty if no secret object is specified. If the secret object
+	// contains more than one secret, all secrets are passed to the plugin
+	// scripts.
+	// +optional
+	SecretRef *SecretReference `json:"secretRef,omitempty" protobuf:"bytes,3,opt,name=secretRef"`
+	// Optional: Defaults to false (read/write). ReadOnly here will force
+	// the ReadOnly setting in VolumeMounts.
+	// +optional
+	ReadOnly bool `json:"readOnly,omitempty" protobuf:"varint,4,opt,name=readOnly"`
+	// Optional: Extra command options if any.
+	// +optional
+	Options map[string]string `json:"options,omitempty" protobuf:"bytes,5,rep,name=options"`
+}
+
 // FlexVolume represents a generic volume resource that is
 // provisioned/attached using an exec based plugin.
 type FlexVolumeSource struct {