Merge pull request #70515 from davidz627/feature/csiNodeInfo

cluster/addons/storage-crds/csinodeinfo.yaml

@@ -13,20 +13,42 @@ spec:
   validation:
     openAPIV3Schema:
       properties:
-        csiDrivers:
-          description: List of CSI drivers running on the node and their properties.
-          items:
-            properties:
-              driver:
-                description: The CSI driver that this object refers to.
-                type: string
-              nodeID:
-                description: The node from the driver point of view.
-                type: string
-              topologyKeys:
-                description: List of keys supported by the driver.
-                items:
-                  type: string
-                type: array
-          type: array
+        spec:
+          description: Specification of CSINodeInfo
+          properties:
+            drivers:
+              description: List of CSI drivers running on the node and their specs.
+              type: array
+              items:
+                properties:
+                  name:
+                    description: The CSI driver that this object refers to.
+                    type: string
+                  nodeID:
+                    description: The node from the driver point of view.
+                    type: string
+                  topologyKeys:
+                    description: List of keys supported by the driver.
+                    items:
+                      type: string
+                    type: array
+        status:
+          description: Status of CSINodeInfo
+          properties:
+            drivers:
+              description: List of CSI drivers running on the node and their statuses.
+              type: array
+              items:
+                properties:
+                  name:
+                    description: The CSI driver that this object refers to.
+                    type: string
+                  available:
+                    description: Whether the CSI driver is installed.
+                    type: boolean
+                  volumePluginMechanism:
+                    description: Indicates to external components the required mechanism
+                      to use for any in-tree plugins replaced by this driver.
+                    pattern: in-tree|csi
+                    type: string
   version: v1alpha1

pkg/volume/csi/csi_plugin.go

@@ -146,7 +146,7 @@ func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string)
 		return err
 	}
 
-	err = nim.AddNodeInfo(pluginName, driverNodeID, maxVolumePerNode, accessibleTopology)
+	err = nim.InstallCSIDriver(pluginName, driverNodeID, maxVolumePerNode, accessibleTopology)
 	if err != nil {
 		glog.Error(log("registrationHandler.RegisterPlugin failed at AddNodeInfo: %v", err))
 		if unregErr := unregisterDriver(pluginName); unregErr != nil {
@@ -188,6 +188,9 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error {
 	csiDrivers = csiDriversStore{driversMap: map[string]csiDriver{}}
 	nim = nodeinfomanager.NewNodeInfoManager(host.GetNodeName(), host)
 
+	// TODO(#70514) Init CSINodeInfo object if the CRD exists and create Driver
+	// objects for migrated drivers.
+
 	return nil
 }
 
@@ -583,8 +586,8 @@ func unregisterDriver(driverName string) error {
 		delete(csiDrivers.driversMap, driverName)
 	}()
 
-	if err := nim.RemoveNodeInfo(driverName); err != nil {
-		glog.Errorf("Error unregistering CSI driver: %v", err)
+	if err := nim.UninstallCSIDriver(driverName); err != nil {
+		glog.Errorf("Error uninstalling CSI driver: %v", err)
 		return err
 	}
 

pkg/volume/csi/nodeinfomanager/BUILD

@@ -49,7 +49,6 @@ go_test(
         "//pkg/volume/testing:go_default_library",
         "//pkg/volume/util:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",

plugin/pkg/admission/noderestriction/admission_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"fmt"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -216,11 +217,22 @@ func Test_nodePlugin_Admit(t *testing.T) {
 			ObjectMeta: metav1.ObjectMeta{
 				Name: "mynode",
 			},
-			CSIDrivers: []csiv1alpha1.CSIDriverInfo{
-				{
-					Driver:       "com.example.csi/mydriver",
-					NodeID:       "com.example.csi/mynode",
-					TopologyKeys: []string{"com.example.csi/zone"},
+			Spec: csiv1alpha1.CSINodeInfoSpec{
+				Drivers: []csiv1alpha1.CSIDriverInfoSpec{
+					{
+						Name:         "com.example.csi/mydriver",
+						NodeID:       "com.example.csi/mynode",
+						TopologyKeys: []string{"com.example.csi/zone"},
+					},
+				},
+			},
+			Status: csiv1alpha1.CSINodeInfoStatus{
+				Drivers: []csiv1alpha1.CSIDriverInfoStatus{
+					{
+						Name:                  "com.example.csi/mydriver",
+						Available:             true,
+						VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree,
+					},
 				},
 			},
 		}
@@ -228,11 +240,22 @@ func Test_nodePlugin_Admit(t *testing.T) {
 			ObjectMeta: metav1.ObjectMeta{
 				Name: "foo",
 			},
-			CSIDrivers: []csiv1alpha1.CSIDriverInfo{
-				{
-					Driver:       "com.example.csi/mydriver",
-					NodeID:       "com.example.csi/foo",
-					TopologyKeys: []string{"com.example.csi/zone"},
+			Spec: csiv1alpha1.CSINodeInfoSpec{
+				Drivers: []csiv1alpha1.CSIDriverInfoSpec{
+					{
+						Name:         "com.example.csi/mydriver",
+						NodeID:       "com.example.csi/foo",
+						TopologyKeys: []string{"com.example.csi/zone"},
+					},
+				},
+			},
+			Status: csiv1alpha1.CSINodeInfoStatus{
+				Drivers: []csiv1alpha1.CSIDriverInfoStatus{
+					{
+						Name:                  "com.example.csi/mydriver",
+						Available:             true,
+						VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree,
+					},
 				},
 			},
 		}

staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/README.md

+# CSINodeInfo and CSIDriverInfo Usage and Lifecycle
+
+CSINodeInfo is an API object representing CSI Information at the Node level.
+CSINodeInfo contains a Spec and a Status, each containing Drivers represent the
+driver spec and status respectively. The CSIDriverInfoSpec represents the
+specification of the driver and is generally not changed, whereas the
+CSIDriverInfoStatus represents the current status of the driver and can be
+updated and reacted to by various components.
+
+## Who creates it and when
+
+CSINodeInfo is created by Kubelet when the first CSI Driver is installed to the
+cluster and it is registered through the Kubelet device registration mechanism.
+
+## Who updates CSIDriverInfo Spec and when
+
+The CSIDriverInfoSpec for a driver is created upon installation of the CSI
+Driver to the cluster and it is registered through the Kubelet device
+registration mechanism. The spec is populated with information about the driver
+through the nodeinfomanager (inside Kubelet) and will remain unchanged from then
+on.
+
+## Who updates Status and when
+
+The CSIDriverInfoStatus for the driver is created upon installation of the CSI
+Driver to the cluster (the same time as the spec) and it is registered through
+the Kubelet device registration mechanism. The Status contains information about
+installation and the required Volume Plugin Mechanism of the driver. When the
+driver is installed/uninstalled through the Kubelet device registration
+mechanism the Available flag is flipped from true/false respectively. The
+migration status will also be updated when the flags for migration are set to
+true/false on the Kubelet for that Driver on that node.
+
+## Consumers of Status and Spec
+
+Currently the only consumer of CSINodeInfo/CSIDriverInfo is the
+csi-external-provisioner. In the future, the Attach Detach Controller (ADC) will
+need to read this object to determine migration status on a per driver per node
+basis. The creation of the CSINodeInfo object could possibly race with the
+Attach/Detach controller as for CSI Migration the controller depend on the
+existence of the API object for the driver but it will not have been created
+yet. The ADC is expected to fail (and retry with exponential backoff) the
+operation if it is expecting the object and it has not yet been created.
+
+## Creation of CSINodeInfo object on Kubelet startup
+
+For CSI Migration Alpha we expect any user who turns on the feature has both
+Kubelet and ADC at a version where the CSINodeInfo's are being created on
+Kubelet startup. We will not promote the feature to Beta (on by default) until
+the CSINodeInfo's are being created on Kubelet startup for a 2 version skew to
+prevent the case where the CSINodeInfo does not exist when the ADC depends on
+it. This prevents the race described above becoming a permanent bad state.

staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/types.go

@@ -18,6 +18,16 @@ package v1alpha1
 
 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+// VolumePluginMechanism is the type of mechanism components should use for volume operations
+type VolumePluginMechanism string
+
+const (
+	// VolumePluginMechanismInTree means components should use the in-tree plugin for volume operations
+	VolumePluginMechanismInTree VolumePluginMechanism = "in-tree"
+	// VolumePluginMechanismCSI means components should use the CSI Driver for volume operations
+	VolumePluginMechanismCSI VolumePluginMechanism = "csi"
+)
+
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -91,18 +101,56 @@ type CSINodeInfo struct {
 	// metadata.name must be the Kubernetes node name.
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 
-	// List of CSI drivers running on the node and their properties.
-	// +patchMergeKey=driver
+	// spec is the specification of CSINodeInfo
+	Spec CSINodeInfoSpec `json:"spec"`
+
+	// status is the current status of CSINodeInfo
+	Status CSINodeInfoStatus `json:"status"`
+}
+
+// CSINodeInfoSpec holds information about the specification of all CSI drivers installed on a node
+type CSINodeInfoSpec struct {
+	// drivers is a list of specifications of CSIDriverInfo
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Drivers []CSIDriverInfoSpec `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+// CSINodeInfoStatus holds information about the status of all CSI drivers installed on a node
+type CSINodeInfoStatus struct {
+	// drivers is a list of the statuses of CSIDriverInfo
+	// +patchMergeKey=name
 	// +patchStrategy=merge
-	CSIDrivers []CSIDriverInfo `json:"csiDrivers" patchStrategy:"merge" patchMergeKey:"driver"`
+	Drivers []CSIDriverInfoStatus `json:"drivers" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+// CSIDriverInfoStatus holds information about the status of one CSI driver installed on a node
+type CSIDriverInfoStatus struct {
+	// name is the name of the CSI driver that this object refers to.
+	// This MUST be the same name returned by the CSI GetPluginName() call for
+	// that driver.
+	Name string `json:"name"`
+
+	// available is a boolean representing whether the driver has been installed
+	// on this node or not.
+	Available bool `json:"available"`
+
+	// volumePluginMechanism announces what mechanism underlies volume plugins.
+	// It is set by Kubelet. It is used by the attach/detach controller, which
+	// needs to know how to perform attachments. The allowed values are:
+	// * "in-tree": the volume operation (e.g., attach/detach) ought to be
+	//   directly performed by the attach/detach controller.
+	// * "csi-plugin": the attach/detach controller ought to request
+	//   the csi plugin to perform the volume operation rather than perform it directly.
+	VolumePluginMechanism VolumePluginMechanism `json:"volumePluginMechanism"`
 }
 
-// CSIDriverInfo contains information about one CSI driver installed on a node.
-type CSIDriverInfo struct {
-	// driver is the name of the CSI driver that this object refers to.
+// CSIDriverInfoSpec holds information about the specification of one CSI driver installed on a node
+type CSIDriverInfoSpec struct {
+	// name is the name of the CSI driver that this object refers to.
 	// This MUST be the same name returned by the CSI GetPluginName() call for
 	// that driver.
-	Driver string `json:"driver"`
+	Name string `json:"name"`
 
 	// nodeID of the node from the driver point of view.
 	// This field enables Kubernetes to communicate with storage systems that do

staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1/zz_generated.deepcopy.go

@@ -52,7 +52,7 @@ func (in *CSIDriver) DeepCopyObject() runtime.Object {
 }
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) {
+func (in *CSIDriverInfoSpec) DeepCopyInto(out *CSIDriverInfoSpec) {
 	*out = *in
 	if in.TopologyKeys != nil {
 		in, out := &in.TopologyKeys, &out.TopologyKeys
@@ -62,12 +62,28 @@ func (in *CSIDriverInfo) DeepCopyInto(out *CSIDriverInfo) {
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfo.
-func (in *CSIDriverInfo) DeepCopy() *CSIDriverInfo {
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoSpec.
+func (in *CSIDriverInfoSpec) DeepCopy() *CSIDriverInfoSpec {
 	if in == nil {
 		return nil
 	}
-	out := new(CSIDriverInfo)
+	out := new(CSIDriverInfoSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSIDriverInfoStatus) DeepCopyInto(out *CSIDriverInfoStatus) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSIDriverInfoStatus.
+func (in *CSIDriverInfoStatus) DeepCopy() *CSIDriverInfoStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(CSIDriverInfoStatus)
 	in.DeepCopyInto(out)
 	return out
 }
@@ -136,13 +152,8 @@ func (in *CSINodeInfo) DeepCopyInto(out *CSINodeInfo) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	if in.CSIDrivers != nil {
-		in, out := &in.CSIDrivers, &out.CSIDrivers
-		*out = make([]CSIDriverInfo, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
 	return
 }
 
@@ -196,3 +207,47 @@ func (in *CSINodeInfoList) DeepCopyObject() runtime.Object {
 	}
 	return nil
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSINodeInfoSpec) DeepCopyInto(out *CSINodeInfoSpec) {
+	*out = *in
+	if in.Drivers != nil {
+		in, out := &in.Drivers, &out.Drivers
+		*out = make([]CSIDriverInfoSpec, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoSpec.
+func (in *CSINodeInfoSpec) DeepCopy() *CSINodeInfoSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(CSINodeInfoSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CSINodeInfoStatus) DeepCopyInto(out *CSINodeInfoStatus) {
+	*out = *in
+	if in.Drivers != nil {
+		in, out := &in.Drivers, &out.Drivers
+		*out = make([]CSIDriverInfoStatus, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CSINodeInfoStatus.
+func (in *CSINodeInfoStatus) DeepCopy() *CSINodeInfoStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(CSINodeInfoStatus)
+	in.DeepCopyInto(out)
+	return out
+}

staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/csinodeinfo.go

@@ -37,6 +37,7 @@ type CSINodeInfosGetter interface {
 type CSINodeInfoInterface interface {
 	Create(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error)
 	Update(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error)
+	UpdateStatus(*v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error)
 	Delete(name string, options *v1.DeleteOptions) error
 	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
 	Get(name string, options v1.GetOptions) (*v1alpha1.CSINodeInfo, error)
@@ -113,6 +114,21 @@ func (c *cSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alph
 	return
 }
 
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+
+func (c *cSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1alpha1.CSINodeInfo, err error) {
+	result = &v1alpha1.CSINodeInfo{}
+	err = c.client.Put().
+		Resource("csinodeinfos").
+		Name(cSINodeInfo.Name).
+		SubResource("status").
+		Body(cSINodeInfo).
+		Do().
+		Into(result)
+	return
+}
+
 // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs.
 func (c *cSINodeInfos) Delete(name string, options *v1.DeleteOptions) error {
 	return c.client.Delete().

staging/src/k8s.io/csi-api/pkg/client/clientset/versioned/typed/csi/v1alpha1/fake/fake_csinodeinfo.go

@@ -94,6 +94,17 @@ func (c *FakeCSINodeInfos) Update(cSINodeInfo *v1alpha1.CSINodeInfo) (result *v1
 	return obj.(*v1alpha1.CSINodeInfo), err
 }
 
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *FakeCSINodeInfos) UpdateStatus(cSINodeInfo *v1alpha1.CSINodeInfo) (*v1alpha1.CSINodeInfo, error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateSubresourceAction(csinodeinfosResource, "status", cSINodeInfo), &v1alpha1.CSINodeInfo{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.CSINodeInfo), err
+}
+
 // Delete takes name of the cSINodeInfo and deletes it. Returns an error if one occurs.
 func (c *FakeCSINodeInfos) Delete(name string, options *v1.DeleteOptions) error {
 	_, err := c.Fake.

staging/src/k8s.io/csi-api/pkg/crd/manifests/csinodeinfo.yaml

@@ -13,20 +13,42 @@ spec:
   validation:
     openAPIV3Schema:
       properties:
-        csiDrivers:
-          description: List of CSI drivers running on the node and their properties.
-          items:
-            properties:
-              driver:
-                description: The CSI driver that this object refers to.
-                type: string
-              nodeID:
-                description: The node from the driver point of view.
-                type: string
-              topologyKeys:
-                description: List of keys supported by the driver.
-                items:
-                  type: string
-                type: array
-          type: array
+        spec:
+          description: Specification of CSINodeInfo
+          properties:
+            drivers:
+              description: List of CSI drivers running on the node and their specs.
+              type: array
+              items:
+                properties:
+                  name:
+                    description: The CSI driver that this object refers to.
+                    type: string
+                  nodeID:
+                    description: The node from the driver point of view.
+                    type: string
+                  topologyKeys:
+                    description: List of keys supported by the driver.
+                    items:
+                      type: string
+                    type: array
+        status:
+          description: Status of CSINodeInfo
+          properties:
+            drivers:
+              description: List of CSI drivers running on the node and their statuses.
+              type: array
+              items:
+                properties:
+                  name:
+                    description: The CSI driver that this object refers to.
+                    type: string
+                  available:
+                    description: Whether the CSI driver is installed.
+                    type: boolean
+                  volumePluginMechanism:
+                    description: Indicates to external components the required mechanism
+                      to use for any in-tree plugins replaced by this driver.
+                    pattern: in-tree|csi
+                    type: string
   version: v1alpha1

test/integration/auth/node_test.go

@@ -420,11 +420,22 @@ func TestNodeAuthorizer(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "node1",
 				},
-				CSIDrivers: []csiv1alpha1.CSIDriverInfo{
-					{
-						Driver:       "com.example.csi/driver1",
-						NodeID:       "com.example.csi/node1",
-						TopologyKeys: []string{"com.example.csi/zone"},
+				Spec: csiv1alpha1.CSINodeInfoSpec{
+					Drivers: []csiv1alpha1.CSIDriverInfoSpec{
+						{
+							Name:         "com.example.csi/driver1",
+							NodeID:       "com.example.csi/node1",
+							TopologyKeys: []string{"com.example.csi/zone"},
+						},
+					},
+				},
+				Status: csiv1alpha1.CSINodeInfoStatus{
+					Drivers: []csiv1alpha1.CSIDriverInfoStatus{
+						{
+							Name:                  "com.example.csi/driver1",
+							Available:             true,
+							VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree,
+						},
 					},
 				},
 			}
@@ -438,13 +449,20 @@ func TestNodeAuthorizer(t *testing.T) {
 			if err != nil {
 				return err
 			}
-			nodeInfo.CSIDrivers = []csiv1alpha1.CSIDriverInfo{
+			nodeInfo.Spec.Drivers = []csiv1alpha1.CSIDriverInfoSpec{
 				{
-					Driver:       "com.example.csi/driver1",
+					Name:         "com.example.csi/driver1",
 					NodeID:       "com.example.csi/node1",
 					TopologyKeys: []string{"com.example.csi/rack"},
 				},
 			}
+			nodeInfo.Status.Drivers = []csiv1alpha1.CSIDriverInfoStatus{
+				{
+					Name:                  "com.example.csi/driver1",
+					Available:             true,
+					VolumePluginMechanism: csiv1alpha1.VolumePluginMechanismInTree,
+				},
+			}
 			_, err = client.CsiV1alpha1().CSINodeInfos().Update(nodeInfo)
 			return err
 		}