Updating CSI e2e test to create CSI CRDs; storing CRD spec in a common location

pkg/volume/csi/nodeinfomanager/nodeinfomanager.go

@@ -353,7 +353,7 @@ func (nim *nodeInfoManager) createNodeInfoObject(
 		return fmt.Errorf("error getting CSI client")
 	}
 
-	var topologyKeys []string
+	topologyKeys := []string{} // must be an empty slice instead of nil to satisfy CRD OpenAPI Schema validation
 	if topology != nil {
 		for k := range topology.Segments {
 			topologyKeys = append(topologyKeys, k)

staging/src/BUILD

@@ -206,6 +206,7 @@ filegroup(
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:all-srcs",
         "//staging/src/k8s.io/csi-api/pkg/client/informers/externalversions:all-srcs",
         "//staging/src/k8s.io/csi-api/pkg/client/listers/csi/v1alpha1:all-srcs",
+        "//staging/src/k8s.io/csi-api/pkg/crd:all-srcs",
         "//staging/src/k8s.io/kube-aggregator:all-srcs",
         "//staging/src/k8s.io/kube-controller-manager/config/v1alpha1:all-srcs",
         "//staging/src/k8s.io/kube-proxy/config/v1alpha1:all-srcs",

staging/src/k8s.io/csi-api/Godeps/Godeps.json

@@ -291,6 +291,18 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
+			"ImportPath": "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apimachinery/pkg/api/errors",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/csi-api/pkg/crd/BUILD

+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["crd.go"],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd",
+    importpath = "k8s.io/csi-api/pkg/crd",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["crd_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = [
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/github.com/ghodss/yaml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

staging/src/k8s.io/csi-api/pkg/crd/crd.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	"reflect"
+)
+
+// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in
+// release 1.12 in order to aid manual CRD installation. This installation will be automated in
+// subsequent releases and as a result this package will be removed.
+
+// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object.
+func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition {
+	return &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName,
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   csiapiv1alpha1.GroupName,
+			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
+			Scope:   apiextensionsv1beta1.ClusterScoped,
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"spec": {
+							Description: "Specification of the CSI Driver.",
+							Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+								"attachRequired": {
+									Description: "Indicates this CSI volume driver requires an attach operation," +
+										" and that Kubernetes should call attach and wait for any attach operation to" +
+										" complete before proceeding to mount.",
+									Type: "boolean",
+								},
+								"podInfoOnMountVersion": {
+									Description: "Indicates this CSI volume driver requires additional pod" +
+										" information (like podName, podUID, etc.) during mount operations.",
+									Type: "string",
+								},
+							},
+						},
+					},
+				},
+			},
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: csiapiv1alpha1.CsiDriverResourcePlural,
+				Kind:   reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(),
+			},
+		},
+	}
+}
+
+// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object.
+func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition {
+	return &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName,
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   csiapiv1alpha1.GroupName,
+			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
+			Scope:   apiextensionsv1beta1.ClusterScoped,
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"csiDrivers": {
+							Description: "List of CSI drivers running on the node and their properties.",
+							Type:        "array",
+							Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
+								Schema: &apiextensionsv1beta1.JSONSchemaProps{
+									Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+										"driver": {
+											Description: "The CSI driver that this object refers to.",
+											Type:        "string",
+										},
+										"nodeID": {
+											Description: "The node from the driver point of view.",
+											Type:        "string",
+										},
+										"topologyKeys": {
+											Description: "List of keys supported by the driver.",
+											Type:        "array",
+											Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
+												Schema: &apiextensionsv1beta1.JSONSchemaProps{
+													Type: "string",
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural,
+				Kind:   reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(),
+			},
+		},
+	}
+}

staging/src/k8s.io/csi-api/pkg/crd/crd_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/ghodss/yaml"
+	"io/ioutil"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/csi-api/pkg/crd"
+	"os"
+)
+
+func TestBootstrapCRDs(t *testing.T) {
+	testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml")
+	testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml")
+}
+
+func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) {
+	filename := filepath.Join("testdata", fixtureFilename)
+	expectedYAML, err := ioutil.ReadFile(filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	yamlData, err := yaml.JSONToYAML(jsonData)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(yamlData) != string(expectedYAML) {
+		t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename)
+
+		const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
+		if os.Getenv(updateEnvVar) == "true" {
+			if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil {
+				t.Logf("Updated data in %s", filename)
+				t.Logf("Verify the diff, commit changes, and rerun the tests")
+			} else {
+				t.Logf("Could not update data in %s: %v", filename, err)
+			}
+		} else {
+			t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML)))
+			t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar)
+		}
+	}
+}

staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml

+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: csidrivers.csi.storage.k8s.io
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSIDriver
+    plural: csidrivers
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        spec:
+          description: Specification of the CSI Driver.
+          properties:
+            attachRequired:
+              description: Indicates this CSI volume driver requires an attach operation,
+                and that Kubernetes should call attach and wait for any attach operation
+                to complete before proceeding to mount.
+              type: boolean
+            podInfoOnMountVersion:
+              description: Indicates this CSI volume driver requires additional pod
+                information (like podName, podUID, etc.) during mount operations.
+              type: string
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml

+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: csinodeinfos.csi.storage.k8s.io
+spec:
+  group: csi.storage.k8s.io
+  names:
+    kind: CSINodeInfo
+    plural: csinodeinfos
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      properties:
+        csiDrivers:
+          description: List of CSI drivers running on the node and their properties.
+          items:
+            properties:
+              driver:
+                description: The CSI driver that this object refers to.
+                type: string
+              nodeID:
+                description: The node from the driver point of view.
+                type: string
+              topologyKeys:
+                description: List of keys supported by the driver.
+                items:
+                  type: string
+                type: array
+          type: array
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: null
+  storedVersions: null

test/e2e/framework/framework.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"k8s.io/api/core/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -67,6 +68,7 @@ type Framework struct {
 
 	ClientSet                        clientset.Interface
 	KubemarkExternalClusterClientSet clientset.Interface
+	APIExtensionsClientSet           apiextensionsclient.Interface
 	CSIClientSet                     csi.Interface
 
 	InternalClientset *internalclientset.Clientset
@@ -176,6 +178,8 @@ func (f *Framework) BeforeEach() {
 		}
 		f.ClientSet, err = clientset.NewForConfig(config)
 		Expect(err).NotTo(HaveOccurred())
+		f.APIExtensionsClientSet, err = apiextensionsclient.NewForConfig(config)
+		Expect(err).NotTo(HaveOccurred())
 		f.InternalClientset, err = internalclientset.NewForConfig(config)
 		Expect(err).NotTo(HaveOccurred())
 		f.AggregatorClient, err = aggregatorclient.NewForConfig(config)

test/e2e/storage/BUILD

@@ -44,6 +44,8 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1:go_default_library",
         "//staging/src/k8s.io/api/storage/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
@@ -64,6 +66,7 @@ go_library(
         "//staging/src/k8s.io/client-go/rest:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
         "//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
+        "//staging/src/k8s.io/csi-api/pkg/crd:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/e2e/framework/metrics:go_default_library",
         "//test/e2e/generated:go_default_library",

test/e2e/storage/csi_objects.go

@@ -35,6 +35,10 @@ import (
 	"k8s.io/kubernetes/test/e2e/manifest"
 
 	. "github.com/onsi/ginkgo"
+
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	csicrd "k8s.io/csi-api/pkg/crd"
 )
 
 var csiImageVersions = map[string]string{
@@ -427,3 +431,26 @@ func deployGCEPDCSIDriver(
 	framework.ExpectNoError(err, "Failed to create DaemonSet: %v", nodeds.Name)
 
 }
+
+func createCSICRDs(c apiextensionsclient.Interface) {
+	By("Creating CSI CRDs")
+	crds := []*apiextensionsv1beta1.CustomResourceDefinition{
+		csicrd.CSIDriverCRD(),
+		csicrd.CSINodeInfoCRD(),
+	}
+
+	for _, crd := range crds {
+		_, err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+		framework.ExpectNoError(err, "Failed to create CSI CRD %q: %v", crd.Name, err)
+	}
+}
+
+func deleteCSICRDs(c apiextensionsclient.Interface) {
+	By("Deleting CSI CRDs")
+	csiDriverCRDName := csicrd.CSIDriverCRD().Name
+	csiNodeInfoCRDName := csicrd.CSINodeInfoCRD().Name
+	err := c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiDriverCRDName, &metav1.DeleteOptions{})
+	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiDriverCRDName, err)
+	err = c.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(csiNodeInfoCRDName, &metav1.DeleteOptions{})
+	framework.ExpectNoError(err, "Failed to delete CSI CRD %q: %v", csiNodeInfoCRDName, err)
+}

test/e2e/storage/csi_volumes.go

@@ -24,10 +24,11 @@ import (
 	"k8s.io/api/core/v1"
 
 	storagev1 "k8s.io/api/storage/v1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	csi "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
+	csiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
 	csiclient "k8s.io/csi-api/pkg/client/clientset/versioned"
 	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -58,19 +59,21 @@ var csiTestDrivers = map[string]func(f *framework.Framework, config framework.Vo
 	"[Feature: GCE PD CSI Plugin] gcePD": initCSIgcePD,
 }
 
-var _ = utils.SIGDescribe("CSI Volumes", func() {
+var _ = utils.SIGDescribe("[Serial] CSI Volumes", func() {
 	f := framework.NewDefaultFramework("csi-mock-plugin")
 
 	var (
-		cs     clientset.Interface
-		csics  csiclient.Interface
-		ns     *v1.Namespace
-		node   v1.Node
-		config framework.VolumeTestConfig
+		cs        clientset.Interface
+		crdclient apiextensionsclient.Interface
+		csics     csiclient.Interface
+		ns        *v1.Namespace
+		node      v1.Node
+		config    framework.VolumeTestConfig
 	)
 
 	BeforeEach(func() {
 		cs = f.ClientSet
+		crdclient = f.APIExtensionsClientSet
 		csics = f.CSIClientSet
 		ns = f.Namespace
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
@@ -83,13 +86,18 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 			WaitForCompletion: true,
 		}
 		csiDriverRegistrarClusterRole(config)
+		createCSICRDs(crdclient)
+	})
+
+	AfterEach(func() {
+		deleteCSICRDs(crdclient)
 	})
 
 	for driverName, initCSIDriver := range csiTestDrivers {
 		curDriverName := driverName
 		curInitCSIDriver := initCSIDriver
 
-		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s", curDriverName), func() {
+		Context(fmt.Sprintf("CSI plugin test using CSI driver: %s [Serial]", curDriverName), func() {
 			var (
 				driver csiTestDriver
 			)
@@ -205,13 +213,13 @@ var _ = utils.SIGDescribe("CSI Volumes", func() {
 	})
 })
 
-func createCSIDriver(csics csiclient.Interface, attachable bool) *csi.CSIDriver {
+func createCSIDriver(csics csiclient.Interface, attachable bool) *csiv1alpha1.CSIDriver {
 	By("Creating CSIDriver instance")
-	driver := &csi.CSIDriver{
+	driver := &csiv1alpha1.CSIDriver{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "csi-hostpath",
 		},
-		Spec: csi.CSIDriverSpec{
+		Spec: csiv1alpha1.CSIDriverSpec{
 			AttachRequired: &attachable,
 		},
 	}