Merge pull request #71192 from roycaihw/crd-publish-openapi

build/root/Makefile.generated_files

@@ -409,8 +409,10 @@ $(CONVERSION_GEN): $(k8s.io/kubernetes/vendor/k8s.io/code-generator/cmd/conversi
 OPENAPI_BASENAME := $(GENERATED_FILE_PREFIX)openapi
 OPENAPI_FILENAME := $(OPENAPI_BASENAME).go
 OPENAPI_OUTPUT_PKG := pkg/generated/openapi
+CRD_OPENAPI_OUTPUT_PKG := staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi
 BOILERPLATE_FILENAME := vendor/k8s.io/code-generator/hack/boilerplate.go.txt
 REPORT_FILENAME := $(OUT_DIR)/violations.report
+IGNORED_REPORT_FILENAME := $(OUT_DIR)/ignored_violations.report
 KNOWN_VIOLATION_FILENAME := api/api-rules/violation_exceptions.list
 # When UPDATE_API_KNOWN_VIOLATIONS is set to be true, let the generator to write
 # updated API violations to the known API violation exceptions list.
@@ -436,10 +438,11 @@ OPENAPI_DIRS := $(shell                                             \
 )
 
 OPENAPI_OUTFILE := $(OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
+CRD_OPENAPI_OUTFILE := $(CRD_OPENAPI_OUTPUT_PKG)/$(OPENAPI_FILENAME)
 
 # This rule is the user-friendly entrypoint for openapi generation.
 .PHONY: gen_openapi
-gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN)
+gen_openapi: $(OPENAPI_OUTFILE) $(OPENAPI_GEN) $(CRD_OPENAPI_OUTFILE)
 
 # For each dir in OPENAPI_DIRS, this establishes a dependency between the
 # output file and the input files that should trigger a rebuild.
@@ -469,6 +472,17 @@ $(OPENAPI_OUTFILE): $(OPENAPI_GEN) $(KNOWN_VIOLATION_FILENAME)
 	diff $(REPORT_FILENAME) $(KNOWN_VIOLATION_FILENAME) ||                          \
 	(echo -e $(API_RULE_CHECK_FAILURE_MESSAGE); exit 1)
 
+# TODO(roycaihw): move the automation to apiextensions-apiserver
+$(CRD_OPENAPI_OUTFILE): $(OPENAPI_GEN)
+	./hack/run-in-gopath.sh $(OPENAPI_GEN)                                          \
+	    --v $(KUBE_VERBOSE)                                                         \
+	    --logtostderr                                                               \
+	    -i "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/autoscaling/v1"         \
+	    -p $(PRJ_SRC_PATH)/$(CRD_OPENAPI_OUTPUT_PKG)                                \
+	    -O $(OPENAPI_BASENAME)                                                      \
+	    -h $(BOILERPLATE_FILENAME)                                                  \
+	    -r $(IGNORED_REPORT_FILENAME)                                               \
+	    "$$@"
 
 # How to build the generator tool.  The deps for this are defined in
 # the $(GO_PKGDEPS_FILE), above.

pkg/features/kube_features.go

@@ -510,6 +510,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	apiextensionsfeatures.CustomResourceValidation:        {Default: true, PreRelease: utilfeature.Beta},
 	apiextensionsfeatures.CustomResourceSubresources:      {Default: true, PreRelease: utilfeature.Beta},
 	apiextensionsfeatures.CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha},
+	apiextensionsfeatures.CustomResourcePublishOpenAPI:    {Default: false, PreRelease: utilfeature.Alpha},
 
 	// features that enable backwards compatibility but are scheduled to be removed
 	// ...

staging/src/k8s.io/apiextensions-apiserver/BUILD

@@ -51,6 +51,7 @@ filegroup(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:all-srcs",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresource:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition:all-srcs",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration:all-srcs",

staging/src/k8s.io/apiextensions-apiserver/Godeps/Godeps.json

@@ -2427,6 +2427,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/pkg/endpoints",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/pkg/endpoints/discovery",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
@@ -2447,6 +2451,10 @@
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},
 		{
+			"ImportPath": "k8s.io/apiserver/pkg/endpoints/openapi",
+			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+		},
+		{
 			"ImportPath": "k8s.io/apiserver/pkg/endpoints/request",
 			"Rev": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 		},

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/BUILD

@@ -32,6 +32,7 @@ go_library(
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/establish:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/finalizer:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/controller/status:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/crdserverscheme:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/features:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go

@@ -31,7 +31,9 @@ import (
 	internalinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion"
 	"k8s.io/apiextensions-apiserver/pkg/controller/establish"
 	"k8s.io/apiextensions-apiserver/pkg/controller/finalizer"
+	openapicontroller "k8s.io/apiextensions-apiserver/pkg/controller/openapi"
 	"k8s.io/apiextensions-apiserver/pkg/controller/status"
+	apiextensionsfeatures "k8s.io/apiextensions-apiserver/pkg/features"
 	"k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -44,6 +46,7 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/webhook"
 )
 
@@ -198,12 +201,20 @@ func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget)
 		crdClient.Apiextensions(),
 		crdHandler,
 	)
+	var openapiController *openapicontroller.Controller
+	if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) {
+		openapiController = openapicontroller.NewController(s.Informers.Apiextensions().InternalVersion().CustomResourceDefinitions())
+	}
 
 	s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-informers", func(context genericapiserver.PostStartHookContext) error {
 		s.Informers.Start(context.StopCh)
 		return nil
 	})
 	s.GenericAPIServer.AddPostStartHookOrDie("start-apiextensions-controllers", func(context genericapiserver.PostStartHookContext) error {
+		if utilfeature.DefaultFeatureGate.Enabled(apiextensionsfeatures.CustomResourcePublishOpenAPI) {
+			go openapiController.Run(s.GenericAPIServer.StaticOpenAPISpec, s.GenericAPIServer.OpenAPIVersionedService, context.StopCh)
+		}
+
 		go crdController.Run(context.StopCh)
 		go namingController.Run(context.StopCh)
 		go establishingController.Run(context.StopCh)

staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/BUILD

@@ -2,25 +2,55 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = ["conversion.go"],
+    srcs = [
+        "aggregator.go",
+        "builder.go",
+        "controller.go",
+        "conversion.go",
+    ],
     importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/controller/openapi",
     importpath = "k8s.io/apiextensions-apiserver/pkg/controller/openapi",
     visibility = ["//visibility:public"],
     deps = [
+        "//staging/src/k8s.io/api/autoscaling/v1:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
+        "//staging/src/k8s.io/client-go/tools/cache:go_default_library",
+        "//staging/src/k8s.io/client-go/util/workqueue:go_default_library",
+        "//vendor/github.com/emicklei/go-restful:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
+        "//vendor/k8s.io/klog:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/builder:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/handler:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["conversion_test.go"],
+    srcs = [
+        "builder_test.go",
+        "conversion_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/json:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",
         "//vendor/github.com/googleapis/gnostic/OpenAPIv2:go_default_library",

staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/aggregator.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"github.com/go-openapi/spec"
+)
+
+// mergeSpecs aggregates all OpenAPI specs, reusing the metadata of the first, static spec as the basis.
+func mergeSpecs(staticSpec *spec.Swagger, crdSpecs ...*spec.Swagger) *spec.Swagger {
+	// create shallow copy of staticSpec, but replace paths and definitions because we modify them.
+	specToReturn := *staticSpec
+	if staticSpec.Definitions != nil {
+		specToReturn.Definitions = spec.Definitions{}
+		for k, s := range staticSpec.Definitions {
+			specToReturn.Definitions[k] = s
+		}
+	}
+	if staticSpec.Paths != nil {
+		specToReturn.Paths = &spec.Paths{
+			Paths: map[string]spec.PathItem{},
+		}
+		for k, p := range staticSpec.Paths.Paths {
+			specToReturn.Paths.Paths[k] = p
+		}
+	}
+
+	for _, s := range crdSpecs {
+		mergeSpec(&specToReturn, s)
+	}
+
+	return &specToReturn
+}
+
+// mergeSpec copies paths and definitions from source to dest, mutating dest, but not source.
+// We assume that conflicts do not matter.
+func mergeSpec(dest, source *spec.Swagger) {
+	if source == nil || source.Paths == nil {
+		return
+	}
+	if dest.Paths == nil {
+		dest.Paths = &spec.Paths{}
+	}
+	for k, v := range source.Definitions {
+		if dest.Definitions == nil {
+			dest.Definitions = spec.Definitions{}
+		}
+		dest.Definitions[k] = v
+	}
+	for k, v := range source.Paths.Paths {
+		if dest.Paths.Paths == nil {
+			dest.Paths.Paths = map[string]spec.PathItem{}
+		}
+		dest.Paths.Paths[k] = v
+	}
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder_test.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/go-openapi/spec"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+func TestNewBuilder(t *testing.T) {
+	type args struct {
+	}
+	tests := []struct {
+		name string
+
+		schema string
+
+		wantedSchema      string
+		wantedItemsSchema string
+	}{
+		{
+			"nil",
+			"",
+			`{"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`, `{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
+		},
+		{"empty",
+			"{}",
+			`{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
+			`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
+		},
+		{"empty properties",
+			`{"properties":{"spec":{},"status":{}}}`,
+			`{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{},"status":{}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
+			`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
+		},
+		{"filled properties",
+			`{"properties":{"spec":{"type":"object"},"status":{"type":"object"}}}`,
+			`{"properties":{"apiVersion":{},"kind":{},"metadata":{},"spec":{"type":"object"},"status":{"type":"object"}},"x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
+			`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
+		},
+		{"type",
+			`{"type":"object"}`,
+			`{"properties":{"apiVersion":{},"kind":{},"metadata":{}},"type":"object","x-kubernetes-group-version-kind":[{"group":"bar.k8s.io","kind":"Foo","version":"v1"}]}`,
+			`{"$ref":"#/definitions/io.k8s.bar.v1.Foo"}`,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var schema *spec.Schema
+			if len(tt.schema) > 0 {
+				schema = &spec.Schema{}
+				if err := json.Unmarshal([]byte(tt.schema), schema); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			got := newBuilder(&apiextensions.CustomResourceDefinition{
+				Spec: apiextensions.CustomResourceDefinitionSpec{
+					Group:   "bar.k8s.io",
+					Version: "v1",
+					Names: apiextensions.CustomResourceDefinitionNames{
+						Plural:   "foos",
+						Singular: "foo",
+						Kind:     "Foo",
+						ListKind: "FooList",
+					},
+					Scope: apiextensions.NamespaceScoped,
+				},
+			}, "v1", schema)
+
+			var wantedSchema, wantedItemsSchema spec.Schema
+			if err := json.Unmarshal([]byte(tt.wantedSchema), &wantedSchema); err != nil {
+				t.Fatal(err)
+			}
+			if err := json.Unmarshal([]byte(tt.wantedItemsSchema), &wantedItemsSchema); err != nil {
+				t.Fatal(err)
+			}
+
+			gotProperties := properties(got.schema.Properties)
+			wantedProperties := properties(wantedSchema.Properties)
+			if !gotProperties.Equal(wantedProperties) {
+				t.Fatalf("unexpected properties, got: %s, expected: %s", gotProperties.List(), wantedProperties.List())
+			}
+
+			// wipe out TypeMeta/ObjectMeta content, with those many lines of descriptions. We trust that they match here.
+			if _, found := got.schema.Properties["kind"]; found {
+				got.schema.Properties["kind"] = spec.Schema{}
+			}
+			if _, found := got.schema.Properties["apiVersion"]; found {
+				got.schema.Properties["apiVersion"] = spec.Schema{}
+			}
+			if _, found := got.schema.Properties["metadata"]; found {
+				got.schema.Properties["metadata"] = spec.Schema{}
+			}
+
+			if !reflect.DeepEqual(&wantedSchema, got.schema) {
+				t.Errorf("unexpected schema: %s\nwant = %#v\ngot = %#v", diff.ObjectDiff(&wantedSchema, got.schema), &wantedSchema, got.schema)
+			}
+
+			gotListProperties := properties(got.listSchema.Properties)
+			if want := sets.NewString("apiVersion", "kind", "metadata", "items"); !gotListProperties.Equal(want) {
+				t.Fatalf("unexpected list properties, got: %s, expected: %s", gotListProperties.List(), want.List())
+			}
+
+			gotListSchema := got.listSchema.Properties["items"].Items.Schema
+			if !reflect.DeepEqual(&wantedItemsSchema, gotListSchema) {
+				t.Errorf("unexpected list schema: %s (want/got)", diff.ObjectDiff(&wantedItemsSchema, &gotListSchema))
+			}
+		})
+	}
+}
+
+func properties(p map[string]spec.Schema) sets.String {
+	ret := sets.NewString()
+	for k := range p {
+		ret.Insert(k)
+	}
+	return ret
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/controller/openapi/controller.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package openapi
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/go-openapi/spec"
+	"k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+	"k8s.io/kube-openapi/pkg/handler"
+
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
+	listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
+)
+
+// Controller watches CustomResourceDefinitions and publishes validation schema
+type Controller struct {
+	crdLister  listers.CustomResourceDefinitionLister
+	crdsSynced cache.InformerSynced
+
+	// To allow injection for testing.
+	syncFn func(string) error
+
+	queue workqueue.RateLimitingInterface
+
+	staticSpec     *spec.Swagger
+	openAPIService *handler.OpenAPIService
+
+	// specs per version and per CRD name
+	lock     sync.Mutex
+	crdSpecs map[string]map[string]*spec.Swagger
+}
+
+// NewController creates a new Controller with input CustomResourceDefinition informer
+func NewController(crdInformer informers.CustomResourceDefinitionInformer) *Controller {
+	c := &Controller{
+		crdLister:  crdInformer.Lister(),
+		crdsSynced: crdInformer.Informer().HasSynced,
+		queue:      workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_openapi_controller"),
+		crdSpecs:   map[string]map[string]*spec.Swagger{},
+	}
+
+	crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    c.addCustomResourceDefinition,
+		UpdateFunc: c.updateCustomResourceDefinition,
+		DeleteFunc: c.deleteCustomResourceDefinition,
+	})
+
+	c.syncFn = c.sync
+	return c
+}
+
+// Run sets openAPIAggregationManager and starts workers
+func (c *Controller) Run(staticSpec *spec.Swagger, openAPIService *handler.OpenAPIService, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+	defer klog.Infof("Shutting down OpenAPI controller")
+
+	klog.Infof("Starting OpenAPI controller")
+
+	c.staticSpec = staticSpec
+	c.openAPIService = openAPIService
+
+	if !cache.WaitForCacheSync(stopCh, c.crdsSynced) {
+		utilruntime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
+		return
+	}
+
+	// only start one worker thread since its a slow moving API
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+func (c *Controller) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *Controller) processNextWorkItem() bool {
+	key, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(key)
+
+	// log slow aggregations
+	start := time.Now()
+	defer func() {
+		elapsed := time.Since(start)
+		if elapsed > time.Second {
+			klog.Warningf("slow openapi aggregation of %q: %s", key.(string), elapsed)
+		}
+	}()
+
+	err := c.syncFn(key.(string))
+	if err == nil {
+		c.queue.Forget(key)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
+	c.queue.AddRateLimited(key)
+	return true
+}
+
+func (c *Controller) sync(name string) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	crd, err := c.crdLister.Get(name)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
+	}
+
+	// do we have to remove all specs of this CRD?
+	if errors.IsNotFound(err) || !apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) {
+		if _, found := c.crdSpecs[name]; !found {
+			return nil
+		}
+		delete(c.crdSpecs, name)
+		return c.updateSpecLocked()
+	}
+
+	// compute CRD spec and see whether it changed
+	oldSpecs := c.crdSpecs[crd.Name]
+	newSpecs := map[string]*spec.Swagger{}
+	anyChanged := false
+	for _, v := range crd.Spec.Versions {
+		if !v.Served {
+			continue
+		}
+		spec, err := BuildSwagger(crd, v.Name)
+		if err != nil {
+			return err
+		}
+		newSpecs[v.Name] = spec
+		if oldSpecs[v.Name] == nil || !reflect.DeepEqual(oldSpecs[v.Name], spec) {
+			anyChanged = true
+		}
+	}
+	if !anyChanged && len(oldSpecs) == len(newSpecs) {
+		return nil
+	}
+
+	// update specs of this CRD
+	c.crdSpecs[crd.Name] = newSpecs
+	return c.updateSpecLocked()
+}
+
+// updateSpecLocked aggregates all OpenAPI specs and updates openAPIService.
+// It is not thread-safe. The caller is responsible to hold proper lock (Controller.lock).
+func (c *Controller) updateSpecLocked() error {
+	crdSpecs := []*spec.Swagger{}
+	for _, versionSpecs := range c.crdSpecs {
+		for _, s := range versionSpecs {
+			crdSpecs = append(crdSpecs, s)
+		}
+	}
+	return c.openAPIService.UpdateSpec(mergeSpecs(c.staticSpec, crdSpecs...))
+}
+
+func (c *Controller) addCustomResourceDefinition(obj interface{}) {
+	castObj := obj.(*apiextensions.CustomResourceDefinition)
+	klog.V(4).Infof("Adding customresourcedefinition %s", castObj.Name)
+	c.enqueue(castObj)
+}
+
+func (c *Controller) updateCustomResourceDefinition(oldObj, newObj interface{}) {
+	castNewObj := newObj.(*apiextensions.CustomResourceDefinition)
+	klog.V(4).Infof("Updating customresourcedefinition %s", castNewObj.Name)
+	c.enqueue(castNewObj)
+}
+
+func (c *Controller) deleteCustomResourceDefinition(obj interface{}) {
+	castObj, ok := obj.(*apiextensions.CustomResourceDefinition)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v", obj)
+			return
+		}
+		castObj, ok = tombstone.Obj.(*apiextensions.CustomResourceDefinition)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not expected %#v", obj)
+			return
+		}
+	}
+	klog.V(4).Infof("Deleting customresourcedefinition %q", castObj.Name)
+	c.enqueue(castObj)
+}
+
+func (c *Controller) enqueue(obj *apiextensions.CustomResourceDefinition) {
+	c.queue.Add(obj.Name)
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/features/kube_features.go

@@ -34,6 +34,12 @@ const (
 	// CustomResourceValidation is a list of validation methods for CustomResources
 	CustomResourceValidation utilfeature.Feature = "CustomResourceValidation"
 
+	// owner: @roycaihw, @sttts
+	// alpha: v1.14
+	//
+	// CustomResourcePublishOpenAPI enables publishing of CRD OpenAPI specs.
+	CustomResourcePublishOpenAPI utilfeature.Feature = "CustomResourcePublishOpenAPI"
+
 	// owner: @sttts, @nikhita
 	// alpha: v1.10
 	// beta: v1.11
@@ -59,4 +65,5 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
 	CustomResourceValidation:        {Default: true, PreRelease: utilfeature.Beta},
 	CustomResourceSubresources:      {Default: true, PreRelease: utilfeature.Beta},
 	CustomResourceWebhookConversion: {Default: false, PreRelease: utilfeature.Alpha},
+	CustomResourcePublishOpenAPI:    {Default: false, PreRelease: utilfeature.Alpha},
 }

staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/BUILD

+package(default_visibility = ["//visibility:public"])
+
+load("//build:code_generation.bzl", "gen_openapi", "openapi_deps")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+gen_openapi(
+    outs = ["zz_generated.openapi.go"],
+    output_pkg = "k8s.io/apiextensions-apiserver/pkg/generated/openapi",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "zz_generated.openapi.go",
+    ],
+    importmap = "k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/generated/openapi",
+    importpath = "k8s.io/apiextensions-apiserver/pkg/generated/openapi",
+    deps = openapi_deps(),  # keep
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

staging/src/k8s.io/apiextensions-apiserver/pkg/generated/openapi/doc.go

+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// openapi generated definitions.
+package openapi

staging/src/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -611,12 +611,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusOK, "OK", producedObject).
 				Writes(producedObject)
 			if isGetterWithOptions {
-				if err := addObjectParams(ws, route, versionedGetOptions); err != nil {
+				if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
 					return nil, err
 				}
 			}
 			if isExporter {
-				if err := addObjectParams(ws, route, versionedExportOptions); err != nil {
+				if err := AddObjectParams(ws, route, versionedExportOptions); err != nil {
 					return nil, err
 				}
 			}
@@ -638,7 +638,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), allMediaTypes...)...).
 				Returns(http.StatusOK, "OK", versionedList).
 				Writes(versionedList)
-			if err := addObjectParams(ws, route, versionedListOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
 				return nil, err
 			}
 			switch {
@@ -674,7 +674,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusCreated, "Created", producedObject).
 				Reads(defaultVersionedObject).
 				Writes(producedObject)
-			if err := addObjectParams(ws, route, versionedUpdateOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -702,7 +702,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusOK, "OK", producedObject).
 				Reads(metav1.Patch{}).
 				Writes(producedObject)
-			if err := addObjectParams(ws, route, versionedPatchOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -715,7 +715,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				handler = restfulCreateResource(creater, reqScope, admit)
 			}
 			handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)
-			article := getArticleForNoun(kind, " ")
+			article := GetArticleForNoun(kind, " ")
 			doc := "create" + article + kind
 			if isSubresource {
 				doc = "create " + subresource + " of" + article + kind
@@ -732,13 +732,13 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusAccepted, "Accepted", producedObject).
 				Reads(defaultVersionedObject).
 				Writes(producedObject)
-			if err := addObjectParams(ws, route, versionedCreateOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
 		case "DELETE": // Delete a resource.
-			article := getArticleForNoun(kind, " ")
+			article := GetArticleForNoun(kind, " ")
 			doc := "delete" + article + kind
 			if isSubresource {
 				doc = "delete " + subresource + " of" + article + kind
@@ -755,7 +755,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if isGracefulDeleter {
 				route.Reads(versionedDeleterObject)
 				route.ParameterNamed("body").Required(false)
-				if err := addObjectParams(ws, route, versionedDeleteOptions); err != nil {
+				if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
 					return nil, err
 				}
 			}
@@ -774,7 +774,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Writes(versionedStatus).
 				Returns(http.StatusOK, "OK", versionedStatus)
-			if err := addObjectParams(ws, route, versionedListOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -794,7 +794,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Produces(allMediaTypes...).
 				Returns(http.StatusOK, "OK", versionedWatchEvent).
 				Writes(versionedWatchEvent)
-			if err := addObjectParams(ws, route, versionedListOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -814,7 +814,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Produces(allMediaTypes...).
 				Returns(http.StatusOK, "OK", versionedWatchEvent).
 				Writes(versionedWatchEvent)
-			if err := addObjectParams(ws, route, versionedListOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -838,7 +838,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 					Consumes("*/*").
 					Writes(connectProducedObject)
 				if versionedConnectOptions != nil {
-					if err := addObjectParams(ws, route, versionedConnectOptions); err != nil {
+					if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil {
 						return nil, err
 					}
 				}
@@ -907,13 +907,13 @@ func addParams(route *restful.RouteBuilder, params []*restful.Parameter) {
 	}
 }
 
-// addObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route.
+// AddObjectParams converts a runtime.Object into a set of go-restful Param() definitions on the route.
 // The object must be a pointer to a struct; only fields at the top level of the struct that are not
 // themselves interfaces or structs are used; only fields with a json tag that is non empty (the standard
 // Go JSON behavior for omitting a field) become query parameters. The name of the query parameter is
 // the JSON field name. If a description struct tag is set on the field, that description is used on the
 // query parameter. In essence, it converts a standard JSON top level object into a query param schema.
-func addObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error {
+func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error {
 	sv, err := conversion.EnforcePtr(obj)
 	if err != nil {
 		return err
@@ -1015,8 +1015,8 @@ func splitSubresource(path string) (string, string, error) {
 	return resource, subresource, nil
 }
 
-// getArticleForNoun returns the article needed for the given noun.
-func getArticleForNoun(noun string, padding string) string {
+// GetArticleForNoun returns the article needed for the given noun.
+func GetArticleForNoun(noun string, padding string) string {
 	if noun[len(noun)-2:] != "ss" && noun[len(noun)-1:] == "s" {
 		// Plurals don't have an article.
 		// Don't catch words like class

staging/src/k8s.io/apiserver/pkg/endpoints/installer_test.go

@@ -82,7 +82,7 @@ func TestGetArticleForNoun(t *testing.T) {
 		},
 	}
 	for _, tt := range tests {
-		if got := getArticleForNoun(tt.noun, tt.padding); got != tt.want {
+		if got := GetArticleForNoun(tt.noun, tt.padding); got != tt.want {
 			t.Errorf("%q. GetArticleForNoun() = %v, want %v", tt.noun, got, tt.want)
 		}
 	}

staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/aggregator/aggregator.go

@@ -19,6 +19,7 @@ package aggregator
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -40,17 +41,35 @@ type SpecAggregator interface {
 	UpdateAPIServiceSpec(apiServiceName string, spec *spec.Swagger, etag string) error
 	RemoveAPIServiceSpec(apiServiceName string) error
 	GetAPIServiceInfo(apiServiceName string) (handler http.Handler, etag string, exists bool)
+	GetAPIServiceNames() []string
 }
 
 const (
 	aggregatorUser                = "system:aggregator"
 	specDownloadTimeout           = 60 * time.Second
-	localDelegateChainNamePattern = "k8s_internal_local_delegation_chain_%010d"
+	localDelegateChainNamePrefix  = "k8s_internal_local_delegation_chain_"
+	localDelegateChainNamePattern = localDelegateChainNamePrefix + "%010d"
 
 	// A randomly generated UUID to differentiate local and remote eTags.
 	locallyGeneratedEtagPrefix = "\"6E8F849B434D4B98A569B9D7718876E9-"
 )
 
+// IsLocalAPIService returns true for local specs from delegates.
+func IsLocalAPIService(apiServiceName string) bool {
+	return strings.HasPrefix(apiServiceName, localDelegateChainNamePrefix)
+}
+
+// GetAPIServicesName returns the names of APIServices recorded in specAggregator.openAPISpecs.
+// We use this function to pass the names of local APIServices to the controller in this package,
+// so that the controller can periodically sync the OpenAPI spec from delegation API servers.
+func (s *specAggregator) GetAPIServiceNames() []string {
+	names := make([]string, len(s.openAPISpecs))
+	for key := range s.openAPISpecs {
+		names = append(names, key)
+	}
+	return names
+}
+
 // BuildAndRegisterAggregator registered OpenAPI aggregator handler. This function is not thread safe as it only being called on startup.
 func BuildAndRegisterAggregator(downloader *Downloader, delegationTarget server.DelegationTarget, webServices []*restful.WebService,
 	config *common.Config, pathHandler common.PathHandler) (SpecAggregator, error) {
@@ -161,6 +180,7 @@ func (s *specAggregator) buildOpenAPISpec() (specToReturn *spec.Swagger, err err
 			return nil, err
 		}
 	}
+
 	return specToReturn, nil
 }
 
@@ -179,7 +199,14 @@ func (s *specAggregator) updateOpenAPISpec() error {
 // tryUpdatingServiceSpecs tries updating openAPISpecs map with specified specInfo, and keeps the map intact
 // if the update fails.
 func (s *specAggregator) tryUpdatingServiceSpecs(specInfo *openAPISpecInfo) error {
+	if specInfo == nil {
+		return fmt.Errorf("invalid input: specInfo must be non-nil")
+	}
 	orgSpecInfo, exists := s.openAPISpecs[specInfo.apiService.Name]
+	// Skip aggregation if OpenAPI spec didn't change
+	if exists && orgSpecInfo != nil && orgSpecInfo.etag == specInfo.etag {
+		return nil
+	}
 	s.openAPISpecs[specInfo.apiService.Name] = specInfo
 	if err := s.updateOpenAPISpec(); err != nil {
 		if exists {

staging/src/k8s.io/kube-aggregator/pkg/controllers/openapi/controller.go

@@ -30,8 +30,9 @@ import (
 )
 
 const (
-	successfulUpdateDelay   = time.Minute
-	failedUpdateMaxExpDelay = time.Hour
+	successfulUpdateDelay      = time.Minute
+	successfulUpdateDelayLocal = time.Second
+	failedUpdateMaxExpDelay    = time.Hour
 )
 
 type syncAction int
@@ -64,6 +65,11 @@ func NewAggregationController(downloader *aggregator.Downloader, openAPIAggregat
 
 	c.syncHandler = c.sync
 
+	// update each service at least once, also those which are not coming from APIServices, namely local services
+	for _, name := range openAPIAggregationManager.GetAPIServiceNames() {
+		c.queue.AddAfter(name, time.Second)
+	}
+
 	return c
 }
 
@@ -104,8 +110,13 @@ func (c *AggregationController) processNextWorkItem() bool {
 
 	switch action {
 	case syncRequeue:
-		klog.Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
-		c.queue.AddAfter(key, successfulUpdateDelay)
+		if aggregator.IsLocalAPIService(key.(string)) {
+			klog.V(7).Infof("OpenAPI AggregationController: action for local item %s: Requeue after %s.", key, successfulUpdateDelayLocal)
+			c.queue.AddAfter(key, successfulUpdateDelayLocal)
+		} else {
+			klog.V(7).Infof("OpenAPI AggregationController: action for item %s: Requeue.", key)
+			c.queue.AddAfter(key, successfulUpdateDelay)
+		}
 	case syncRequeueRateLimited:
 		klog.Infof("OpenAPI AggregationController: action for item %s: Rate Limited Requeue.", key)
 		c.queue.AddRateLimited(key)

test/e2e/apimachinery/BUILD

@@ -12,6 +12,7 @@ go_library(
         "certs.go",
         "chunking.go",
         "crd_conversion_webhook.go",
+        "crd_publish_openapi.go",
         "crd_watch.go",
         "custom_resource_definition.go",
         "etcd_failure.go",
@@ -40,7 +41,9 @@ go_library(
         "//staging/src/k8s.io/api/rbac/v1:go_default_library",
         "//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
+        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apiserver/validation:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration:go_default_library",
         "//staging/src/k8s.io/apiextensions-apiserver/test/integration/fixtures:go_default_library",
@@ -60,6 +63,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/yaml:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
@@ -77,9 +81,12 @@ go_library(
         "//test/e2e/framework/metrics:go_default_library",
         "//test/utils:go_default_library",
         "//test/utils/image:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/kube-openapi/pkg/util:go_default_library",
+        "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],
 )
 

test/integration/master/BUILD

@@ -66,6 +66,7 @@ go_test(
         "//test/integration/framework:go_default_library",
         "//test/utils:go_default_library",
         "//vendor/github.com/evanphx/json-patch:go_default_library",
+        "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/stretchr/testify/require:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ] + select({

test/integration/master/crd_test.go

@@ -18,15 +18,19 @@ package master
 
 import (
 	"encoding/json"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/go-openapi/spec"
+
 	networkingv1 "k8s.io/api/networking/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
@@ -130,6 +134,90 @@ func TestCRD(t *testing.T) {
 	}
 }
 
+func TestCRDOpenAPI(t *testing.T) {
+	result := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--feature-gates=CustomResourcePublishOpenAPI=true"}, framework.SharedEtcd())
+	defer result.TearDownFn()
+	kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	apiextensionsclient, err := apiextensionsclientset.NewForConfig(result.ClientConfig)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	t.Logf("Trying to create a custom resource without conflict")
+	crd := &apiextensionsv1beta1.CustomResourceDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "foos.cr.bar.com",
+		},
+		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+			Group:   "cr.bar.com",
+			Version: "v1",
+			Scope:   apiextensionsv1beta1.NamespaceScoped,
+			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+				Plural: "foos",
+				Kind:   "Foo",
+			},
+			Validation: &apiextensionsv1beta1.CustomResourceValidation{
+				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
+					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
+						"foo": {Type: "string"},
+					},
+				},
+			},
+		},
+	}
+	etcd.CreateTestCRDs(t, apiextensionsclient, false, crd)
+	waitForSpec := func(expectedType string) {
+		t.Logf(`Waiting for {properties: {"foo": {"type":"%s"}}} to show up in schema`, expectedType)
+		lastMsg := ""
+		if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
+			lastMsg = ""
+			bs, err := kubeclient.RESTClient().Get().AbsPath("openapi", "v2").DoRaw()
+			if err != nil {
+				return false, err
+			}
+			spec := spec.Swagger{}
+			if err := json.Unmarshal(bs, &spec); err != nil {
+				return false, err
+			}
+			if spec.SwaggerProps.Paths == nil {
+				lastMsg = "spec.SwaggerProps.Paths is nil"
+				return false, nil
+			}
+			d, ok := spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"]
+			if !ok {
+				lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"] not found`
+				return false, nil
+			}
+			p, ok := d.Properties["foo"]
+			if !ok {
+				lastMsg = `spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"] not found`
+				return false, nil
+			}
+			if !p.Type.Contains(expectedType) {
+				lastMsg = fmt.Sprintf(`spec.SwaggerProps.Definitions["com.bar.cr.v1.Foo"].Properties["foo"].Type should be %q, but got: %q`, expectedType, p.Type)
+				return false, nil
+			}
+			return true, nil
+		}); err != nil {
+			t.Fatalf("Failed to see %s OpenAPI spec in discovery: %v, last message: %s", crd.Name, err, lastMsg)
+		}
+	}
+	waitForSpec("string")
+	crd, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(crd.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	prop := crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"]
+	prop.Type = "boolean"
+	crd.Spec.Validation.OpenAPIV3Schema.Properties["foo"] = prop
+	if _, err = apiextensionsclient.ApiextensionsV1beta1().CustomResourceDefinitions().Update(crd); err != nil {
+		t.Fatal(err)
+	}
+	waitForSpec("boolean")
+}
+
 type Foo struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`