add basic wiring for kubernetes-discovery to become a kube-like api server

cmd/kubernetes-discovery/BUILD

@@ -16,6 +16,7 @@ go_binary(
     tags = ["automanaged"],
     deps = [
         "//cmd/kubernetes-discovery/pkg/cmd/server:go_default_library",
+        "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/util/logs:go_default_library",
     ],
 )

cmd/kubernetes-discovery/main.go

@@ -21,6 +21,7 @@ import (
 	"runtime"
 
 	"k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/cmd/server"
+	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
 	"k8s.io/kubernetes/pkg/util/logs"
 )
 
@@ -34,6 +35,6 @@ func main() {
 
 	cmd := server.NewCommandStartDiscoveryServer(os.Stdout, os.Stderr)
 	if err := cmd.Execute(); err != nil {
-		os.Exit(1)
+		cmdutil.CheckErr(err)
 	}
 }

cmd/kubernetes-discovery/pkg/apiserver/BUILD

+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["apiserver.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/genericapiserver:go_default_library",
+        "//pkg/registry/generic:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
+        "//pkg/version:go_default_library",
+    ],
+)

cmd/kubernetes-discovery/pkg/apiserver/apiserver.go

+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apiserver
+
+import (
+	"k8s.io/kubernetes/pkg/genericapiserver"
+	"k8s.io/kubernetes/pkg/registry/generic"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/version"
+)
+
+// TODO move to genericapiserver or something like that
+// RESTOptionsGetter is used to construct storage for a particular resource
+type RESTOptionsGetter interface {
+	NewFor(resource schema.GroupResource) generic.RESTOptions
+}
+
+type Config struct {
+	GenericConfig *genericapiserver.Config
+
+	// RESTOptionsGetter is used to construct storage for a particular resource
+	RESTOptionsGetter RESTOptionsGetter
+}
+
+// APIDiscoveryServer contains state for a Kubernetes cluster master/api server.
+type APIDiscoveryServer struct {
+	GenericAPIServer *genericapiserver.GenericAPIServer
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() completedConfig {
+	c.GenericConfig.Complete()
+
+	version := version.Get()
+	c.GenericConfig.Version = &version
+
+	return completedConfig{c}
+}
+
+// SkipComplete provides a way to construct a server instance without config completion.
+func (c *Config) SkipComplete() completedConfig {
+	return completedConfig{c}
+}
+
+// New returns a new instance of APIDiscoveryServer from the given config.
+func (c completedConfig) New() (*APIDiscoveryServer, error) {
+	genericServer, err := c.Config.GenericConfig.SkipComplete().New() // completion is done in Complete, no need for a second time
+	if err != nil {
+		return nil, err
+	}
+
+	s := &APIDiscoveryServer{
+		GenericAPIServer: genericServer,
+	}
+
+	// TODO switch to constants once we have an API
+	// TODO install RESTStorage for API
+	// apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo("apifederation.k8s.io")
+	// apiGroupInfo.GroupMeta.GroupVersion = schema.GroupVersion{Group: "apifederation.k8s.io", Version: "v1alpha1"}
+
+	// v1alpha1storage := map[string]rest.Storage{}
+	// // v1alpha1storage["apiservices"] = apiserverstorage.NewREST(c.RESTOptionsGetter.NewFor(apifederation.Resource("apiservices")))
+
+	// apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage
+
+	// if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
+	// 	return nil, err
+	// }
+
+	return s, nil
+}

cmd/kubernetes-discovery/pkg/cmd/server/BUILD

@@ -15,8 +15,19 @@ go_library(
     srcs = ["start.go"],
     tags = ["automanaged"],
     deps = [
+        "//cmd/kubernetes-discovery/pkg/apiserver:go_default_library",
         "//cmd/kubernetes-discovery/pkg/legacy:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/apimachinery/registered:go_default_library",
+        "//pkg/genericapiserver:go_default_library",
+        "//pkg/genericapiserver/options:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
+        "//pkg/registry/generic:go_default_library",
+        "//pkg/registry/generic/registry:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
+        "//pkg/storage/storagebackend:go_default_library",
+        "//pkg/util/wait:go_default_library",
+        "//vendor:github.com/pborman/uuid",
         "//vendor:github.com/spf13/cobra",
     ],
 )

cmd/kubernetes-discovery/pkg/cmd/server/start.go

@@ -17,15 +17,32 @@ limitations under the License.
 package server
 
 import (
+	"fmt"
 	"io"
 
+	"github.com/pborman/uuid"
 	"github.com/spf13/cobra"
 
+	"k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/apiserver"
 	"k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/legacy"
-	kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/apimachinery/registered"
+	"k8s.io/kubernetes/pkg/genericapiserver"
+	genericoptions "k8s.io/kubernetes/pkg/genericapiserver/options"
+	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	"k8s.io/kubernetes/pkg/registry/generic"
+	"k8s.io/kubernetes/pkg/registry/generic/registry"
+	"k8s.io/kubernetes/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/storage/storagebackend"
+	"k8s.io/kubernetes/pkg/util/wait"
 )
 
+const defaultEtcdPathPrefix = "/registry/kubernetes.io/kubernetes-discovery"
+
 type DiscoveryServerOptions struct {
+	Etcd          *genericoptions.EtcdOptions
+	SecureServing *genericoptions.SecureServingOptions
+
 	StdOut io.Writer
 	StdErr io.Writer
 }
@@ -33,20 +50,30 @@ type DiscoveryServerOptions struct {
 // NewCommandStartMaster provides a CLI handler for 'start master' command
 func NewCommandStartDiscoveryServer(out, err io.Writer) *cobra.Command {
 	o := &DiscoveryServerOptions{
+		Etcd:          genericoptions.NewEtcdOptions(),
+		SecureServing: genericoptions.NewSecureServingOptions(),
+
 		StdOut: out,
 		StdErr: err,
 	}
+	o.Etcd.StorageConfig.Prefix = defaultEtcdPathPrefix
+	o.Etcd.StorageConfig.Codec = api.Codecs.LegacyCodec(registered.EnabledVersionsForGroup(api.GroupName)...)
+	o.SecureServing.ServingOptions.BindPort = 9090
 
 	cmd := &cobra.Command{
 		Short: "Launch a discovery summarizer and proxy server",
 		Long:  "Launch a discovery summarizer and proxy server",
 		Run: func(c *cobra.Command, args []string) {
-			kcmdutil.CheckErr(o.Complete())
-			kcmdutil.CheckErr(o.Validate(args))
-			kcmdutil.CheckErr(o.RunDiscoveryServer())
+			cmdutil.CheckErr(o.Complete())
+			cmdutil.CheckErr(o.Validate(args))
+			cmdutil.CheckErr(o.RunDiscoveryServer())
 		},
 	}
 
+	flags := cmd.Flags()
+	o.Etcd.AddFlags(flags)
+	o.SecureServing.AddFlags(flags)
+
 	return cmd
 }
 
@@ -59,11 +86,38 @@ func (o *DiscoveryServerOptions) Complete() error {
 }
 
 func (o DiscoveryServerOptions) RunDiscoveryServer() error {
-	if true {
-		// for now this is the only option.  later, only use this if no etcd is configured
+	// if we don't have an etcd to back the server, we must be a legacy server
+	if len(o.Etcd.StorageConfig.ServerList) == 0 {
 		return o.RunLegacyDiscoveryServer()
 	}
 
+	// TODO have a "real" external address
+	if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil {
+		return fmt.Errorf("error creating self-signed certificates: %v", err)
+	}
+
+	genericAPIServerConfig := genericapiserver.NewConfig()
+	if _, err := genericAPIServerConfig.ApplySecureServingOptions(o.SecureServing); err != nil {
+		return err
+	}
+
+	var err error
+	privilegedLoopbackToken := uuid.NewRandom().String()
+	if genericAPIServerConfig.LoopbackClientConfig, err = genericAPIServerConfig.SecureServingInfo.NewSelfClientConfig(privilegedLoopbackToken); err != nil {
+		return err
+	}
+
+	config := apiserver.Config{
+		GenericConfig:     genericAPIServerConfig,
+		RESTOptionsGetter: restOptionsFactory{storageConfig: &o.Etcd.StorageConfig},
+	}
+
+	server, err := config.Complete().New()
+	if err != nil {
+		return err
+	}
+	server.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
+
 	return nil
 }
 
@@ -77,3 +131,17 @@ func (o DiscoveryServerOptions) RunLegacyDiscoveryServer() error {
 	}
 	return s.Run(port)
 }
+
+type restOptionsFactory struct {
+	storageConfig *storagebackend.Config
+}
+
+func (f restOptionsFactory) NewFor(resource schema.GroupResource) generic.RESTOptions {
+	return generic.RESTOptions{
+		StorageConfig:           f.storageConfig,
+		Decorator:               registry.StorageWithCacher,
+		DeleteCollectionWorkers: 1,
+		EnableGarbageCollection: false,
+		ResourcePrefix:          f.storageConfig.Prefix + "/" + resource.Group + "/" + resource.Resource,
+	}
+}

cmd/kubernetes-discovery/pkg/legacy/doc.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package discoverysummarizer contains code for the discovery summarizer
+// Package discoverysummarizer contains code for the legacy discovery summarizer
 // (program to summarize discovery information from all federated api servers)
 // as per https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/federated-api-servers.md
 package legacy // import "k8s.io/kubernetes/cmd/kubernetes-discovery/pkg/legacy"

hack/local-up-cluster.sh

@@ -135,7 +135,7 @@ do
 done
 
 if [ "x$GO_OUT" == "x" ]; then
-    make -C "${KUBE_ROOT}" WHAT="cmd/kubectl cmd/hyperkube"
+    make -C "${KUBE_ROOT}" WHAT="cmd/kubectl cmd/hyperkube cmd/kubernetes-discovery"
 else
     echo "skipped the build."
 fi
@@ -198,6 +198,8 @@ ENABLE_CONTROLLER_ATTACH_DETACH=${ENABLE_CONTROLLER_ATTACH_DETACH:-"true"} # cur
 CERT_DIR=${CERT_DIR:-"/var/run/kubernetes"}
 ROOT_CA_FILE=$CERT_DIR/apiserver.crt
 EXPERIMENTAL_CRI=${EXPERIMENTAL_CRI:-"false"}
+DISCOVERY_SECURE_PORT=${DISCOVERY_SECURE_PORT:-9090}
+
 
 # Ensure CERT_DIR is created for auto-generated crt/key and kubeconfig
 mkdir -p "${CERT_DIR}" &>/dev/null || sudo mkdir -p "${CERT_DIR}"
@@ -300,6 +302,10 @@ cleanup()
   [[ -n "${APISERVER_PID-}" ]] && APISERVER_PIDS=$(pgrep -P ${APISERVER_PID} ; ps -o pid= -p ${APISERVER_PID})
   [[ -n "${APISERVER_PIDS-}" ]] && sudo kill ${APISERVER_PIDS}
 
+  # Check if the discovery server is still running
+  [[ -n "${DISCOVERY_PID-}" ]] && DISCOVERY_PIDS=$(pgrep -P ${DISCOVERY_PID} ; ps -o pid= -p ${DISCOVERY_PID})
+  [[ -n "${DISCOVERY_PIDS-}" ]] && sudo kill ${DISCOVERY_PIDS}
+
   # Check if the controller-manager is still running
   [[ -n "${CTLRMGR_PID-}" ]] && CTLRMGR_PIDS=$(pgrep -P ${CTLRMGR_PID} ; ps -o pid= -p ${CTLRMGR_PID})
   [[ -n "${CTLRMGR_PIDS-}" ]] && sudo kill ${CTLRMGR_PIDS}
@@ -504,6 +510,26 @@ EOF
     fi
 }
 
+# start_discovery relies on certificates created by start_apiserver
+function start_discovery {
+    # TODO generate serving certificates
+    
+    DISCOVERY_SERVER_LOG=/tmp/kubernetes-discovery.log
+    ${CONTROLPLANE_SUDO} "${GO_OUT}/kubernetes-discovery" \
+      --cert-dir="${CERT_DIR}" \
+      --client-ca-file="${CERT_DIR}/client-ca-bundle.crt" \
+      --bind-address="${API_BIND_ADDR}" \
+      --secure-port="${DISCOVERY_SECURE_PORT}" \
+      --tls-ca-file="${ROOT_CA_FILE}" \
+      --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}"  >"${DISCOVERY_SERVER_LOG}" 2>&1 &
+    DISCOVERY_PID=$!
+
+    # Wait for kubernetes-discovery to come up before launching the rest of the components.
+    echo "Waiting for kubernetes-discovery to come up"
+    kube::util::wait_for_url "https://${API_HOST}:${DISCOVERY_SECURE_PORT}/version" "kubernetes-discovery: " 1 ${WAIT_FOR_URL_API_SERVER} || exit 1
+}
+
+
 function start_controller_manager {
     node_cidr_args=""
     if [[ "${NET_PLUGIN}" == "kubenet" ]]; then
@@ -778,6 +804,7 @@ if [[ "${START_MODE}" != "kubeletonly" ]]; then
   start_etcd
   set_service_accounts
   start_apiserver
+  start_discovery
   start_controller_manager
   start_kubeproxy
   start_kubedns