Create nodes health checks for non-OnlyLocal services

pkg/cloudprovider/providers/gce/gce.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -80,7 +81,7 @@ type GCECloud struct {
 	serviceBeta              *computebeta.Service
 	containerService         *container.Service
 	clientBuilder            controller.ControllerClientBuilder
-	ClusterId                ClusterId
+	ClusterID                ClusterID
 	projectID                string
 	region                   string
 	localZone                string   // The zone in which we are running
@@ -92,6 +93,11 @@ type GCECloud struct {
 	useMetadataServer        bool
 	operationPollRateLimiter flowcontrol.RateLimiter
 	manager                  ServiceManager
+	// sharedResourceLock is used to serialize GCE operations that may mutate shared state to
+	// prevent inconsistencies. For example, load balancers manipulation methods will take the
+	// lock to prevent shared resources from being prematurely deleted while the operation is
+	// in progress.
+	sharedResourceLock sync.Mutex
 }
 
 type ServiceManager interface {
@@ -270,10 +276,10 @@ func CreateGCECloud(projectID, region, zone string, managedZones []string, netwo
 }
 
 // Initialize takes in a clientBuilder and spawns a goroutine for watching the clusterid configmap.
-// This must be called before utilizing the funcs of gce.ClusterId
+// This must be called before utilizing the funcs of gce.ClusterID
 func (gce *GCECloud) Initialize(clientBuilder controller.ControllerClientBuilder) {
 	gce.clientBuilder = clientBuilder
-	go gce.watchClusterId()
+	go gce.watchClusterID()
 }
 
 // LoadBalancer returns an implementation of LoadBalancer for Google Compute Engine.

pkg/cloudprovider/providers/gce/gce_clusterid.go

 /*
-Copyright 2014 The Kubernetes Authors.
+Copyright 2017 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -49,18 +49,18 @@ const (
 	updateFuncFrequency = 10 * time.Minute
 )
 
-type ClusterId struct {
+type ClusterID struct {
 	idLock     sync.RWMutex
 	client     clientset.Interface
 	cfgMapKey  string
 	store      cache.Store
-	providerId *string
-	clusterId  *string
+	providerID *string
+	clusterID  *string
 }
 
-// Continually watches for changes to the cluser id config map
-func (gce *GCECloud) watchClusterId() {
-	gce.ClusterId = ClusterId{
+// Continually watches for changes to the cluster id config map
+func (gce *GCECloud) watchClusterID() {
+	gce.ClusterID = ClusterID{
 		cfgMapKey: fmt.Sprintf("%v/%v", UIDNamespace, UIDConfigMapName),
 		client:    gce.clientBuilder.ClientOrDie("cloud-provider"),
 	}
@@ -77,8 +77,8 @@ func (gce *GCECloud) watchClusterId() {
 				return
 			}
 
-			glog.V(4).Infof("Observed new configmap for clusterid: %v, %v; setting local values", m.Name, m.Data)
-			gce.ClusterId.setIds(m)
+			glog.V(4).Infof("Observed new configmap for clusteriD: %v, %v; setting local values", m.Name, m.Data)
+			gce.ClusterID.update(m)
 		},
 		UpdateFunc: func(old, cur interface{}) {
 			m, ok := cur.(*v1.ConfigMap)
@@ -96,71 +96,71 @@ func (gce *GCECloud) watchClusterId() {
 				return
 			}
 
-			glog.V(4).Infof("Observed updated configmap for clusterid %v, %v; setting local values", m.Name, m.Data)
-			gce.ClusterId.setIds(m)
+			glog.V(4).Infof("Observed updated configmap for clusteriD %v, %v; setting local values", m.Name, m.Data)
+			gce.ClusterID.update(m)
 		},
 	}
 
-	listerWatcher := cache.NewListWatchFromClient(gce.ClusterId.client.Core().RESTClient(), "configmaps", UIDNamespace, fields.Everything())
+	listerWatcher := cache.NewListWatchFromClient(gce.ClusterID.client.Core().RESTClient(), "configmaps", UIDNamespace, fields.Everything())
 	var controller cache.Controller
-	gce.ClusterId.store, controller = cache.NewInformer(newSingleObjectListerWatcher(listerWatcher, UIDConfigMapName), &v1.ConfigMap{}, updateFuncFrequency, mapEventHandler)
+	gce.ClusterID.store, controller = cache.NewInformer(newSingleObjectListerWatcher(listerWatcher, UIDConfigMapName), &v1.ConfigMap{}, updateFuncFrequency, mapEventHandler)
 
 	controller.Run(nil)
 }
 
-// GetId returns the id which is unique to this cluster
+// GetID returns the id which is unique to this cluster
 // if federated, return the provider id (unique to the cluster)
 // if not federated, return the cluster id
-func (ci *ClusterId) GetId() (string, error) {
+func (ci *ClusterID) GetID() (string, error) {
 	if err := ci.getOrInitialize(); err != nil {
 		return "", err
 	}
 
 	ci.idLock.RLock()
 	defer ci.idLock.RUnlock()
-	if ci.clusterId == nil {
+	if ci.clusterID == nil {
 		return "", errors.New("Could not retrieve cluster id")
 	}
 
 	// If provider ID is set, (Federation is enabled) use this field
-	if ci.providerId != nil && *ci.providerId != *ci.clusterId {
-		return *ci.providerId, nil
+	if ci.providerID != nil {
+		return *ci.providerID, nil
 	}
 
-	// providerId is not set, use the cluster id
-	return *ci.clusterId, nil
+	// providerID is not set, use the cluster id
+	return *ci.clusterID, nil
 }
 
 // GetFederationId returns the id which could represent the entire Federation
 // or just the cluster if not federated.
-func (ci *ClusterId) GetFederationId() (string, bool, error) {
+func (ci *ClusterID) GetFederationId() (string, bool, error) {
 	if err := ci.getOrInitialize(); err != nil {
 		return "", false, err
 	}
 
 	ci.idLock.RLock()
 	defer ci.idLock.RUnlock()
-	if ci.clusterId == nil {
+	if ci.clusterID == nil {
 		return "", false, errors.New("Could not retrieve cluster id")
 	}
 
 	// If provider ID is not set, return false
-	if ci.providerId == nil || *ci.clusterId == *ci.providerId {
+	if ci.providerID == nil || *ci.clusterID == *ci.providerID {
 		return "", false, nil
 	}
 
-	return *ci.clusterId, true, nil
+	return *ci.clusterID, true, nil
 }
 
 // getOrInitialize either grabs the configmaps current value or defines the value
-// and sets the configmap. This is for the case of the user calling GetClusterId()
+// and sets the configmap. This is for the case of the user calling GetClusterID()
 // before the watch has begun.
-func (ci *ClusterId) getOrInitialize() error {
+func (ci *ClusterID) getOrInitialize() error {
 	if ci.store == nil {
-		return errors.New("GCECloud.ClusterId is not ready. Call Initialize() before using.")
+		return errors.New("GCECloud.ClusterID is not ready. Call Initialize() before using.")
 	}
 
-	if ci.clusterId != nil {
+	if ci.clusterID != nil {
 		return nil
 	}
 
@@ -177,7 +177,7 @@ func (ci *ClusterId) getOrInitialize() error {
 		return err
 	}
 
-	glog.V(4).Infof("Creating clusterid: %v", newId)
+	glog.V(4).Infof("Creating clusteriD: %v", newId)
 	cfg := &v1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      UIDConfigMapName,
@@ -194,12 +194,12 @@ func (ci *ClusterId) getOrInitialize() error {
 		return err
 	}
 
-	glog.V(2).Infof("Created a config map containing clusterid: %v", newId)
-	ci.setIds(cfg)
+	glog.V(2).Infof("Created a config map containing clusteriD: %v", newId)
+	ci.update(cfg)
 	return nil
 }
 
-func (ci *ClusterId) getConfigMap() (bool, error) {
+func (ci *ClusterID) getConfigMap() (bool, error) {
 	item, exists, err := ci.store.GetByKey(ci.cfgMapKey)
 	if err != nil {
 		return false, err
@@ -214,18 +214,18 @@ func (ci *ClusterId) getConfigMap() (bool, error) {
 		glog.Error(err)
 		return false, err
 	}
-	ci.setIds(m)
+	ci.update(m)
 	return true, nil
 }
 
-func (ci *ClusterId) setIds(m *v1.ConfigMap) {
+func (ci *ClusterID) update(m *v1.ConfigMap) {
 	ci.idLock.Lock()
 	defer ci.idLock.Unlock()
-	if clusterId, exists := m.Data[UIDCluster]; exists {
-		ci.clusterId = &clusterId
+	if clusterID, exists := m.Data[UIDCluster]; exists {
+		ci.clusterID = &clusterID
 	}
 	if provId, exists := m.Data[UIDProvider]; exists {
-		ci.providerId = &provId
+		ci.providerID = &provId
 	}
 }
 

pkg/cloudprovider/providers/gce/gce_healthchecks.go

@@ -17,11 +17,23 @@ limitations under the License.
 package gce
 
 import (
+	"fmt"
 	"time"
 
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/master/ports"
+	utilversion "k8s.io/kubernetes/pkg/util/version"
+
+	"github.com/golang/glog"
 	compute "google.golang.org/api/compute/v1"
 )
 
+const (
+	minNodesHealthCheckVersion = "1.7.0"
+	nodesHealthCheckPath       = "/healthz"
+	lbNodesHealthCheckPort     = ports.ProxyHealthzPort
+)
+
 func newHealthcheckMetricContext(request string) *metricContext {
 	return &metricContext{
 		start:      time.Now(),
@@ -178,3 +190,59 @@ func (gce *GCECloud) ListHealthChecks() (*compute.HealthCheckList, error) {
 	v, err := gce.service.HealthChecks.List(gce.projectID).Do()
 	return v, mc.Observe(err)
 }
+
+// GetNodesHealthCheckPort returns the health check port used by the GCE load
+// balancers (l4) for performing health checks on nodes.
+func GetNodesHealthCheckPort() int32 {
+	return lbNodesHealthCheckPort
+}
+
+// getNodesHealthCheckPath returns the health check path used by the GCE load
+// balancers (l4) for performing health checks on nodes.
+func getNodesHealthCheckPath() string {
+	return nodesHealthCheckPath
+}
+
+// makeNodesHealthCheckName returns name of the health check resource used by
+// the GCE load balancers (l4) for performing health checks on nodes.
+func makeNodesHealthCheckName(clusterID string) string {
+	return fmt.Sprintf("k8s-%v-node", clusterID)
+}
+
+// MakeHealthCheckFirewallName returns the firewall name used by the GCE load
+// balancers (l4) for performing health checks.
+func MakeHealthCheckFirewallName(clusterID, hcName string, isNodesHealthCheck bool) string {
+	// TODO: Change below fwName to match the proposed schema: k8s-{clusteriD}-{namespace}-{name}-{shortid}-hc.
+	fwName := "k8s-" + hcName + "-http-hc"
+	if isNodesHealthCheck {
+		fwName = makeNodesHealthCheckName(clusterID) + "-http-hc"
+	}
+	return fwName
+}
+
+// isAtLeastMinNodesHealthCheckVersion checks if a version is higher than
+// `minNodesHealthCheckVersion`.
+func isAtLeastMinNodesHealthCheckVersion(vstring string) bool {
+	minVersion, err := utilversion.ParseGeneric(minNodesHealthCheckVersion)
+	if err != nil {
+		glog.Errorf("MinNodesHealthCheckVersion (%s) is not a valid version string: %v", minNodesHealthCheckVersion, err)
+		return false
+	}
+	version, err := utilversion.ParseGeneric(vstring)
+	if err != nil {
+		glog.Errorf("vstring (%s) is not a valid version string: %v", vstring, err)
+		return false
+	}
+	return version.AtLeast(minVersion)
+}
+
+// supportsNodesHealthCheck returns false if anyone of the nodes has version
+// lower than `minNodesHealthCheckVersion`.
+func supportsNodesHealthCheck(nodes []*v1.Node) bool {
+	for _, node := range nodes {
+		if !isAtLeastMinNodesHealthCheckVersion(node.Status.NodeInfo.KubeProxyVersion) {
+			return false
+		}
+	}
+	return true
+}

pkg/cloudprovider/providers/gce/gce_healthchecks_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gce
+
+import (
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+)
+
+func TestIsAtLeastMinNodesHealthCheckVersion(t *testing.T) {
+	testCases := []struct {
+		version string
+		expect  bool
+	}{
+		{"v1.7.1", true},
+		{"v1.7.0-alpha.2.597+276d289b90d322", true},
+		{"v1.6.0-beta.3.472+831q821c907t31a", false},
+		{"v1.5.2", false},
+	}
+
+	for _, tc := range testCases {
+		if res := isAtLeastMinNodesHealthCheckVersion(tc.version); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.version, tc.expect, res)
+		}
+	}
+}
+
+func TestSupportsNodesHealthCheck(t *testing.T) {
+	testCases := []struct {
+		desc   string
+		nodes  []*v1.Node
+		expect bool
+	}{
+		{
+			"All nodes support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.1",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.0-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+			},
+			true,
+		},
+		{
+			"All nodes don't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.6.0-beta.3.472+831q821c907t31a",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+		{
+			"One node doesn't support nodes health check",
+			[]*v1.Node{
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.1",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.7.0-alpha.2.597+276d289b90d322",
+						},
+					},
+				},
+				{
+					Status: v1.NodeStatus{
+						NodeInfo: v1.NodeSystemInfo{
+							KubeProxyVersion: "v1.5.2",
+						},
+					},
+				},
+			},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		if res := supportsNodesHealthCheck(tc.nodes); res != tc.expect {
+			t.Errorf("%v: want %v, got %v", tc.desc, tc.expect, res)
+		}
+	}
+}

pkg/cloudprovider/providers/gce/gce_util.go

@@ -19,10 +19,12 @@ package gce
 import (
 	"errors"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	"cloud.google.com/go/compute/metadata"
 	compute "google.golang.org/api/compute/v1"
@@ -105,6 +107,14 @@ func isHTTPErrorCode(err error, code int) bool {
 	return ok && apiErr.Code == code
 }
 
+func isInUsedByError(err error) bool {
+	apiErr, ok := err.(*googleapi.Error)
+	if !ok || apiErr.Code != http.StatusBadRequest {
+		return false
+	}
+	return strings.Contains(apiErr.Message, "being used by")
+}
+
 // splitProviderID splits a provider's id into core components.
 // A providerID is build out of '${ProviderName}://${project-id}/${zone}/${instance-name}'
 // See cloudprovider.GetInstanceProviderID.
@@ -115,3 +125,12 @@ func splitProviderID(providerID string) (project, zone, instance string, err err
 	}
 	return matches[1], matches[2], matches[3], nil
 }
+
+func equalStringSets(x, y []string) bool {
+	if len(x) != len(y) {
+		return false
+	}
+	xString := sets.NewString(x...)
+	yString := sets.NewString(y...)
+	return xString.Equal(yString)
+}