only update dns records when ready address available for endpoints

federation/pkg/federation-controller/service/endpoint_helper.go

@@ -120,27 +120,54 @@ func (cc *clusterClientCache) processEndpointDeletion(cachedService *cachedServi
 func (cc *clusterClientCache) processEndpointUpdate(cachedService *cachedService, endpoint *v1.Endpoints, clusterName string, serviceController *ServiceController) error {
 	glog.V(4).Infof("Processing endpoint update for %s/%s, cluster %s", endpoint.Namespace, endpoint.Name, clusterName)
 	cachedService.rwlock.Lock()
+	var reachable bool
 	defer cachedService.rwlock.Unlock()
-	for _, subset := range endpoint.Subsets {
-		if len(subset.Addresses) > 0 {
-			cachedService.endpointMap[clusterName] = 1
-		}
-	}
 	_, ok := cachedService.endpointMap[clusterName]
 	if !ok {
-		// first time get endpoints, update dns record
-		glog.V(4).Infof("Cached endpoint was not found for %s/%s, cluster %s, building one", endpoint.Namespace, endpoint.Name, clusterName)
-		cachedService.endpointMap[clusterName] = 1
-		if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil {
-			glog.V(4).Infof("Error ensuring DNS Records: %v", err)
-			for i := 0; i < clientRetryCount; i++ {
-				time.Sleep(cachedService.nextDNSUpdateDelay())
-				err := serviceController.ensureDnsRecords(clusterName, cachedService)
-				if err == nil {
-					return nil
+		for _, subset := range endpoint.Subsets {
+			if len(subset.Addresses) > 0 {
+				reachable = true
+				break
+			}
+		}
+		if reachable {
+			// first time get endpoints, update dns record
+			glog.V(4).Infof("Reachable endpoint was found for %s/%s, cluster %s, building endpointMap", endpoint.Namespace, endpoint.Name, clusterName)
+			cachedService.endpointMap[clusterName] = 1
+			if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil {
+				glog.V(4).Infof("Error ensuring DNS Records: %v", err)
+				for i := 0; i < clientRetryCount; i++ {
+					time.Sleep(cachedService.nextDNSUpdateDelay())
+					err := serviceController.ensureDnsRecords(clusterName, cachedService)
+					if err == nil {
+						return nil
+					}
+				}
+				return err
+			}
+		}
+	} else {
+		for _, subset := range endpoint.Subsets {
+			if len(subset.Addresses) > 0 {
+				reachable = true
+				break
+			}
+		}
+		if !reachable {
+			// first time get endpoints, update dns record
+			glog.V(4).Infof("Reachable endpoint was lost for %s/%s, cluster %s, deleting endpointMap", endpoint.Namespace, endpoint.Name, clusterName)
+			delete(cachedService.endpointMap, clusterName)
+			if err := serviceController.ensureDnsRecords(clusterName, cachedService); err != nil {
+				glog.V(4).Infof("Error ensuring DNS Records: %v", err)
+				for i := 0; i < clientRetryCount; i++ {
+					time.Sleep(cachedService.nextDNSUpdateDelay())
+					err := serviceController.ensureDnsRecords(clusterName, cachedService)
+					if err == nil {
+						return nil
+					}
 				}
+				return err
 			}
-			return err
 		}
 	}
 	return nil

federation/pkg/federation-controller/service/servicecontroller.go

@@ -76,7 +76,7 @@ type cachedService struct {
 	appliedState *v1.Service
 	// cluster endpoint map hold subset info from kubernetes clusters
 	// key clusterName
-	// value is a flag that if there is ready address, 1 means there is ready address, 0 means no ready address
+	// value is a flag that if there is ready address, 1 means there is ready address
 	endpointMap map[string]int
 	// serviceStatusMap map holds service status info from kubernetes clusters, keyed on clusterName
 	serviceStatusMap map[string]v1.LoadBalancerStatus
@@ -101,7 +101,7 @@ type ServiceController struct {
 	dns              dnsprovider.Interface
 	federationClient federation_release_1_3.Interface
 	federationName   string
-	zoneName 		 string
+	zoneName         string
 	// each federation should be configured with a single zone (e.g. "mycompany.com")
 	dnsZones     dnsprovider.Zones
 	serviceCache *serviceCache
@@ -134,7 +134,7 @@ func New(federationClient federation_release_1_3.Interface, dns dnsprovider.Inte
 		dns:              dns,
 		federationClient: federationClient,
 		federationName:   federationName,
-		zoneName:   	  zoneName,
+		zoneName:         zoneName,
 		serviceCache:     &serviceCache{fedServiceMap: make(map[string]*cachedService)},
 		clusterCache: &clusterClientCache{
 			rwlock:    sync.Mutex{},