Fix syncing empty list of apiserver addresses during initial startup

pkg/agent/tunnel/tunnel.go

@@ -38,7 +38,11 @@ import (
 )
 
 var (
-	endpointDebounceDelay = time.Second
+	// endpointDebounceDelay sets how long we wait before updating apiserver
+	// addresses when the kubernetes endpoint list changes. When the apiserver is
+	// starting up it adds then removes then re-adds itself a few times in quick
+	// succession, and we want to avoid closing connections unnecessarily.
+	endpointDebounceDelay = 3 * time.Second
 	defaultDialer         = net.Dialer{}
 )
 
@@ -335,14 +339,9 @@ func (a *agentTunnel) watchEndpoints(ctx context.Context, apiServerReady <-chan
 		<-done
 	}()
 
-	var cancelUpdate context.CancelFunc
-
 	for {
 		select {
 		case <-ctx.Done():
-			if cancelUpdate != nil {
-				cancelUpdate()
-			}
 			return
 		case ev, ok := <-watch.ResultChan():
 			endpoint, ok := ev.Object.(*v1.Endpoints)
@@ -351,20 +350,15 @@ func (a *agentTunnel) watchEndpoints(ctx context.Context, apiServerReady <-chan
 				continue
 			}
 
-			if cancelUpdate != nil {
-				cancelUpdate()
-			}
-
-			var debounceCtx context.Context
-			debounceCtx, cancelUpdate = context.WithCancel(ctx)
-
 			// When joining the cluster, the apiserver adds, removes, and then re-adds itself to
 			// the endpoint list several times.  This causes a bit of thrashing if we react to
 			// endpoint changes immediately.  Instead, perform the endpoint update in a
 			// goroutine that sleeps for a short period before checking for changes and updating
 			// the proxy addresses.  If another update occurs, the previous update operation
 			// will be cancelled and a new one queued.
-			go syncProxyAddresses(debounceCtx, util.GetAddresses(endpoint))
+			addresses := util.GetAddresses(endpoint)
+			logrus.Debugf("Syncing apiserver addresses from tunnel watch: %v", addresses)
+			syncProxyAddresses(addresses)
 		}
 	}
 }
@@ -478,15 +472,15 @@ func (a *agentTunnel) dialContext(ctx context.Context, network, address string)
 }
 
 // proxySyncer is a common signature for functions that sync the proxy address list with a context
-type proxySyncer func(ctx context.Context, addresses []string)
+type proxySyncer func(addresses []string)
 
 // getProxySyncer returns a function that can be called to update the list of supervisors.
 // This function is responsible for connecting to or disconnecting websocket tunnels,
 // as well as updating the proxy loadbalancer server list.
 func (a *agentTunnel) getProxySyncer(ctx context.Context, wg *sync.WaitGroup, tlsConfig *tls.Config, proxy proxy.Proxy) proxySyncer {
 	disconnect := map[string]context.CancelFunc{}
-	// Attempt to connect to supervisors, storing their cancellation function for later when we
-	// need to disconnect.
+	// Attempt to connect to inital list of addresses, storing their cancellation
+	// function for later when we need to disconnect.
 	for _, address := range proxy.SupervisorAddresses() {
 		if _, ok := disconnect[address]; !ok {
 			conn := a.connect(ctx, wg, address, tlsConfig)
@@ -495,13 +489,33 @@ func (a *agentTunnel) getProxySyncer(ctx context.Context, wg *sync.WaitGroup, tl
 		}
 	}
 
-	// return a function that can be called to update the address list.
-	// servers will be connected to or disconnected from as necessary,
-	// and the proxy addresses updated.
-	return func(debounceCtx context.Context, addresses []string) {
+	var cancelUpdate context.CancelFunc
+
+	// return a function that can be called to update the address list.  servers will be
+	// connected to or disconnected from as necessary, and the proxy addresses updated.
+	// The update is done in a goroutine that waits a short period in order to reduce
+	// thrashing during apiserver startup. Each time the function is called, the context for
+	// the goroutine started by the previous call is cancelled to prevent it from updating
+	// if the delay has not yet expired.
+	return func(addresses []string) {
+		if len(addresses) == 0 {
+			logrus.Debugf("Skipping apiserver addresses sync: %v", addresses)
+			return
+		}
+
+		if cancelUpdate != nil {
+			cancelUpdate()
+		}
+
+		var debounceCtx context.Context
+		debounceCtx, cancelUpdate = context.WithCancel(ctx)
+
+		go func() {
 			select {
 			case <-time.After(endpointDebounceDelay):
+				logrus.Debugf("Settled apiserver addresses sync: %v", addresses)
 			case <-debounceCtx.Done():
+				logrus.Debugf("Cancelled apiserver addresses sync: %v", addresses)
 				return
 			}
 
@@ -514,8 +528,16 @@ func (a *agentTunnel) getProxySyncer(ctx context.Context, wg *sync.WaitGroup, tl
 			proxy.Update(addresses)
 			newAddresses := sets.New(proxy.SupervisorAddresses()...)
 
+			addedAddresses := newAddresses.Difference(curAddresses)
+			removedAddresses := curAddresses.Difference(newAddresses)
+			if addedAddresses.Len() == 0 && removedAddresses.Len() == 0 {
+				return
+			}
+
+			logrus.Debugf("Sync apiserver addresses - connecting: %v, disconnecting: %v", addedAddresses.UnsortedList(), removedAddresses.UnsortedList())
+
 			// add new servers
-		for address := range newAddresses.Difference(curAddresses) {
+			for address := range addedAddresses {
 				if _, ok := disconnect[address]; !ok {
 					conn := a.connect(ctx, nil, address, tlsConfig)
 					logrus.Infof("Started tunnel to %s", address)
@@ -525,13 +547,14 @@ func (a *agentTunnel) getProxySyncer(ctx context.Context, wg *sync.WaitGroup, tl
 			}
 
 			// remove old servers
-		for address := range curAddresses.Difference(newAddresses) {
+			for address := range removedAddresses {
 				if cancel, ok := disconnect[address]; ok {
 					cancel()
 					delete(disconnect, address)
 					logrus.Infof("Stopped tunnel to %s", address)
 				}
 			}
+		}()
 	}
 }
 
@@ -550,10 +573,11 @@ func getAPIServersRequester(node *daemonconfig.Node, proxy proxy.Proxy, syncProx
 			}
 		}
 
-		if addresses, err := agentconfig.GetAPIServers(ctx, info); err != nil {
+		if addresses, err := agentconfig.GetAPIServers(ctx, info); err != nil || len(addresses) == 0 {
 			logrus.Warnf("Failed to get apiserver addresses from supervisor: %v", err)
 		} else {
-			syncProxyAddresses(ctx, addresses)
+			logrus.Debugf("Syncing apiserver addresses from server: %v", addresses)
+			syncProxyAddresses(addresses)
 		}
 	}
 }