Merge pull request #59652 from feiskyer/vmss-cache

pkg/cloudprovider/providers/azure/BUILD

@@ -28,6 +28,7 @@ go_library(
         "azure_storageaccount.go",
         "azure_vmsets.go",
         "azure_vmss.go",
+        "azure_vmss_cache.go",
         "azure_wrap.go",
         "azure_zones.go",
     ],
@@ -75,6 +76,7 @@ go_test(
         "azure_storage_test.go",
         "azure_storageaccount_test.go",
         "azure_test.go",
+        "azure_vmss_cache_test.go",
         "azure_vmss_test.go",
         "azure_wrap_test.go",
     ],

pkg/cloudprovider/providers/azure/azure.go

@@ -244,7 +244,10 @@ func NewCloud(configReader io.Reader) (cloudprovider.Interface, error) {
 	}
 
 	if strings.EqualFold(vmTypeVMSS, az.Config.VMType) {
-		az.vmSet = newScaleSet(&az)
+		az.vmSet, err = newScaleSet(&az)
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		az.vmSet = newAvailabilitySet(&az)
 	}

pkg/cloudprovider/providers/azure/azure_fakes.go

@@ -674,7 +674,7 @@ func (fVMC *fakeVirtualMachineScaleSetVMsClient) Get(resourceGroupName string, V
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 
-	vmKey := fmt.Sprintf("%s-%s", VMScaleSetName, instanceID)
+	vmKey := fmt.Sprintf("%s_%s", VMScaleSetName, instanceID)
 	if scaleSetMap, ok := fVMC.FakeStore[resourceGroupName]; ok {
 		if entity, ok := scaleSetMap[vmKey]; ok {
 			return entity, nil

pkg/cloudprovider/providers/azure/azure_vmss_cache.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+var (
+	vmssNameSeparator = "_"
+
+	nodeNameToScaleSetMappingKey = "k8sNodeNameToScaleSetMappingKey"
+	availabilitySetNodesKey      = "k8sAvailabilitySetNodesKey"
+
+	vmssCacheTTL                      = time.Minute
+	vmssVMCacheTTL                    = time.Minute
+	availabilitySetNodesCacheTTL      = 15 * time.Minute
+	nodeNameToScaleSetMappingCacheTTL = 15 * time.Minute
+)
+
+// nodeNameToScaleSetMapping maps nodeName to scaleSet name.
+// The map is required because vmss nodeName is not equal to its vmName.
+type nodeNameToScaleSetMapping map[string]string
+
+func (ss *scaleSet) makeVmssVMName(scaleSetName, instanceID string) string {
+	return fmt.Sprintf("%s%s%s", scaleSetName, vmssNameSeparator, instanceID)
+}
+
+func (ss *scaleSet) extractVmssVMName(name string) (string, string, error) {
+	ret := strings.Split(name, vmssNameSeparator)
+	if len(ret) != 2 {
+		glog.Errorf("Failed to extract vmssVMName %q", name)
+		return "", "", ErrorNotVmssInstance
+	}
+
+	return ret[0], ret[1], nil
+}
+
+func (ss *scaleSet) newVmssCache() (*timedCache, error) {
+	getter := func(key string) (interface{}, error) {
+		result, err := ss.VirtualMachineScaleSetsClient.Get(ss.ResourceGroup, key)
+		exists, realErr := checkResourceExistsFromError(err)
+		if realErr != nil {
+			return nil, realErr
+		}
+
+		if !exists {
+			return nil, nil
+		}
+
+		return &result, nil
+	}
+
+	return newTimedcache(vmssCacheTTL, getter)
+}
+
+func (ss *scaleSet) newNodeNameToScaleSetMappingCache() (*timedCache, error) {
+	getter := func(key string) (interface{}, error) {
+		scaleSetNames, err := ss.listScaleSetsWithRetry()
+		if err != nil {
+			return nil, err
+		}
+
+		localCache := make(nodeNameToScaleSetMapping)
+		for _, ssName := range scaleSetNames {
+			vms, err := ss.listScaleSetVMsWithRetry(ssName)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, vm := range vms {
+				if vm.OsProfile == nil || vm.OsProfile.ComputerName == nil {
+					glog.Warningf("failed to get computerName for vmssVM (%q)", vm.Name)
+					continue
+				}
+
+				computerName := strings.ToLower(*vm.OsProfile.ComputerName)
+				localCache[computerName] = ssName
+			}
+		}
+
+		return localCache, nil
+	}
+
+	return newTimedcache(nodeNameToScaleSetMappingCacheTTL, getter)
+}
+
+func (ss *scaleSet) newAvailabilitySetNodesCache() (*timedCache, error) {
+	getter := func(key string) (interface{}, error) {
+		vmList, err := ss.Cloud.VirtualMachineClientListWithRetry()
+		if err != nil {
+			return nil, err
+		}
+
+		localCache := sets.NewString()
+		for _, vm := range vmList {
+			localCache.Insert(*vm.Name)
+		}
+
+		return localCache, nil
+	}
+
+	return newTimedcache(availabilitySetNodesCacheTTL, getter)
+}
+
+func (ss *scaleSet) newVmssVMCache() (*timedCache, error) {
+	getter := func(key string) (interface{}, error) {
+		// vmssVM name's format is 'scaleSetName_instanceID'
+		ssName, instanceID, err := ss.extractVmssVMName(key)
+		if err != nil {
+			return nil, err
+		}
+
+		// Not found, the VM doesn't belong to any known scale sets.
+		if ssName == "" {
+			return nil, nil
+		}
+
+		result, err := ss.VirtualMachineScaleSetVMsClient.Get(ss.ResourceGroup, ssName, instanceID)
+		exists, realErr := checkResourceExistsFromError(err)
+		if realErr != nil {
+			return nil, realErr
+		}
+
+		if !exists {
+			return nil, nil
+		}
+
+		return &result, nil
+	}
+
+	return newTimedcache(vmssVMCacheTTL, getter)
+}
+
+func (ss *scaleSet) getScaleSetNameByNodeName(nodeName string) (string, error) {
+	getScaleSetName := func(nodeName string) (string, error) {
+		nodeNameMapping, err := ss.nodeNameToScaleSetMappingCache.Get(nodeNameToScaleSetMappingKey)
+		if err != nil {
+			return "", err
+		}
+
+		realMapping := nodeNameMapping.(nodeNameToScaleSetMapping)
+		if ssName, ok := realMapping[nodeName]; ok {
+			return ssName, nil
+		}
+
+		return "", nil
+	}
+
+	ssName, err := getScaleSetName(nodeName)
+	if err != nil {
+		return "", err
+	}
+
+	if ssName != "" {
+		return ssName, nil
+	}
+
+	// ssName is still not found, it is likely that new Nodes are created.
+	// Force refresh the cache and try again.
+	ss.nodeNameToScaleSetMappingCache.Delete(nodeNameToScaleSetMappingKey)
+	return getScaleSetName(nodeName)
+}
+
+func (ss *scaleSet) isNodeManagedByAvailabilitySet(nodeName string) (bool, error) {
+	cached, err := ss.availabilitySetNodesCache.Get(availabilitySetNodesKey)
+	if err != nil {
+		return false, err
+	}
+
+	availabilitySetNodes := cached.(sets.String)
+	return availabilitySetNodes.Has(nodeName), nil
+}

pkg/cloudprovider/providers/azure/azure_vmss_cache_test.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtractVmssVMName(t *testing.T) {
+	ss := &scaleSet{}
+	cases := []struct {
+		description        string
+		vmName             string
+		expectError        bool
+		expectedScaleSet   string
+		expectedInstanceID string
+	}{
+		{
+			description: "wrong vmss VM name should report error",
+			vmName:      "vm1234",
+			expectError: true,
+		},
+		{
+			description: "wrong VM name separator should report error",
+			vmName:      "vm-1234",
+			expectError: true,
+		},
+		{
+			description:        "correct vmss VM name should return correct scaleSet and instanceID",
+			vmName:             "vm_1234",
+			expectedScaleSet:   "vm",
+			expectedInstanceID: "1234",
+		},
+	}
+
+	for _, c := range cases {
+		ssName, instanceID, err := ss.extractVmssVMName(c.vmName)
+		if c.expectError {
+			assert.Error(t, err, c.description)
+			continue
+		}
+
+		assert.Equal(t, c.expectedScaleSet, ssName, c.description)
+		assert.Equal(t, c.expectedInstanceID, instanceID, c.description)
+	}
+}

pkg/cloudprovider/providers/azure/azure_vmss_test.go

@@ -24,12 +24,15 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func newTestScaleSet(scaleSetName string, vmList []string) *scaleSet {
+func newTestScaleSet(scaleSetName string, vmList []string) (*scaleSet, error) {
 	cloud := getTestCloud()
 	setTestVirtualMachineCloud(cloud, scaleSetName, vmList)
-	ss := newScaleSet(cloud)
+	ss, err := newScaleSet(cloud)
+	if err != nil {
+		return nil, err
+	}
 
-	return ss.(*scaleSet)
+	return ss.(*scaleSet), nil
 }
 
 func setTestVirtualMachineCloud(ss *Cloud, scaleSetName string, vmList []string) {
@@ -46,16 +49,16 @@ func setTestVirtualMachineCloud(ss *Cloud, scaleSetName string, vmList []string)
 	ssVMs := make(map[string]map[string]compute.VirtualMachineScaleSetVM)
 	ssVMs["rg"] = make(map[string]compute.VirtualMachineScaleSetVM)
 	for i := range vmList {
-		ID := fmt.Sprintf("azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%d", scaleSetName, i)
+		ID := fmt.Sprintf("/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/%s/virtualMachines/%d", scaleSetName, i)
 		nodeName := vmList[i]
 		instanceID := fmt.Sprintf("%d", i)
-		vmKey := fmt.Sprintf("%s-%s", scaleSetName, nodeName)
+		vmName := fmt.Sprintf("%s_%s", scaleSetName, instanceID)
 		networkInterfaces := []compute.NetworkInterfaceReference{
 			{
 				ID: &nodeName,
 			},
 		}
-		ssVMs["rg"][vmKey] = compute.VirtualMachineScaleSetVM{
+		ssVMs["rg"][vmName] = compute.VirtualMachineScaleSetVM{
 			VirtualMachineScaleSetVMProperties: &compute.VirtualMachineScaleSetVMProperties{
 				OsProfile: &compute.OSProfile{
 					ComputerName: &nodeName,
@@ -66,6 +69,7 @@ func setTestVirtualMachineCloud(ss *Cloud, scaleSetName string, vmList []string)
 			},
 			ID:         &ID,
 			InstanceID: &instanceID,
+			Name:       &vmName,
 			Location:   &ss.Location,
 		}
 	}
@@ -116,28 +120,29 @@ func TestGetInstanceIDByNodeName(t *testing.T) {
 		{
 			description: "scaleSet should get instance by node name",
 			scaleSet:    "ss",
-			vmList:      []string{"vm1", "vm2"},
-			nodeName:    "vm1",
-			expected:    "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/ss/virtualMachines/0",
+			vmList:      []string{"vmssee6c2000000", "vmssee6c2000001"},
+			nodeName:    "vmssee6c2000001",
+			expected:    "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/ss/virtualMachines/1",
 		},
 		{
 			description: "scaleSet should get instance by node name with upper cases hostname",
 			scaleSet:    "ss",
-			vmList:      []string{"VM1", "vm2"},
-			nodeName:    "vm1",
-			expected:    "azure:///subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/ss/virtualMachines/0",
+			vmList:      []string{"VMSSEE6C2000000", "VMSSEE6C2000001"},
+			nodeName:    "vmssee6c2000000",
+			expected:    "/subscriptions/script/resourceGroups/rg/providers/Microsoft.Compute/virtualMachineScaleSets/ss/virtualMachines/0",
 		},
 		{
 			description: "scaleSet should not get instance for non-exist nodes",
 			scaleSet:    "ss",
-			vmList:      []string{"VM1", "vm2"},
-			nodeName:    "vm3",
+			vmList:      []string{"vmssee6c2000000", "vmssee6c2000001"},
+			nodeName:    "agente6c2000005",
 			expectError: true,
 		},
 	}
 
 	for _, test := range testCases {
-		ss := newTestScaleSet(test.scaleSet, test.vmList)
+		ss, err := newTestScaleSet(test.scaleSet, test.vmList)
+		assert.NoError(t, err, test.description)
 
 		real, err := ss.GetInstanceIDByNodeName(test.nodeName)
 		if test.expectError {