Merge pull request #58204 from cosmincojocar/monitoring_wrapped_azure_clients

pkg/cloudprovider/providers/azure/BUILD

@@ -19,6 +19,7 @@ go_library(
         "azure_instances.go",
         "azure_loadbalancer.go",
         "azure_managedDiskController.go",
+        "azure_metrics.go",
         "azure_routes.go",
         "azure_storage.go",
         "azure_storageaccount.go",
@@ -48,6 +49,7 @@ go_library(
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
@@ -63,6 +65,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "azure_loadbalancer_test.go",
+        "azure_metrics_test.go",
         "azure_test.go",
         "azure_util_cache_test.go",
         "azure_util_test.go",

pkg/cloudprovider/providers/azure/azure_backoff.go

@@ -89,7 +89,7 @@ func (az *Cloud) VirtualMachineClientListWithRetry() ([]compute.VirtualMachine,
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.VirtualMachinesClient.ListNextResults(result)
+				result, retryErr = az.VirtualMachinesClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("VirtualMachinesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						az.ResourceGroup, retryErr)
@@ -176,7 +176,7 @@ func (az *Cloud) ListLBWithRetry() ([]network.LoadBalancer, error) {
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.LoadBalancerClient.ListNextResults(result)
+				result, retryErr = az.LoadBalancerClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("LoadBalancerClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						az.ResourceGroup,
@@ -225,7 +225,7 @@ func (az *Cloud) ListPIPWithRetry(pipResourceGroup string) ([]network.PublicIPAd
 		if result.NextLink != nil {
 			err := wait.ExponentialBackoff(az.requestBackoff(), func() (bool, error) {
 				var retryErr error
-				result, retryErr = az.PublicIPAddressesClient.ListNextResults(result)
+				result, retryErr = az.PublicIPAddressesClient.ListNextResults(az.ResourceGroup, result)
 				if retryErr != nil {
 					glog.Errorf("PublicIPAddressesClient.ListNextResults(%v) - backoff: failure, will retry,err=%v",
 						pipResourceGroup,

pkg/cloudprovider/providers/azure/azure_fakes.go

@@ -144,7 +144,7 @@ func (fLBC fakeAzureLBClient) List(resourceGroupName string) (result network.Loa
 	return result, nil
 }
 
-func (fLBC fakeAzureLBClient) ListNextResults(lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
+func (fLBC fakeAzureLBClient) ListNextResults(resourceGroupName string, lastResult network.LoadBalancerListResult) (result network.LoadBalancerListResult, err error) {
 	fLBC.mutex.Lock()
 	defer fLBC.mutex.Unlock()
 	result.Response.Response = &http.Response{
@@ -264,7 +264,7 @@ func (fAPC fakeAzurePIPClient) Get(resourceGroupName string, publicIPAddressName
 	}
 }
 
-func (fAPC fakeAzurePIPClient) ListNextResults(lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
+func (fAPC fakeAzurePIPClient) ListNextResults(resourceGroupName string, lastResults network.PublicIPAddressListResult) (result network.PublicIPAddressListResult, err error) {
 	fAPC.mutex.Lock()
 	defer fAPC.mutex.Unlock()
 	return network.PublicIPAddressListResult{}, nil
@@ -411,7 +411,7 @@ func (fVMC fakeAzureVirtualMachinesClient) List(resourceGroupName string) (resul
 	result.Value = &value
 	return result, nil
 }
-func (fVMC fakeAzureVirtualMachinesClient) ListNextResults(lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
+func (fVMC fakeAzureVirtualMachinesClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineListResult) (result compute.VirtualMachineListResult, err error) {
 	fVMC.mutex.Lock()
 	defer fVMC.mutex.Unlock()
 	return compute.VirtualMachineListResult{}, nil
@@ -659,7 +659,7 @@ func (fVMC fakeVirtualMachineScaleSetVMsClient) List(resourceGroupName string, v
 	return result, nil
 }
 
-func (fVMC fakeVirtualMachineScaleSetVMsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
+func (fVMC fakeVirtualMachineScaleSetVMsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetVMListResult) (result compute.VirtualMachineScaleSetVMListResult, err error) {
 	return result, nil
 }
 
@@ -764,7 +764,7 @@ func (fVMSSC fakeVirtualMachineScaleSetsClient) List(resourceGroupName string) (
 	return result, nil
 }
 
-func (fVMSSC fakeVirtualMachineScaleSetsClient) ListNextResults(lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
+func (fVMSSC fakeVirtualMachineScaleSetsClient) ListNextResults(resourceGroupName string, lastResults compute.VirtualMachineScaleSetListResult) (result compute.VirtualMachineScaleSetListResult, err error) {
 	return result, nil
 }
 

pkg/cloudprovider/providers/azure/azure_metrics.go

+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type apiCallMetrics struct {
+	latency *prometheus.HistogramVec
+	errors  *prometheus.CounterVec
+}
+
+var (
+	metricLabels = []string{
+		"request",         // API function that is being invoked
+		"resource_group",  // Resource group of the resource being monitored
+		"subscription_id", // Subscription ID of the resource being monitored
+	}
+
+	apiMetrics = registerAPIMetrics(metricLabels...)
+)
+
+type metricContext struct {
+	start      time.Time
+	attributes []string
+}
+
+func newMetricContext(prefix, request, resouceGroup, subscriptionID string) *metricContext {
+	return &metricContext{
+		start:      time.Now(),
+		attributes: []string{prefix + "_" + request, resouceGroup, subscriptionID},
+	}
+}
+
+func (mc *metricContext) Observe(err error) {
+	apiMetrics.latency.WithLabelValues(mc.attributes...).Observe(
+		time.Since(mc.start).Seconds())
+	if err != nil {
+		apiMetrics.errors.WithLabelValues(mc.attributes...).Inc()
+	}
+}
+
+func registerAPIMetrics(attributes ...string) *apiCallMetrics {
+	metrics := &apiCallMetrics{
+		latency: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Name: "cloudprovider_azure_api_request_duration_seconds",
+				Help: "Latency of an Azure API call",
+			},
+			attributes,
+		),
+		errors: prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Name: "cloudprovider_azure_api_request_errors",
+				Help: "Number of errors for an Azure API call",
+			},
+			attributes,
+		),
+	}
+
+	prometheus.MustRegister(metrics.latency)
+	prometheus.MustRegister(metrics.errors)
+
+	return metrics
+}

pkg/cloudprovider/providers/azure/azure_metrics_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azure
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAzureMetricLabelCardinality(t *testing.T) {
+	mc := newMetricContext("test", "create", "resource_group", "subscription_id")
+	assert.Len(t, mc.attributes, len(metricLabels), "cardinalities of labels and values must match")
+}
+
+func TestAzureMetricLabelPrefix(t *testing.T) {
+	mc := newMetricContext("prefix", "request", "resource_group", "subscription_id")
+	found := false
+	for _, attribute := range mc.attributes {
+		if attribute == "prefix_request" {
+			found = true
+		}
+	}
+	assert.True(t, found, "request label must be prefixed")
+}

pkg/cloudprovider/providers/azure/azure_util_vmss.go

@@ -422,7 +422,7 @@ func (ss *scaleSet) listScaleSetsWithRetry() ([]string, error) {
 
 		if result.NextLink != nil {
 			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(result)
+				result, err = ss.VirtualMachineScaleSetsClient.ListNextResults(ss.ResourceGroup, result)
 				if err != nil {
 					glog.Errorf("VirtualMachineScaleSetsClient.ListNextResults for %v failed: %v", ss.ResourceGroup, err)
 					return false, err
@@ -468,7 +468,7 @@ func (ss *scaleSet) listScaleSetVMsWithRetry(scaleSetName string) ([]compute.Vir
 
 		if result.NextLink != nil {
 			backoffError := wait.ExponentialBackoff(ss.requestBackoff(), func() (bool, error) {
-				result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(result)
+				result, err = ss.VirtualMachineScaleSetVMsClient.ListNextResults(ss.ResourceGroup, result)
 				if err != nil {
 					glog.Errorf("VirtualMachineScaleSetVMsClient.ListNextResults for %v failed: %v", scaleSetName, err)
 					return false, err