Merge pull request #16271 from jayunit100/hammer-backoff-metrics-part1

pkg/client/metrics/metrics.go

@@ -38,6 +38,15 @@ var (
 		},
 		[]string{"verb", "url"},
 	)
+
+	RequestResult = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: restClientSubsystem,
+			Name:      "request_status_codes",
+			Help:      "Number of http requests, partitioned by metadata",
+		},
+		[]string{"code", "method", "host"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -48,6 +57,7 @@ func Register() {
 	// Register the metrics.
 	registerMetrics.Do(func() {
 		prometheus.MustRegister(RequestLatency)
+		prometheus.MustRegister(RequestResult)
 	})
 }
 

pkg/client/unversioned/request.go

@@ -586,6 +586,7 @@ func (r *Request) Watch() (watch.Interface, error) {
 		client = http.DefaultClient
 	}
 	resp, err := client.Do(req)
+	updateURLMetrics(r, resp, err)
 	if err != nil {
 		// The watch stream mechanism handles many common partial data errors, so closed
 		// connections can be retried in many cases.
@@ -603,6 +604,23 @@ func (r *Request) Watch() (watch.Interface, error) {
 	return watch.NewStreamWatcher(watchjson.NewDecoder(resp.Body, r.codec)), nil
 }
 
+// updateURLMetrics is a convenience function for pushing metrics.
+// It also handles corner cases for incomplete/invalid request data.
+func updateURLMetrics(req *Request, resp *http.Response, err error) {
+	url := "none"
+	if req.baseURL != nil {
+		url = req.baseURL.Host
+	}
+
+	// If we have an error (i.e. apiserver down) we report that as a metric label.
+	if err != nil {
+		metrics.RequestResult.WithLabelValues(err.Error(), req.verb, url).Inc()
+	} else {
+		//Metrics for failure codes
+		metrics.RequestResult.WithLabelValues(strconv.Itoa(resp.StatusCode), req.verb, url).Inc()
+	}
+}
+
 // Stream formats and executes the request, and offers streaming of the response.
 // Returns io.ReadCloser which could be used for streaming of the response, or an error
 // Any non-2xx http status code causes an error.  If we get a non-2xx code, we try to convert the body into an APIStatus object.
@@ -621,6 +639,7 @@ func (r *Request) Stream() (io.ReadCloser, error) {
 		client = http.DefaultClient
 	}
 	resp, err := client.Do(req)
+	updateURLMetrics(r, resp, err)
 	if err != nil {
 		return nil, err
 	}
@@ -657,6 +676,12 @@ func (r *Request) Stream() (io.ReadCloser, error) {
 // fn at most once. It will return an error if a problem occurred prior to connecting to the
 // server - the provided function is responsible for handling server errors.
 func (r *Request) request(fn func(*http.Request, *http.Response)) error {
+	//Metrics for total request latency
+	start := time.Now()
+	defer func() {
+		metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start))
+	}()
+
 	if r.err != nil {
 		return r.err
 	}
@@ -687,6 +712,7 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error {
 		req.Header = r.headers
 
 		resp, err := client.Do(req)
+		updateURLMetrics(r, resp, err)
 		if err != nil {
 			return err
 		}
@@ -720,10 +746,6 @@ func (r *Request) request(fn func(*http.Request, *http.Response)) error {
 //  * If the server responds with a status: *errors.StatusError or *errors.UnexpectedObjectError
 //  * http.Client.Do errors are returned directly.
 func (r *Request) Do() Result {
-	start := time.Now()
-	defer func() {
-		metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start))
-	}()
 	var result Result
 	err := r.request(func(req *http.Request, resp *http.Response) {
 		result = r.transformResponse(resp, req)
@@ -736,10 +758,6 @@ func (r *Request) Do() Result {
 
 // DoRaw executes the request but does not process the response body.
 func (r *Request) DoRaw() ([]byte, error) {
-	start := time.Now()
-	defer func() {
-		metrics.RequestLatency.WithLabelValues(r.verb, r.finalURLTemplate()).Observe(metrics.SinceInMicroseconds(start))
-	}()
 	var result Result
 	err := r.request(func(req *http.Request, resp *http.Response) {
 		result.body, result.err = ioutil.ReadAll(resp.Body)