Add system namespaces to admission metrics. Add tests and leverage test code from PR#55086

plugin/pkg/admission/namespace/autoprovision/admission_test.go

@@ -136,12 +136,9 @@ func TestIgnoreAdmission(t *testing.T) {
 		t.Errorf("unexpected error initializing handler: %v", err)
 	}
 	informerFactory.Start(wait.NeverStop)
-	chainHandler := admission.NewChainHandler(handler)
 
-	pod := newPod(namespace)
-	err = chainHandler.Admit(admission.NewAttributesRecord(&pod, nil, api.Kind("Pod").WithVersion("version"), pod.Namespace, pod.Name, api.Resource("pods").WithVersion("version"), "", admission.Update, nil))
-	if err != nil {
-		t.Errorf("unexpected error returned from admission handler")
+	if handler.Handles(admission.Update) {
+		t.Errorf("expected not to handle Update")
 	}
 	if hasCreateNamespaceAction(mockClient) {
 		t.Errorf("unexpected create namespace action")

plugin/pkg/admission/persistentvolume/label/admission_test.go

@@ -77,8 +77,7 @@ func mockVolumeLabels(labels map[string]string) *mockVolumes {
 
 // TestAdmission
 func TestAdmission(t *testing.T) {
-	pvHandler := NewPersistentVolumeLabel()
-	handler := admission.NewChainHandler(pvHandler)
+	handler := NewPersistentVolumeLabel()
 	ignoredPV := api.PersistentVolume{
 		ObjectMeta: metav1.ObjectMeta{Name: "noncloud", Namespace: "myns"},
 		Spec: api.PersistentVolumeSpec{
@@ -101,9 +100,8 @@ func TestAdmission(t *testing.T) {
 	}
 
 	// Non-cloud PVs are ignored
-	err := handler.Admit(admission.NewAttributesRecord(&ignoredPV, nil, api.Kind("PersistentVolume").WithVersion("version"), ignoredPV.Namespace, ignoredPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, nil))
-	if err != nil {
-		t.Errorf("Unexpected error returned from admission handler (on ignored pv): %v", err)
+	if handler.Handles(admission.Delete) {
+		t.Errorf("Expected to only handle create")
 	}
 
 	// We only add labels on creation
@@ -113,7 +111,7 @@ func TestAdmission(t *testing.T) {
 	}
 
 	// Errors from the cloudprovider block creation of the volume
-	pvHandler.ebsVolumes = mockVolumeFailure(fmt.Errorf("invalid volume"))
+	handler.ebsVolumes = mockVolumeFailure(fmt.Errorf("invalid volume"))
 	err = handler.Admit(admission.NewAttributesRecord(&awsPV, nil, api.Kind("PersistentVolume").WithVersion("version"), awsPV.Namespace, awsPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, nil))
 	if err == nil {
 		t.Errorf("Expected error when aws pv info fails")
@@ -121,7 +119,7 @@ func TestAdmission(t *testing.T) {
 
 	// Don't add labels if the cloudprovider doesn't return any
 	labels := make(map[string]string)
-	pvHandler.ebsVolumes = mockVolumeLabels(labels)
+	handler.ebsVolumes = mockVolumeLabels(labels)
 	err = handler.Admit(admission.NewAttributesRecord(&awsPV, nil, api.Kind("PersistentVolume").WithVersion("version"), awsPV.Namespace, awsPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, nil))
 	if err != nil {
 		t.Errorf("Expected no error when creating aws pv")
@@ -131,7 +129,7 @@ func TestAdmission(t *testing.T) {
 	}
 
 	// Don't panic if the cloudprovider returns nil, nil
-	pvHandler.ebsVolumes = mockVolumeFailure(nil)
+	handler.ebsVolumes = mockVolumeFailure(nil)
 	err = handler.Admit(admission.NewAttributesRecord(&awsPV, nil, api.Kind("PersistentVolume").WithVersion("version"), awsPV.Namespace, awsPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, nil))
 	if err != nil {
 		t.Errorf("Expected no error when cloud provider returns empty labels")
@@ -141,7 +139,7 @@ func TestAdmission(t *testing.T) {
 	labels = make(map[string]string)
 	labels["a"] = "1"
 	labels["b"] = "2"
-	pvHandler.ebsVolumes = mockVolumeLabels(labels)
+	handler.ebsVolumes = mockVolumeLabels(labels)
 	err = handler.Admit(admission.NewAttributesRecord(&awsPV, nil, api.Kind("PersistentVolume").WithVersion("version"), awsPV.Namespace, awsPV.Name, api.Resource("persistentvolumes").WithVersion("version"), "", admission.Create, nil))
 	if err != nil {
 		t.Errorf("Expected no error when creating aws pv")

plugin/pkg/admission/serviceaccount/admission_test.go

@@ -35,13 +35,10 @@ import (
 )
 
 func TestIgnoresNonCreate(t *testing.T) {
-	pod := &api.Pod{}
 	for _, op := range []admission.Operation{admission.Delete, admission.Connect} {
-		attrs := admission.NewAttributesRecord(pod, nil, api.Kind("Pod").WithVersion("version"), "myns", "myname", api.Resource("pods").WithVersion("version"), "", op, nil)
-		handler := admission.NewChainHandler(NewServiceAccount())
-		err := handler.Admit(attrs)
-		if err != nil {
-			t.Errorf("Expected %s operation allowed, got err: %v", op, err)
+		handler := NewServiceAccount()
+		if handler.Handles(op) {
+			t.Errorf("Expected not to handle operation %s", op)
 		}
 	}
 }
@@ -50,7 +47,7 @@ func TestIgnoresUpdateOfInitializedPod(t *testing.T) {
 	pod := &api.Pod{}
 	oldPod := &api.Pod{}
 	attrs := admission.NewAttributesRecord(pod, oldPod, api.Kind("Pod").WithVersion("version"), "myns", "myname", api.Resource("pods").WithVersion("version"), "", admission.Update, nil)
-	handler := admission.NewChainHandler(NewServiceAccount())
+	handler := NewServiceAccount()
 	err := handler.Admit(attrs)
 	if err != nil {
 		t.Errorf("Expected update of initialized pod allowed, got err: %v", err)

staging/src/k8s.io/apiserver/pkg/admission/BUILD

@@ -13,10 +13,17 @@ go_test(
         "config_test.go",
         "errors_test.go",
         "handler_test.go",
+        "metrics_test.go",
+        "testutil_test.go",
     ],
     importpath = "k8s.io/apiserver/pkg/admission",
     library = ":go_default_library",
     deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/prometheus/client_model/go:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/apis/apiserver:go_default_library",
@@ -45,6 +52,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/announced:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apimachinery/registered:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",

staging/src/k8s.io/apiserver/pkg/admission/chain.go

@@ -37,7 +37,7 @@ func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error {
 	var err error
 	start := time.Now()
 	defer func() {
-		ObserveAdmissionStep(time.Since(start), err != nil, a, stepMutating)
+		Metrics.ObserveAdmissionStep(time.Since(start), err != nil, a, stepMutating)
 	}()
 
 	for _, handler := range admissionHandler {
@@ -47,7 +47,7 @@ func (admissionHandler chainAdmissionHandler) Admit(a Attributes) error {
 		if mutator, ok := handler.(MutationInterface); ok {
 			t := time.Now()
 			err = mutator.Admit(a)
-			ObserveAdmissionController(time.Since(t), err != nil, handler, a)
+			Metrics.ObserveAdmissionController(time.Since(t), err != nil, handler, a)
 			if err != nil {
 				return err
 			}
@@ -61,7 +61,7 @@ func (admissionHandler chainAdmissionHandler) Validate(a Attributes) error {
 	var err error
 	start := time.Now()
 	defer func() {
-		ObserveAdmissionStep(time.Since(start), err != nil, a, stepValidating)
+		Metrics.ObserveAdmissionStep(time.Since(start), err != nil, a, stepValidating)
 	}()
 
 	for _, handler := range admissionHandler {
@@ -71,7 +71,7 @@ func (admissionHandler chainAdmissionHandler) Validate(a Attributes) error {
 		if validator, ok := handler.(ValidationInterface); ok {
 			t := time.Now()
 			err = validator.Validate(a)
-			ObserveAdmissionController(time.Since(t), err != nil, handler, a)
+			Metrics.ObserveAdmissionController(time.Since(t), err != nil, handler, a)
 			if err != nil {
 				return err
 			}

staging/src/k8s.io/apiserver/pkg/admission/chain_test.go

@@ -56,17 +56,22 @@ func makeHandler(name string, accept bool, ops ...Operation) Interface {
 }
 
 func TestAdmitAndValidate(t *testing.T) {
+	sysns := "kube-system"
+	otherns := "default"
 	tests := []struct {
 		name      string
+		ns        string
 		operation Operation
 		chain     chainAdmissionHandler
 		accept    bool
+		reject    bool
 		calls     map[string]bool
 	}{
 		{
 			name:      "all accept",
+			ns:        sysns,
 			operation: Create,
-			chain: []Interface{
+			chain: []NamedHandler{
 				makeHandler("a", true, Update, Delete, Create),
 				makeHandler("b", true, Delete, Create),
 				makeHandler("c", true, Create),
@@ -76,8 +81,9 @@ func TestAdmitAndValidate(t *testing.T) {
 		},
 		{
 			name:      "ignore handler",
+			ns:        otherns,
 			operation: Create,
-			chain: []Interface{
+			chain: []NamedHandler{
 				makeHandler("a", true, Update, Delete, Create),
 				makeHandler("b", false, Delete),
 				makeHandler("c", true, Create),
@@ -87,8 +93,9 @@ func TestAdmitAndValidate(t *testing.T) {
 		},
 		{
 			name:      "ignore all",
+			ns:        sysns,
 			operation: Connect,
-			chain: []Interface{
+			chain: []NamedHandler{
 				makeHandler("a", true, Update, Delete, Create),
 				makeHandler("b", false, Delete),
 				makeHandler("c", true, Create),
@@ -98,22 +105,26 @@ func TestAdmitAndValidate(t *testing.T) {
 		},
 		{
 			name:      "reject one",
+			ns:        otherns,
 			operation: Delete,
-			chain: []Interface{
+			chain: []NamedHandler{
 				makeHandler("a", true, Update, Delete, Create),
 				makeHandler("b", false, Delete),
 				makeHandler("c", true, Create),
 			},
 			calls:  map[string]bool{"a": true, "b": true},
 			accept: false,
+			reject: true,
 		},
 	}
 	for _, test := range tests {
+		Metrics.reset()
+		t.Logf("testcase = %s", test.name)
 		// call admit and check that validate was not called at all
-		err := test.chain.Admit(NewAttributesRecord(nil, nil, schema.GroupVersionKind{}, "", "", schema.GroupVersionResource{}, "", test.operation, nil))
+		err = test.chain.Admit(NewAttributesRecord(nil, nil, schema.GroupVersionKind{}, "", "", schema.GroupVersionResource{}, "", test.operation, nil))
 		accepted := (err == nil)
 		if accepted != test.accept {
-			t.Errorf("%s: unexpected result of admit call: %v\n", test.name, accepted)
+			t.Errorf("unexpected result of admit call: %v", accepted)
 		}
 		for _, h := range test.chain {
 			fake := h.(*FakeHandler)
@@ -148,6 +159,18 @@ func TestAdmitAndValidate(t *testing.T) {
 				t.Errorf("%s: admit handler %s called during admit", test.name, fake.name)
 			}
 		}
+
+		labels := metricLabels{
+			isSystemNs: test.ns == sysns,
+		}
+		if test.reject {
+			expectCountMetric(t, "apiserver_admission_step_rejected_total", labels, 1)
+		}
+
+		if test.accept {
+			expectCountMetric(t, "apiserver_admission_step_total", labels, 1)
+		}
+
 	}
 }
 

staging/src/k8s.io/apiserver/pkg/admission/metrics.go

@@ -18,11 +18,14 @@ package admission
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"k8s.io/api/admissionregistration/v1alpha1"
 
 	"github.com/prometheus/client_golang/prometheus"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
@@ -34,26 +37,42 @@ var (
 	latencyBuckets       = prometheus.ExponentialBuckets(10000, 2.0, 8)
 	latencySummaryMaxAge = 5 * time.Hour
 
-	// Admission step metrics. Each step is identified by a distinct type label value.
-	stepMetrics = newAdmissionMetrics("step_",
-		[]string{"operation", "group", "version", "resource", "subresource", "type"},
+	Metrics = newAdmissionMetrics()
+)
+
+type AdmissionMetrics struct {
+	step            *metricSet
+	controller      *metricSet
+	externalWebhook *metricSet
+}
+
+func newAdmissionMetrics() *AdmissionMetrics {
+	// Admission metrics for a step of the admission flow. The entire admission flow is broken down into a series of steps
+	// Each step is identified by a distinct type label value.
+	step := newMetricSet("step_",
+		[]string{"operation", "group", "version", "resource", "subresource", "type", "is_system_ns"},
 		"Admission sub-step %s, broken out for each operation and API resource and step type (validating or mutating).")
 
-	// Build-in admission controller metrics. Each admission controller is identified by name.
-	controllerMetrics = newAdmissionMetrics("controller_",
-		[]string{"name", "type", "operation", "group", "version", "resource", "subresource"},
+	// Built-in admission controller metrics. Each admission controller is identified by name.
+	controller := newMetricSet("controller_",
+		[]string{"name", "type", "operation", "group", "version", "resource", "subresource", "is_system_ns"},
 		"Admission controller %s, identified by name and broken out for each operation and API resource and type (validating or mutating).")
 
 	// External admission webhook metrics. Each webhook is identified by name.
-	externalWebhookMetrics = newAdmissionMetrics("external_webhook_",
-		[]string{"name", "type", "operation", "group", "version", "resource", "subresource"},
+	externalWebhook := newMetricSet("external_webhook_",
+		[]string{"name", "type", "operation", "group", "version", "resource", "subresource", "is_system_ns"},
 		"External admission webhook %s, identified by name and broken out for each operation and API resource and type (validating or mutating).")
-)
 
-func init() {
-	stepMetrics.mustRegister()
-	controllerMetrics.mustRegister()
-	externalWebhookMetrics.mustRegister()
+	step.mustRegister()
+	controller.mustRegister()
+	externalWebhook.mustRegister()
+	return &AdmissionMetrics{step: step, controller: controller, externalWebhook: externalWebhook}
+}
+
+func (m *AdmissionMetrics) reset() {
+	m.step.reset()
+	m.controller.reset()
+	m.externalWebhook.reset()
 }
 
 // namedHandler requires each admission.Interface be named, primarly for metrics tracking purposes.
@@ -63,51 +82,63 @@ type NamedHandler interface {
 }
 
 // ObserveAdmissionStep records admission related metrics for a admission step, identified by step type.
-func ObserveAdmissionStep(elapsed time.Duration, rejected bool, attr Attributes, stepType string) {
+func (m *AdmissionMetrics) ObserveAdmissionStep(elapsed time.Duration, rejected bool, attr Attributes, stepType string) {
 	gvr := attr.GetResource()
-	stepMetrics.observe(elapsed, rejected, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), stepType)
+	m.step.observe(elapsed, rejected, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), stepType, isSystemNsLabel(attr))
 }
 
-// ObserveAdmissionController records admission related metrics for a build-in admission controller, identified by it's plugin handler name.
-func ObserveAdmissionController(elapsed time.Duration, rejected bool, handler NamedHandler, attr Attributes) {
+// ObserveAdmissionController records admission related metrics for a built-in admission controller, identified by it's plugin handler name.
+func (m *AdmissionMetrics) ObserveAdmissionController(elapsed time.Duration, rejected bool, handler NamedHandler, attr Attributes) {
 	t := typeToLabel(handler)
 	gvr := attr.GetResource()
-	controllerMetrics.observe(elapsed, rejected, handler.GetName(), t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource())
+	m.controller.observe(elapsed, rejected, handler.GetName(), t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), isSystemNsLabel(attr))
 }
 
 // ObserveExternalWebhook records admission related metrics for a external admission webhook.
-func ObserveExternalWebhook(elapsed time.Duration, rejected bool, hook *v1alpha1.ExternalAdmissionHook, attr Attributes) {
+func (m *AdmissionMetrics) ObserveExternalWebhook(elapsed time.Duration, rejected bool, hook *v1alpha1.ExternalAdmissionHook, attr Attributes) {
 	t := "validating" // TODO: pass in type (validating|mutating) once mutating webhook functionality has been implemented
 	gvr := attr.GetResource()
-	externalWebhookMetrics.observe(elapsed, rejected, hook.Name, t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource())
+	m.externalWebhook.observe(elapsed, rejected, hook.Name, t, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), isSystemNsLabel(attr))
+}
+
+// isSystemNsLabel returns the value to use for the `is_system_ns` metric label.
+func isSystemNsLabel(a Attributes) string {
+	return strconv.FormatBool(a.GetNamespace() == metav1.NamespaceSystem)
 }
 
 func typeToLabel(i Interface) string {
 	switch i.(type) {
-	case ValidationInterface:
-		return "validating"
 	case MutationInterface:
 		return "mutating"
+	case ValidationInterface:
+		return "validating"
 	default:
 		return "UNRECOGNIZED_ADMISSION_TYPE"
 	}
 }
 
-type admissionMetrics struct {
+type metricSet struct {
 	total            *prometheus.CounterVec
 	rejectedTotal    *prometheus.CounterVec
 	latencies        *prometheus.HistogramVec
 	latenciesSummary *prometheus.SummaryVec
 }
 
-func (m *admissionMetrics) mustRegister() {
+func (m *metricSet) mustRegister() {
 	prometheus.MustRegister(m.total)
 	prometheus.MustRegister(m.rejectedTotal)
 	prometheus.MustRegister(m.latencies)
 	prometheus.MustRegister(m.latenciesSummary)
 }
 
-func (m *admissionMetrics) observe(elapsed time.Duration, rejected bool, labels ...string) {
+func (m *metricSet) reset() {
+	m.total.Reset()
+	m.rejectedTotal.Reset()
+	m.latencies.Reset()
+	m.latenciesSummary.Reset()
+}
+
+func (m *metricSet) observe(elapsed time.Duration, rejected bool, labels ...string) {
 	elapsedMicroseconds := float64(elapsed / time.Microsecond)
 	m.total.WithLabelValues(labels...).Inc()
 	if rejected {
@@ -117,8 +148,8 @@ func (m *admissionMetrics) observe(elapsed time.Duration, rejected bool, labels
 	m.latenciesSummary.WithLabelValues(labels...).Observe(elapsedMicroseconds)
 }
 
-func newAdmissionMetrics(name string, labels []string, helpTemplate string) *admissionMetrics {
-	return &admissionMetrics{
+func newMetricSet(name string, labels []string, helpTemplate string) *metricSet {
+	return &metricSet{
 		total: prometheus.NewCounterVec(
 			prometheus.CounterOpts{
 				Namespace: namespace,

staging/src/k8s.io/apiserver/pkg/admission/metrics_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"testing"
+	"time"
+
+	"k8s.io/api/admissionregistration/v1alpha1"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+var (
+	kind     = schema.GroupVersionKind{Group: "kgroup", Version: "kversion", Kind: "kind"}
+	resource = schema.GroupVersionResource{Group: "rgroup", Version: "rversion", Resource: "resource"}
+	attr     = NewAttributesRecord(nil, nil, kind, "ns", "name", resource, "subresource", Create, nil)
+)
+
+func TestObserveAdmissionStep(t *testing.T) {
+	Metrics.reset()
+	Metrics.ObserveAdmissionStep(2*time.Second, false, attr, "mutating")
+	wantLabels := metricLabels{
+		operation:   string(Create),
+		group:       resource.Group,
+		version:     resource.Version,
+		resource:    resource.Resource,
+		subresource: "subresource",
+		tpe:         "mutating",
+		isSystemNs:  false,
+	}
+	expectCountMetric(t, "apiserver_admission_step_total", wantLabels, 1)
+}
+
+func TestObserveAdmissionController(t *testing.T) {
+	Metrics.reset()
+	handler := makeHandler("a", true, Create)
+	Metrics.ObserveAdmissionController(2*time.Second, false, handler, attr)
+	wantLabels := metricLabels{
+		name:        "a",
+		operation:   string(Create),
+		group:       resource.Group,
+		version:     resource.Version,
+		resource:    resource.Resource,
+		subresource: "subresource",
+		tpe:         "mutating",
+		isSystemNs:  false,
+	}
+	expectCountMetric(t, "apiserver_admission_controller_total", wantLabels, 1)
+}
+
+func TestObserveExternalWebhook(t *testing.T) {
+	Metrics.reset()
+	hook := &v1alpha1.ExternalAdmissionHook{Name: "x"}
+	Metrics.ObserveExternalWebhook(2*time.Second, false, hook, attr)
+	wantLabels := metricLabels{
+		name:        "x",
+		operation:   string(Create),
+		group:       resource.Group,
+		version:     resource.Version,
+		resource:    resource.Resource,
+		subresource: "subresource",
+		tpe:         "validating",
+		isSystemNs:  false,
+	}
+	expectCountMetric(t, "apiserver_admission_external_webhook_total", wantLabels, 1)
+}

staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/admission.go

@@ -309,7 +309,7 @@ func (a *GenericAdmissionWebhook) Admit(attr admission.Attributes) error {
 
 			t := time.Now()
 			err := a.callHook(ctx, hook, versionedAttr)
-			admission.ObserveExternalWebhook(time.Since(t), err != nil, hook, attr)
+			admission.Metrics.ObserveExternalWebhook(time.Since(t), err != nil, hook, attr)
 			if err == nil {
 				return
 			}

staging/src/k8s.io/apiserver/pkg/admission/testutil_test.go

+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package admission
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	ptype "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type FakeHandler struct {
+	*Handler
+	name                     string
+	admit, admitCalled       bool
+	validate, validateCalled bool
+}
+
+func (h *FakeHandler) GetName() string {
+	return h.name
+}
+
+func (h *FakeHandler) Admit(a Attributes) (err error) {
+	h.admitCalled = true
+	if h.admit {
+		return nil
+	}
+	return fmt.Errorf("Don't admit")
+}
+
+func (h *FakeHandler) Validate(a Attributes) (err error) {
+	h.admitCalled = true
+	if h.admit {
+		return nil
+	}
+	return fmt.Errorf("Don't admit")
+}
+
+// makeHandler creates a mock handler for testing purposes.
+func makeHandler(name string, admit bool, ops ...Operation) *FakeHandler {
+	return &FakeHandler{
+		name:    name,
+		admit:   admit,
+		Handler: NewHandler(ops...),
+	}
+}
+
+type metricLabels struct {
+	operation   string
+	group       string
+	version     string
+	resource    string
+	subresource string
+	name        string
+	tpe         string
+	isSystemNs  bool
+}
+
+// matches checks if the reciever matches the pattern. Empty strings in the pattern are treated as wildcards.
+func (l metricLabels) matches(pattern metricLabels) bool {
+	return matches(l.operation, pattern.operation) &&
+		matches(l.group, pattern.group) &&
+		matches(l.version, pattern.version) &&
+		matches(l.resource, pattern.resource) &&
+		matches(l.subresource, pattern.subresource) &&
+		matches(l.tpe, pattern.tpe) &&
+		l.isSystemNs == pattern.isSystemNs
+}
+
+// matches checks if a string matches a "pattern" string, where an empty pattern string is treated as a wildcard.
+func matches(s string, pattern string) bool {
+	return pattern == "" || s == pattern
+}
+
+// readLabels marshalls the labels from a prometheus metric type to a simple struct, producing test errors if
+// if any unrecognized labels are encountered.
+func readLabels(t *testing.T, metric *ptype.Metric) metricLabels {
+	l := metricLabels{}
+	for _, lp := range metric.GetLabel() {
+		val := lp.GetValue()
+		switch lp.GetName() {
+		case "operation":
+			l.operation = val
+		case "group":
+			l.group = val
+		case "version":
+			l.version = val
+		case "resource":
+			l.resource = val
+		case "subresource":
+			l.subresource = val
+		case "name":
+			l.name = val
+		case "type":
+			l.tpe = val
+		case "is_system_ns":
+			ns, err := strconv.ParseBool(lp.GetValue())
+			if err != nil {
+				t.Errorf("Expected boole for is_system_ns label value, got %s", lp.GetValue())
+			} else {
+				l.isSystemNs = ns
+			}
+		default:
+			t.Errorf("Unexpected metric label %s", lp.GetName())
+		}
+	}
+	return l
+}
+
+// expectCounterMetric ensures that exactly one counter metric with the given name and patternLabels exists and has
+// the provided count.
+func expectCountMetric(t *testing.T, name string, patternLabels metricLabels, wantCount int64) {
+	metrics, err := prometheus.DefaultGatherer.Gather()
+	require.NoError(t, err)
+
+	count := 0
+	for _, mf := range metrics {
+		if mf.GetName() != name {
+			continue // Ignore other metrics.
+		}
+		for _, metric := range mf.GetMetric() {
+			gotLabels := readLabels(t, metric)
+			if !gotLabels.matches(patternLabels) {
+				continue
+			}
+			count += 1
+			assert.EqualValues(t, wantCount, metric.GetCounter().GetValue())
+		}
+	}
+	if count != 1 {
+		t.Errorf("Want 1 metric with name %s, got %d", name, count)
+	}
+}