Skip to content

K
k3s
Commit f1fef11b authored by Jacob Simpson's avatar Jacob Simpson
Browse files
Options

Add a kubelet metric to track certificate expiration.

parent ad6c85ca
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 4 deletions
pkg/kubelet/certificate/BUILD
View file @ f1fef11b
......@@ -16,8 +16,10 @@ go_library(
],
deps = [
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/metrics:go_default_library",
"//pkg/util/file:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
......@@ -41,6 +43,7 @@ go_test(
],
library = ":go_default_library",
deps = [
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
......
pkg/kubelet/certificate/certificate_manager.go
View file @ f1fef11b
......@@ -28,6 +28,7 @@ import (
"time"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
certificates "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
......@@ -36,10 +37,13 @@ import (
"k8s.io/apimachinery/pkg/watch"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
"k8s.io/client-go/util/cert"
"k8s.io/kubernetes/pkg/kubelet/metrics"
)
const (
syncPeriod = 1 * time.Hour
syncPeriod = 1 * time.Hour
certificateManagerSubsystem = "certificate_manager"
certificateExpirationKey = "expiration_seconds"
)
// Manager maintains and updates the certificates in use by this certificate
......@@ -59,6 +63,10 @@ type Manager interface {
// Config is the set of configuration parameters available for a new Manager.
type Config struct {
// Name is a name describing the certificate being managed by this
// certificate manager. It will be used for recording metrics relevant to
// the certificate.
Name string
// CertificateSigningRequestClient will be used for signing new certificate
// requests generated when a key rotation occurs. It must be set either at
// initialization or by using CertificateSigningRequestClient before
......@@ -128,12 +136,17 @@ type manager struct {
cert *tls.Certificate
rotationDeadline time.Time
forceRotation bool
certificateExpiration prometheus.Gauge
}
// NewManager returns a new certificate manager. A certificate manager is
// responsible for being the authoritative source of certificates in the
// Kubelet and handling updates due to rotation.
func NewManager(config *Config) (Manager, error) {
if config.Name == "" {
return nil, fmt.Errorf("the 'Name' is required to disambiguate metric values of different certificate manager instances")
}
cert, forceRotation, err := getCurrentCertificateOrBootstrap(
config.CertificateStore,
config.BootstrapCertificatePEM,
......@@ -142,6 +155,17 @@ func NewManager(config *Config) (Manager, error) {
return nil, err
}
var certificateExpiration = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: metrics.KubeletSubsystem,
Subsystem: certificateManagerSubsystem,
Name: fmt.Sprintf("%s_%s", config.Name, certificateExpirationKey),
Help: "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
},
)
prometheus.MustRegister(certificateExpiration)
m := manager{
certSigningRequestClient: config.CertificateSigningRequestClient,
template: config.Template,
......@@ -149,6 +173,7 @@ func NewManager(config *Config) (Manager, error) {
certStore: config.CertificateStore,
cert: cert,
forceRotation: forceRotation,
certificateExpiration: certificateExpiration,
}
return &m, nil
......@@ -319,7 +344,8 @@ func (m *manager) setRotationDeadline() {
jitteryDuration := wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3)
m.rotationDeadline = m.cert.Leaf.NotBefore.Add(jitteryDuration)
glog.V(2).Infof("Certificate rotation deadline is %v", m.rotationDeadline)
glog.V(2).Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, m.rotationDeadline)
m.certificateExpiration.Set(float64(notAfter.Unix()))
}
func (m *manager) updateCached(cert *tls.Certificate) {
......
pkg/kubelet/certificate/certificate_manager_test.go
View file @ f1fef11b
......@@ -26,6 +26,8 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
certificates "k8s.io/api/certificates/v1beta1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
watch "k8s.io/apimachinery/pkg/watch"
......@@ -135,6 +137,7 @@ func TestNewManagerNoRotation(t *testing.T) {
cert: storeCertData.certificate,
}
if _, err := NewManager(&Config{
Name: "test_no_rotation",
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
CertificateStore: store,
......@@ -170,6 +173,11 @@ func TestShouldRotate(t *testing.T) {
},
template: &x509.CertificateRequest{},
usages: []certificates.KeyUsage{},
certificateExpiration: prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "test_gauge_name",
},
),
}
m.setRotationDeadline()
if m.shouldRotate() != test.shouldRotate {
......@@ -212,6 +220,11 @@ func TestSetRotationDeadline(t *testing.T) {
},
template: &x509.CertificateRequest{},
usages: []certificates.KeyUsage{},
certificateExpiration: prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "test_gauge_name",
},
),
}
lowerBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.7))
upperBound := tc.notBefore.Add(time.Duration(float64(tc.notAfter.Sub(tc.notBefore)) * 0.9))
......@@ -282,6 +295,7 @@ func TestNewManagerBootstrap(t *testing.T) {
var cm Manager
cm, err := NewManager(&Config{
Name: "test_bootstrap",
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
CertificateStore: store,
......@@ -319,6 +333,7 @@ func TestNewManagerNoBootstrap(t *testing.T) {
}
cm, err := NewManager(&Config{
Name: "test_no_bootstrap",
Template: &x509.CertificateRequest{},
Usages: []certificates.KeyUsage{},
CertificateStore: store,
......@@ -454,13 +469,14 @@ func TestInitializeCertificateSigningRequestClient(t *testing.T) {
},
}
for _, tc := range testCases {
for i, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
certificateStore := &fakeStore{
cert: tc.storeCert.certificate,
}
certificateManager, err := NewManager(&Config{
Name: fmt.Sprintf("test_initialize_client_%d", i),
Template: &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{"system:nodes"},
......@@ -555,13 +571,14 @@ func TestInitializeOtherRESTClients(t *testing.T) {
},
}
for _, tc := range testCases {
for i, tc := range testCases {
t.Run(tc.description, func(t *testing.T) {
certificateStore := &fakeStore{
cert: tc.storeCert.certificate,
}
certificateManager, err := NewManager(&Config{
Name: fmt.Sprintf("test_initialize_other_rest_clients_%d", i),
Template: &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{"system:nodes"},
......
pkg/kubelet/certificate/kubelet.go
View file @ f1fef11b
......@@ -46,6 +46,7 @@ func NewKubeletServerCertificateManager(kubeClient clientset.Interface, kubeCfg
return nil, fmt.Errorf("failed to initialize server certificate store: %v", err)
}
m, err := NewManager(&Config{
Name: "server",
CertificateSigningRequestClient: certSigningRequestClient,
Template: &x509.CertificateRequest{
Subject: pkix.Name{
......@@ -92,6 +93,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
return nil, fmt.Errorf("failed to initialize client certificate store: %v", err)
}
m, err := NewManager(&Config{
Name: "client",
Template: &x509.CertificateRequest{
Subject: pkix.Name{
CommonName: fmt.Sprintf("system:node:%s", nodeName),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment