diff --git a/cmd/kube-controller-manager/app/BUILD b/cmd/kube-controller-manager/app/BUILD index d735068257..5ef6c297e4 100644 --- a/cmd/kube-controller-manager/app/BUILD +++ b/cmd/kube-controller-manager/app/BUILD @@ -49,6 +49,7 @@ go_library( "//pkg/controller:go_default_library", "//pkg/controller/bootstrap:go_default_library", "//pkg/controller/certificates/approver:go_default_library", + "//pkg/controller/certificates/cleaner:go_default_library", "//pkg/controller/certificates/signer:go_default_library", "//pkg/controller/cronjob:go_default_library", "//pkg/controller/daemon:go_default_library", diff --git a/cmd/kube-controller-manager/app/certificates.go b/cmd/kube-controller-manager/app/certificates.go index 96819b4b44..b3e4e8499d 100644 --- a/cmd/kube-controller-manager/app/certificates.go +++ b/cmd/kube-controller-manager/app/certificates.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/kubernetes/pkg/controller/certificates/approver" + "k8s.io/kubernetes/pkg/controller/certificates/cleaner" "k8s.io/kubernetes/pkg/controller/certificates/signer" ) @@ -57,10 +58,9 @@ func startCSRApprovingController(ctx ControllerContext) (bool, error) { if !ctx.AvailableResources[schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"}] { return false, nil } - c := ctx.ClientBuilder.ClientOrDie("certificate-controller") approver, err := approver.NewCSRApprovingController( - c, + ctx.ClientBuilder.ClientOrDie("certificate-controller"), ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(), ) if err != nil { @@ -73,3 +73,12 @@ func startCSRApprovingController(ctx ControllerContext) (bool, error) { return true, nil } + +func startCSRCleanerController(ctx ControllerContext) (bool, error) { + cleaner := cleaner.NewCSRCleanerController( + ctx.ClientBuilder.ClientOrDie("certificate-controller").CertificatesV1beta1().CertificateSigningRequests(), + ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(), + ) + go cleaner.Run(1, ctx.Stop) + return true, nil +} diff --git a/cmd/kube-controller-manager/app/controllermanager.go b/cmd/kube-controller-manager/app/controllermanager.go index a737776ccf..6cd1e6925d 100644 --- a/cmd/kube-controller-manager/app/controllermanager.go +++ b/cmd/kube-controller-manager/app/controllermanager.go @@ -348,6 +348,7 @@ func NewControllerInitializers() map[string]InitFunc { controllers["cronjob"] = startCronJobController controllers["csrsigning"] = startCSRSigningController controllers["csrapproving"] = startCSRApprovingController + controllers["csrcleaner"] = startCSRCleanerController controllers["ttl"] = startTTLController controllers["bootstrapsigner"] = startBootstrapSignerController controllers["tokencleaner"] = startTokenCleanerController diff --git a/pkg/controller/certificates/BUILD b/pkg/controller/certificates/BUILD index e81b0fcdbb..fbc86657cd 100644 --- a/pkg/controller/certificates/BUILD +++ b/pkg/controller/certificates/BUILD @@ -43,6 +43,7 @@ filegroup( srcs = [ ":package-srcs", "//pkg/controller/certificates/approver:all-srcs", + "//pkg/controller/certificates/cleaner:all-srcs", "//pkg/controller/certificates/signer:all-srcs", ], tags = ["automanaged"], diff --git a/pkg/controller/certificates/cleaner/BUILD b/pkg/controller/certificates/cleaner/BUILD new file mode 100644 index 0000000000..1be7ad96f1 --- /dev/null +++ b/pkg/controller/certificates/cleaner/BUILD @@ -0,0 +1,43 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["cleaner.go"], + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", + "//vendor/k8s.io/client-go/informers/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/client-go/listers/certificates/v1beta1:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["cleaner_test.go"], + library = ":go_default_library", + deps = [ + "//vendor/k8s.io/api/certificates/v1beta1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library", + ], +) diff --git a/pkg/controller/certificates/cleaner/cleaner.go b/pkg/controller/certificates/cleaner/cleaner.go new file mode 100644 index 0000000000..0cbc8c232c --- /dev/null +++ b/pkg/controller/certificates/cleaner/cleaner.go @@ -0,0 +1,200 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cleaner implements an automated cleaner that does garbage collection +// on CSRs that meet specific criteria. With automated CSR requests and +// automated approvals, the volume of CSRs only increases over time, at a rapid +// rate if the certificate duration is short. +package cleaner + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "time" + + "github.com/golang/glog" + + capi "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + certificatesinformers "k8s.io/client-go/informers/certificates/v1beta1" + csrclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" + certificateslisters "k8s.io/client-go/listers/certificates/v1beta1" +) + +const ( + // The interval to list all CSRs and check each one against the criteria to + // automatically clean it up. + pollingInterval = 1 * time.Hour + // The time periods after which these different CSR statuses should be + // cleaned up. + approvedExpiration = 1 * time.Hour + deniedExpiration = 1 * time.Hour + pendingExpiration = 24 * time.Hour +) + +// CSRCleanerController is a controller that garbage collects old certificate +// signing requests (CSRs). Since there are mechanisms that automatically +// create CSRs, and mechanisms that automatically approve CSRs, in order to +// prevent a build up of CSRs over time, it is necessary to GC them. CSRs will +// be removed if they meet one of the following criteria: the CSR is Approved +// with a certificate and is old enough to be past the GC issued deadline, the +// CSR is denied and is old enough to be past the GC denied deadline, the CSR +// is Pending and is old enough to be past the GC pending deadline, the CSR is +// approved with a certificate and the certificate is expired. +type CSRCleanerController struct { + csrClient csrclient.CertificateSigningRequestInterface + csrLister certificateslisters.CertificateSigningRequestLister +} + +// NewCSRCleanerController creates a new CSRCleanerController. +func NewCSRCleanerController( + csrClient csrclient.CertificateSigningRequestInterface, + csrInformer certificatesinformers.CertificateSigningRequestInformer, +) *CSRCleanerController { + return &CSRCleanerController{ + csrClient: csrClient, + csrLister: csrInformer.Lister(), + } +} + +// Run the main goroutine responsible for watching and syncing jobs. +func (ccc *CSRCleanerController) Run(workers int, stopCh <-chan struct{}) { + defer utilruntime.HandleCrash() + + glog.Infof("Starting CSR cleaner controller") + defer glog.Infof("Shutting down CSR cleaner controller") + + for i := 0; i < workers; i++ { + go wait.Until(ccc.worker, pollingInterval, stopCh) + } + + <-stopCh +} + +// worker runs a thread that dequeues CSRs, handles them, and marks them done. +func (ccc *CSRCleanerController) worker() { + csrs, err := ccc.csrLister.List(labels.Everything()) + if err != nil { + glog.Errorf("Unable to list CSRs: %v", err) + return + } + for _, csr := range csrs { + if err := ccc.handle(csr); err != nil { + glog.Errorf("Error while attempting to clean CSR %q: %v", csr.Name, err) + } + } +} + +func (ccc *CSRCleanerController) handle(csr *capi.CertificateSigningRequest) error { + isIssuedExpired, err := isIssuedExpired(csr) + if err != nil { + return err + } + if isIssuedPastDeadline(csr) || isDeniedPastDeadline(csr) || isPendingPastDeadline(csr) || isIssuedExpired { + if err := ccc.csrClient.Delete(csr.Name, nil); err != nil { + return fmt.Errorf("unable to delete CSR %q: %v", csr.Name, err) + } + } + return nil +} + +// isIssuedExpired checks if the CSR has been issued a certificate and if the +// expiration of the certificate (the NotAfter value) has passed. +func isIssuedExpired(csr *capi.CertificateSigningRequest) (bool, error) { + for _, c := range csr.Status.Conditions { + isExpired, err := isExpired(csr) + if err != nil { + return false, err + } + if c.Type == capi.CertificateApproved && isIssued(csr) && isExpired { + glog.Infof("Cleaning CSR %q as the associated certificate is expired.", csr.Name, approvedExpiration) + return true, nil + } + } + return false, nil +} + +// isPendingPastDeadline checks if the certificate has a Pending status and the +// creation time of the CSR is passed the deadline that pending requests are +// maintained for. +func isPendingPastDeadline(csr *capi.CertificateSigningRequest) bool { + // If there are no Conditions on the status, the CSR will appear via + // `kubectl` as `Pending`. + if len(csr.Status.Conditions) == 0 && isOlderThan(csr.CreationTimestamp, pendingExpiration) { + glog.Infof("Cleaning CSR %q as it is more than %v old and unhandled.", csr.Name, pendingExpiration) + return true + } + return false +} + +// isDeniedPastDeadline checks if the certificate has a Denied status and the +// creation time of the CSR is passed the deadline that denied requests are +// maintained for. +func isDeniedPastDeadline(csr *capi.CertificateSigningRequest) bool { + for _, c := range csr.Status.Conditions { + if c.Type == capi.CertificateDenied && isOlderThan(c.LastUpdateTime, deniedExpiration) { + glog.Infof("Cleaning CSR %q as it is more than %v old and denied.", csr.Name, deniedExpiration) + return true + } + } + return false +} + +// isIssuedPastDeadline checks if the certificate has an Issued status and the +// creation time of the CSR is passed the deadline that issued requests are +// maintained for. +func isIssuedPastDeadline(csr *capi.CertificateSigningRequest) bool { + for _, c := range csr.Status.Conditions { + if c.Type == capi.CertificateApproved && isIssued(csr) && isOlderThan(c.LastUpdateTime, approvedExpiration) { + glog.Infof("Cleaning CSR %q as it is more than %v old and approved.", csr.Name, approvedExpiration) + return true + } + } + return false +} + +// isOlderThan checks that t is a non-zero time after time.Now() + d. +func isOlderThan(t metav1.Time, d time.Duration) bool { + return !t.IsZero() && t.Sub(time.Now()) < -1*d +} + +// isIssued checks if the CSR has `Issued` status. There is no explicit +// 'Issued' status. Implicitly, if there is a certificate associated with the +// CSR, the CSR statuses that are visible via `kubectl` will include 'Issued'. +func isIssued(csr *capi.CertificateSigningRequest) bool { + return csr.Status.Certificate != nil +} + +// isExpired checks if the CSR has a certificate and the date in the `NotAfter` +// field has gone by. +func isExpired(csr *capi.CertificateSigningRequest) (bool, error) { + if csr.Status.Certificate == nil { + return false, nil + } + block, _ := pem.Decode(csr.Status.Certificate) + if block == nil { + return false, fmt.Errorf("expected the certificate associated with the CSR to be PEM encoded") + } + certs, err := x509.ParseCertificates(block.Bytes) + if err != nil { + return false, fmt.Errorf("unable to parse certificate data: %v", err) + } + return time.Now().After(certs[0].NotAfter), nil +} diff --git a/pkg/controller/certificates/cleaner/cleaner_test.go b/pkg/controller/certificates/cleaner/cleaner_test.go new file mode 100644 index 0000000000..7a8d8f6e1d --- /dev/null +++ b/pkg/controller/certificates/cleaner/cleaner_test.go @@ -0,0 +1,201 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cleaner + +import ( + "testing" + "time" + + capi "k8s.io/api/certificates/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const ( + expiredCert = `-----BEGIN CERTIFICATE----- +MIICIzCCAc2gAwIBAgIJAOApTlMFDOUnMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MQowCAYD +VQQDDAEqMB4XDTE3MTAwNDIwNDgzOFoXDTE3MTAwMzIwNDgzOFowbTELMAkGA1UE +BhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMRgwFgYDVQQK +DA9HbG9iYWwgU2VjdXJpdHkxFjAUBgNVBAsMDUlUIERlcGFydG1lbnQxCjAIBgNV +BAMMASowXDANBgkqhkiG9w0BAQEFAANLADBIAkEA3Gt0KmuRXDxvqZUiX/xqAn1t +nZZX98guZvPPyxnQtV3YpA274W0sX3jL+U71Ya+3kaUstXQa4YrWBUHiXoqJnwID +AQABo1AwTjAdBgNVHQ4EFgQUtDsIpzHoUiLsO88f9fm+G0tYSPowHwYDVR0jBBgw +FoAUtDsIpzHoUiLsO88f9fm+G0tYSPowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B +AQsFAANBADfrlKof5CUkxGlX9Rifxv/mWOk8ZuTLWfMYQH2nycBHnmOxy6sR+87W +/Mb/uRz0TXVnGVcbu5E8Bz7e/Far1ZI= +-----END CERTIFICATE-----` + unexpiredCert = `-----BEGIN CERTIFICATE----- +MIICJTCCAc+gAwIBAgIJAIRjMToP+pPEMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV +BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE +CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MQowCAYD +VQQDDAEqMCAXDTE3MTAwNDIwNDUyNFoYDzIxMTcwOTEwMjA0NTI0WjBtMQswCQYD +VQQGEwJHQjEPMA0GA1UECAwGTG9uZG9uMQ8wDQYDVQQHDAZMb25kb24xGDAWBgNV +BAoMD0dsb2JhbCBTZWN1cml0eTEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDEKMAgG +A1UEAwwBKjBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQC7j9BAV5HqIJGi6r4G4YeI +ioHxH2loVu8IOKSK7xVs3v/EjR/eXbQzM+jZU7duyZqn6YjySZNLl0K0MfHCHBgX +AgMBAAGjUDBOMB0GA1UdDgQWBBTwxV40NFSNW7lpQ3eUWX7Mxs03yzAfBgNVHSME +GDAWgBTwxV40NFSNW7lpQ3eUWX7Mxs03yzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA0EALDi9OidANHflx8q+w3p0rJo9gpA6cJcFpEtP2Lv4kvOtB1f6L0jY +MLd7MVm4cS/MNcx4L7l23UC3Hx4+nAxvIg== +-----END CERTIFICATE-----` +) + +func TestCleanerWithApprovedExpiredCSR(t *testing.T) { + testCases := []struct { + name string + created metav1.Time + certificate []byte + conditions []capi.CertificateSigningRequestCondition + expectedActions []string + }{ + { + "no delete approved not passed deadline", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + []byte(unexpiredCert), + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateApproved, + LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)), + }, + }, + []string{}, + }, + { + "no delete approved passed deadline not issued", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + nil, + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateApproved, + LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)), + }, + }, + []string{}, + }, + { + "delete approved passed deadline", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + []byte(unexpiredCert), + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateApproved, + LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Hour)), + }, + }, + []string{"delete"}, + }, + { + "no delete denied not passed deadline", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + nil, + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateDenied, + LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)), + }, + }, + []string{}, + }, + { + "delete denied passed deadline", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + nil, + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateDenied, + LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Hour)), + }, + }, + []string{"delete"}, + }, + { + "no delete pending not passed deadline", + metav1.NewTime(time.Now().Add(-5 * time.Hour)), + nil, + []capi.CertificateSigningRequestCondition{}, + []string{}, + }, + { + "delete pending passed deadline", + metav1.NewTime(time.Now().Add(-25 * time.Hour)), + nil, + []capi.CertificateSigningRequestCondition{}, + []string{"delete"}, + }, + { + "no delete approved not passed deadline unexpired", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + []byte(unexpiredCert), + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateApproved, + LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)), + }, + }, + []string{}, + }, + { + "delete approved not passed deadline expired", + metav1.NewTime(time.Now().Add(-1 * time.Minute)), + []byte(expiredCert), + []capi.CertificateSigningRequestCondition{ + { + Type: capi.CertificateApproved, + LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)), + }, + }, + []string{"delete"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + csr := &capi.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "fake-csr", + CreationTimestamp: tc.created, + }, + Status: capi.CertificateSigningRequestStatus{ + Certificate: tc.certificate, + Conditions: tc.conditions, + }, + } + + client := fake.NewSimpleClientset(csr) + s := &CSRCleanerController{ + csrClient: client.CertificatesV1beta1().CertificateSigningRequests(), + } + + err := s.handle(csr) + if err != nil { + t.Fatalf("failed to clean CSR: %v", err) + } + + actions := client.Actions() + if len(actions) != len(tc.expectedActions) { + t.Fatalf("got %d actions, wanted %d actions", len(actions), len(tc.expectedActions)) + } + for i := 0; i < len(actions); i++ { + if a := actions[i]; !a.Matches(tc.expectedActions[i], "certificatesigningrequests") { + t.Errorf("got action %#v, wanted %v", a, tc.expectedActions[i]) + } + } + }) + } +} diff --git a/pkg/registry/certificates/certificates/strategy.go b/pkg/registry/certificates/certificates/strategy.go index 8ef65a2c01..7428691952 100644 --- a/pkg/registry/certificates/certificates/strategy.go +++ b/pkg/registry/certificates/certificates/strategy.go @@ -19,6 +19,7 @@ package certificates import ( "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/validation/field" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -142,6 +143,11 @@ func (csrStatusStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, ol // approval and certificate issuance. newCSR.Spec = oldCSR.Spec newCSR.Status.Conditions = oldCSR.Status.Conditions + for i := range newCSR.Status.Conditions { + if newCSR.Status.Conditions[i].LastUpdateTime.IsZero() { + newCSR.Status.Conditions[i].LastUpdateTime = metav1.Now() + } + } } func (csrStatusStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList { @@ -159,6 +165,9 @@ type csrApprovalStrategy struct { var ApprovalStrategy = csrApprovalStrategy{Strategy} +// PrepareForUpdate prepares the new certificate signing request by limiting +// the data that is updated to only the conditions. Also, if there is no +// existing LastUpdateTime on a condition, the current date/time will be set. func (csrApprovalStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) { newCSR := obj.(*certificates.CertificateSigningRequest) oldCSR := old.(*certificates.CertificateSigningRequest) @@ -166,6 +175,14 @@ func (csrApprovalStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, // Updating the approval should only update the conditions. newCSR.Spec = oldCSR.Spec oldCSR.Status.Conditions = newCSR.Status.Conditions + for i := range newCSR.Status.Conditions { + // The Conditions are an array of values, some of which may be + // pre-existing and unaltered by this update, so a LastUpdateTime is + // added only if one isn't already set. + if newCSR.Status.Conditions[i].LastUpdateTime.IsZero() { + newCSR.Status.Conditions[i].LastUpdateTime = metav1.Now() + } + } newCSR.Status = oldCSR.Status }