github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

New controller to GC CSRs.

pull/6/head
Jacob Simpson 2017-08-30 09:58:04 -07:00
parent 33f911cb53
commit 2a6099b8f9
8 changed files with 475 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/kube-controller-manager/app/BUILD
View File

@ -49,6 +49,7 @@ go_library(
"//pkg/controller:go_default_library",
"//pkg/controller/bootstrap:go_default_library",
"//pkg/controller/certificates/approver:go_default_library",
"//pkg/controller/certificates/cleaner:go_default_library",
"//pkg/controller/certificates/signer:go_default_library",
"//pkg/controller/cronjob:go_default_library",
"//pkg/controller/daemon:go_default_library",

13
cmd/kube-controller-manager/app/certificates.go
View File

@ -25,6 +25,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/controller/certificates/approver"
"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
"k8s.io/kubernetes/pkg/controller/certificates/signer"
)
@ -57,10 +58,9 @@ func startCSRApprovingController(ctx ControllerContext) (bool, error) {
if !ctx.AvailableResources[schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"}] {
return false, nil
}
c := ctx.ClientBuilder.ClientOrDie("certificate-controller")
approver, err := approver.NewCSRApprovingController(
c,
ctx.ClientBuilder.ClientOrDie("certificate-controller"),
ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(),
)
if err != nil {
@ -73,3 +73,12 @@ func startCSRApprovingController(ctx ControllerContext) (bool, error) {
return true, nil
}
func startCSRCleanerController(ctx ControllerContext) (bool, error) {
cleaner := cleaner.NewCSRCleanerController(
ctx.ClientBuilder.ClientOrDie("certificate-controller").CertificatesV1beta1().CertificateSigningRequests(),
ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(),
)
go cleaner.Run(1, ctx.Stop)
return true, nil
}

1
cmd/kube-controller-manager/app/controllermanager.go
View File

@ -348,6 +348,7 @@ func NewControllerInitializers() map[string]InitFunc {
controllers["cronjob"] = startCronJobController
controllers["csrsigning"] = startCSRSigningController
controllers["csrapproving"] = startCSRApprovingController
controllers["csrcleaner"] = startCSRCleanerController
controllers["ttl"] = startTTLController
controllers["bootstrapsigner"] = startBootstrapSignerController
controllers["tokencleaner"] = startTokenCleanerController

1
pkg/controller/certificates/BUILD
View File

@ -43,6 +43,7 @@ filegroup(
srcs = [
":package-srcs",
"//pkg/controller/certificates/approver:all-srcs",
"//pkg/controller/certificates/cleaner:all-srcs",
"//pkg/controller/certificates/signer:all-srcs",
],
tags = ["automanaged"],

43
pkg/controller/certificates/cleaner/BUILD Normal file
View File

@ -0,0 +1,43 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = ["cleaner.go"],
visibility = ["//visibility:public"],
deps = [
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/client-go/informers/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/client-go/listers/certificates/v1beta1:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_test",
srcs = ["cleaner_test.go"],
library = ":go_default_library",
deps = [
"//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

200
pkg/controller/certificates/cleaner/cleaner.go Normal file
View File

@ -0,0 +1,200 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package cleaner implements an automated cleaner that does garbage collection
// on CSRs that meet specific criteria. With automated CSR requests and
// automated approvals, the volume of CSRs only increases over time, at a rapid
// rate if the certificate duration is short.
package cleaner
import (
"crypto/x509"
"encoding/pem"
"fmt"
"time"
"github.com/golang/glog"
capi "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
certificatesinformers "k8s.io/client-go/informers/certificates/v1beta1"
csrclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
certificateslisters "k8s.io/client-go/listers/certificates/v1beta1"
)
const (
// The interval to list all CSRs and check each one against the criteria to
// automatically clean it up.
pollingInterval = 1 * time.Hour
// The time periods after which these different CSR statuses should be
// cleaned up.
approvedExpiration = 1 * time.Hour
deniedExpiration = 1 * time.Hour
pendingExpiration = 24 * time.Hour
)
// CSRCleanerController is a controller that garbage collects old certificate
// signing requests (CSRs). Since there are mechanisms that automatically
// create CSRs, and mechanisms that automatically approve CSRs, in order to
// prevent a build up of CSRs over time, it is necessary to GC them. CSRs will
// be removed if they meet one of the following criteria: the CSR is Approved
// with a certificate and is old enough to be past the GC issued deadline, the
// CSR is denied and is old enough to be past the GC denied deadline, the CSR
// is Pending and is old enough to be past the GC pending deadline, the CSR is
// approved with a certificate and the certificate is expired.
type CSRCleanerController struct {
csrClient csrclient.CertificateSigningRequestInterface
csrLister certificateslisters.CertificateSigningRequestLister
}
// NewCSRCleanerController creates a new CSRCleanerController.
func NewCSRCleanerController(
csrClient csrclient.CertificateSigningRequestInterface,
csrInformer certificatesinformers.CertificateSigningRequestInformer,
) *CSRCleanerController {
return &CSRCleanerController{
csrClient: csrClient,
csrLister: csrInformer.Lister(),
}
}
// Run the main goroutine responsible for watching and syncing jobs.
func (ccc *CSRCleanerController) Run(workers int, stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
glog.Infof("Starting CSR cleaner controller")
defer glog.Infof("Shutting down CSR cleaner controller")
for i := 0; i < workers; i++ {
go wait.Until(ccc.worker, pollingInterval, stopCh)
}
<-stopCh
}
// worker runs a thread that dequeues CSRs, handles them, and marks them done.
func (ccc *CSRCleanerController) worker() {
csrs, err := ccc.csrLister.List(labels.Everything())
if err != nil {
glog.Errorf("Unable to list CSRs: %v", err)
return
}
for _, csr := range csrs {
if err := ccc.handle(csr); err != nil {
glog.Errorf("Error while attempting to clean CSR %q: %v", csr.Name, err)
}
}
}
func (ccc *CSRCleanerController) handle(csr *capi.CertificateSigningRequest) error {
isIssuedExpired, err := isIssuedExpired(csr)
if err != nil {
return err
}
if isIssuedPastDeadline(csr) || isDeniedPastDeadline(csr) || isPendingPastDeadline(csr) || isIssuedExpired {
if err := ccc.csrClient.Delete(csr.Name, nil); err != nil {
return fmt.Errorf("unable to delete CSR %q: %v", csr.Name, err)
}
}
return nil
}
// isIssuedExpired checks if the CSR has been issued a certificate and if the
// expiration of the certificate (the NotAfter value) has passed.
func isIssuedExpired(csr *capi.CertificateSigningRequest) (bool, error) {
for _, c := range csr.Status.Conditions {
isExpired, err := isExpired(csr)
if err != nil {
return false, err
}
if c.Type == capi.CertificateApproved && isIssued(csr) && isExpired {
glog.Infof("Cleaning CSR %q as the associated certificate is expired.", csr.Name, approvedExpiration)
return true, nil
}
}
return false, nil
}
// isPendingPastDeadline checks if the certificate has a Pending status and the
// creation time of the CSR is passed the deadline that pending requests are
// maintained for.
func isPendingPastDeadline(csr *capi.CertificateSigningRequest) bool {
// If there are no Conditions on the status, the CSR will appear via
// `kubectl` as `Pending`.
if len(csr.Status.Conditions) == 0 && isOlderThan(csr.CreationTimestamp, pendingExpiration) {
glog.Infof("Cleaning CSR %q as it is more than %v old and unhandled.", csr.Name, pendingExpiration)
return true
}
return false
}
// isDeniedPastDeadline checks if the certificate has a Denied status and the
// creation time of the CSR is passed the deadline that denied requests are
// maintained for.
func isDeniedPastDeadline(csr *capi.CertificateSigningRequest) bool {
for _, c := range csr.Status.Conditions {
if c.Type == capi.CertificateDenied && isOlderThan(c.LastUpdateTime, deniedExpiration) {
glog.Infof("Cleaning CSR %q as it is more than %v old and denied.", csr.Name, deniedExpiration)
return true
}
}
return false
}
// isIssuedPastDeadline checks if the certificate has an Issued status and the
// creation time of the CSR is passed the deadline that issued requests are
// maintained for.
func isIssuedPastDeadline(csr *capi.CertificateSigningRequest) bool {
for _, c := range csr.Status.Conditions {
if c.Type == capi.CertificateApproved && isIssued(csr) && isOlderThan(c.LastUpdateTime, approvedExpiration) {
glog.Infof("Cleaning CSR %q as it is more than %v old and approved.", csr.Name, approvedExpiration)
return true
}
}
return false
}
// isOlderThan checks that t is a non-zero time after time.Now() + d.
func isOlderThan(t metav1.Time, d time.Duration) bool {
return !t.IsZero() && t.Sub(time.Now()) < -1*d
}
// isIssued checks if the CSR has `Issued` status. There is no explicit
// 'Issued' status. Implicitly, if there is a certificate associated with the
// CSR, the CSR statuses that are visible via `kubectl` will include 'Issued'.
func isIssued(csr *capi.CertificateSigningRequest) bool {
return csr.Status.Certificate != nil
}
// isExpired checks if the CSR has a certificate and the date in the `NotAfter`
// field has gone by.
func isExpired(csr *capi.CertificateSigningRequest) (bool, error) {
if csr.Status.Certificate == nil {
return false, nil
}
block, _ := pem.Decode(csr.Status.Certificate)
if block == nil {
return false, fmt.Errorf("expected the certificate associated with the CSR to be PEM encoded")
}
certs, err := x509.ParseCertificates(block.Bytes)
if err != nil {
return false, fmt.Errorf("unable to parse certificate data: %v", err)
}
return time.Now().After(certs[0].NotAfter), nil
}

201
pkg/controller/certificates/cleaner/cleaner_test.go Normal file
View File

@ -0,0 +1,201 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cleaner
import (
"testing"
"time"
capi "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
)
const (
expiredCert = `-----BEGIN CERTIFICATE-----
MIICIzCCAc2gAwIBAgIJAOApTlMFDOUnMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV
BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE
CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MQowCAYD
VQQDDAEqMB4XDTE3MTAwNDIwNDgzOFoXDTE3MTAwMzIwNDgzOFowbTELMAkGA1UE
BhMCR0IxDzANBgNVBAgMBkxvbmRvbjEPMA0GA1UEBwwGTG9uZG9uMRgwFgYDVQQK
DA9HbG9iYWwgU2VjdXJpdHkxFjAUBgNVBAsMDUlUIERlcGFydG1lbnQxCjAIBgNV
BAMMASowXDANBgkqhkiG9w0BAQEFAANLADBIAkEA3Gt0KmuRXDxvqZUiX/xqAn1t
nZZX98guZvPPyxnQtV3YpA274W0sX3jL+U71Ya+3kaUstXQa4YrWBUHiXoqJnwID
AQABo1AwTjAdBgNVHQ4EFgQUtDsIpzHoUiLsO88f9fm+G0tYSPowHwYDVR0jBBgw
FoAUtDsIpzHoUiLsO88f9fm+G0tYSPowDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0B
AQsFAANBADfrlKof5CUkxGlX9Rifxv/mWOk8ZuTLWfMYQH2nycBHnmOxy6sR+87W
/Mb/uRz0TXVnGVcbu5E8Bz7e/Far1ZI=
-----END CERTIFICATE-----`
unexpiredCert = `-----BEGIN CERTIFICATE-----
MIICJTCCAc+gAwIBAgIJAIRjMToP+pPEMA0GCSqGSIb3DQEBCwUAMG0xCzAJBgNV
BAYTAkdCMQ8wDQYDVQQIDAZMb25kb24xDzANBgNVBAcMBkxvbmRvbjEYMBYGA1UE
CgwPR2xvYmFsIFNlY3VyaXR5MRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50MQowCAYD
VQQDDAEqMCAXDTE3MTAwNDIwNDUyNFoYDzIxMTcwOTEwMjA0NTI0WjBtMQswCQYD
VQQGEwJHQjEPMA0GA1UECAwGTG9uZG9uMQ8wDQYDVQQHDAZMb25kb24xGDAWBgNV
BAoMD0dsb2JhbCBTZWN1cml0eTEWMBQGA1UECwwNSVQgRGVwYXJ0bWVudDEKMAgG
A1UEAwwBKjBcMA0GCSqGSIb3DQEBAQUAA0sAMEgCQQC7j9BAV5HqIJGi6r4G4YeI
ioHxH2loVu8IOKSK7xVs3v/EjR/eXbQzM+jZU7duyZqn6YjySZNLl0K0MfHCHBgX
AgMBAAGjUDBOMB0GA1UdDgQWBBTwxV40NFSNW7lpQ3eUWX7Mxs03yzAfBgNVHSME
GDAWgBTwxV40NFSNW7lpQ3eUWX7Mxs03yzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3
DQEBCwUAA0EALDi9OidANHflx8q+w3p0rJo9gpA6cJcFpEtP2Lv4kvOtB1f6L0jY
MLd7MVm4cS/MNcx4L7l23UC3Hx4+nAxvIg==
-----END CERTIFICATE-----`
)
func TestCleanerWithApprovedExpiredCSR(t *testing.T) {
testCases := []struct {
name string
created metav1.Time
certificate []byte
conditions []capi.CertificateSigningRequestCondition
expectedActions []string
}{
{
"no delete approved not passed deadline",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
[]byte(unexpiredCert),
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateApproved,
LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)),
},
},
[]string{},
},
{
"no delete approved passed deadline not issued",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
nil,
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateApproved,
LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)),
},
},
[]string{},
},
{
"delete approved passed deadline",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
[]byte(unexpiredCert),
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateApproved,
LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Hour)),
},
},
[]string{"delete"},
},
{
"no delete denied not passed deadline",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
nil,
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateDenied,
LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)),
},
},
[]string{},
},
{
"delete denied passed deadline",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
nil,
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateDenied,
LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Hour)),
},
},
[]string{"delete"},
},
{
"no delete pending not passed deadline",
metav1.NewTime(time.Now().Add(-5 * time.Hour)),
nil,
[]capi.CertificateSigningRequestCondition{},
[]string{},
},
{
"delete pending passed deadline",
metav1.NewTime(time.Now().Add(-25 * time.Hour)),
nil,
[]capi.CertificateSigningRequestCondition{},
[]string{"delete"},
},
{
"no delete approved not passed deadline unexpired",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
[]byte(unexpiredCert),
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateApproved,
LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)),
},
},
[]string{},
},
{
"delete approved not passed deadline expired",
metav1.NewTime(time.Now().Add(-1 * time.Minute)),
[]byte(expiredCert),
[]capi.CertificateSigningRequestCondition{
{
Type: capi.CertificateApproved,
LastUpdateTime: metav1.NewTime(time.Now().Add(-50 * time.Minute)),
},
},
[]string{"delete"},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
csr := &capi.CertificateSigningRequest{
ObjectMeta: metav1.ObjectMeta{
Name: "fake-csr",
CreationTimestamp: tc.created,
},
Status: capi.CertificateSigningRequestStatus{
Certificate: tc.certificate,
Conditions: tc.conditions,
},
}
client := fake.NewSimpleClientset(csr)
s := &CSRCleanerController{
csrClient: client.CertificatesV1beta1().CertificateSigningRequests(),
}
err := s.handle(csr)
if err != nil {
t.Fatalf("failed to clean CSR: %v", err)
}
actions := client.Actions()
if len(actions) != len(tc.expectedActions) {
t.Fatalf("got %d actions, wanted %d actions", len(actions), len(tc.expectedActions))
}
for i := 0; i < len(actions); i++ {
if a := actions[i]; !a.Matches(tc.expectedActions[i], "certificatesigningrequests") {
t.Errorf("got action %#v, wanted %v", a, tc.expectedActions[i])
}
}
})
}
}

17
pkg/registry/certificates/certificates/strategy.go
View File

@ -19,6 +19,7 @@ package certificates
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/validation/field"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
@ -142,6 +143,11 @@ func (csrStatusStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, ol
// approval and certificate issuance.
newCSR.Spec = oldCSR.Spec
newCSR.Status.Conditions = oldCSR.Status.Conditions
for i := range newCSR.Status.Conditions {
if newCSR.Status.Conditions[i].LastUpdateTime.IsZero() {
newCSR.Status.Conditions[i].LastUpdateTime = metav1.Now()
}
}
}
func (csrStatusStrategy) ValidateUpdate(ctx genericapirequest.Context, obj, old runtime.Object) field.ErrorList {
@ -159,6 +165,9 @@ type csrApprovalStrategy struct {
var ApprovalStrategy = csrApprovalStrategy{Strategy}
// PrepareForUpdate prepares the new certificate signing request by limiting
// the data that is updated to only the conditions. Also, if there is no
// existing LastUpdateTime on a condition, the current date/time will be set.
func (csrApprovalStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj, old runtime.Object) {
newCSR := obj.(*certificates.CertificateSigningRequest)
oldCSR := old.(*certificates.CertificateSigningRequest)
@ -166,6 +175,14 @@ func (csrApprovalStrategy) PrepareForUpdate(ctx genericapirequest.Context, obj,
// Updating the approval should only update the conditions.
newCSR.Spec = oldCSR.Spec
oldCSR.Status.Conditions = newCSR.Status.Conditions
for i := range newCSR.Status.Conditions {
// The Conditions are an array of values, some of which may be
// pre-existing and unaltered by this update, so a LastUpdateTime is
// added only if one isn't already set.
if newCSR.Status.Conditions[i].LastUpdateTime.IsZero() {
newCSR.Status.Conditions[i].LastUpdateTime = metav1.Now()
}
}
newCSR.Status = oldCSR.Status
}