From a06901a8687e2270215cb98ca45f1d50b237c748 Mon Sep 17 00:00:00 2001 From: pospispa Date: Thu, 9 Nov 2017 13:56:41 +0100 Subject: [PATCH] Admission Controller PVC Finalizer Plugin This admission plugin puts finalizer to every created PVC. The finalizer is removed by PVCProtectionController when the PVC is not referenced by any pods and thus the PVC can be deleted. --- cluster/centos/config-default.sh | 2 +- cluster/gce/config-default.sh | 2 +- cluster/libvirt-coreos/util.sh | 2 +- cluster/vagrant/config-default.sh | 3 +- cmd/kube-apiserver/app/options/BUILD | 1 + cmd/kube-apiserver/app/options/plugins.go | 2 + pkg/volume/util/finalizer.go | 5 + plugin/BUILD | 1 + .../persistentvolumeclaim/pvcprotection/BUILD | 51 ++++++++ .../pvcprotection/admission.go | 111 ++++++++++++++++++ .../pvcprotection/admission_test.go | 106 +++++++++++++++++ 11 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD create mode 100644 plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go create mode 100644 plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go diff --git a/cluster/centos/config-default.sh b/cluster/centos/config-default.sh index 90d13bee12..eca05cb3cc 100755 --- a/cluster/centos/config-default.sh +++ b/cluster/centos/config-default.sh @@ -120,7 +120,7 @@ export FLANNEL_NET=${FLANNEL_NET:-"172.16.0.0/16"} # Admission Controllers to invoke prior to persisting objects in cluster # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,ResourceQuota"} +export ADMISSION_CONTROL=${ADMISSION_CONTROL:-"Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeClaimResize,DefaultTolerationSeconds,Priority,PVCProtection,ResourceQuota"} # Extra options to set on the Docker command line. # This is useful for setting --insecure-registry for local registries. diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh index f1dc45feec..7b2bddf773 100755 --- a/cluster/gce/config-default.sh +++ b/cluster/gce/config-default.sh @@ -294,7 +294,7 @@ if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then fi # Admission Controllers to invoke prior to persisting objects in cluster -ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority +ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,NodeRestriction,Priority,PVCProtection if [[ "${ENABLE_POD_SECURITY_POLICY:-}" == "true" ]]; then ADMISSION_CONTROL="${ADMISSION_CONTROL},PodSecurityPolicy" diff --git a/cluster/libvirt-coreos/util.sh b/cluster/libvirt-coreos/util.sh index 9fe2170ea4..545d985045 100644 --- a/cluster/libvirt-coreos/util.sh +++ b/cluster/libvirt-coreos/util.sh @@ -27,7 +27,7 @@ source "$KUBE_ROOT/cluster/common.sh" export LIBVIRT_DEFAULT_URI=qemu:///system export SERVICE_ACCOUNT_LOOKUP=${SERVICE_ACCOUNT_LOOKUP:-true} -export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,ResourceQuota} +export ADMISSION_CONTROL=${ADMISSION_CONTROL:-Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,PersistentVolumeClaimResize,DefaultTolerationSeconds,PVCProtection,ResourceQuota} readonly POOL=kubernetes readonly POOL_PATH=/var/lib/libvirt/images/kubernetes diff --git a/cluster/vagrant/config-default.sh b/cluster/vagrant/config-default.sh index 7eea6e8e77..63b49146db 100755 --- a/cluster/vagrant/config-default.sh +++ b/cluster/vagrant/config-default.sh @@ -56,7 +56,7 @@ MASTER_PASSWD="${MASTER_PASSWD:-vagrant}" # Admission Controllers to invoke prior to persisting objects in cluster # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely. -ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota +ADMISSION_CONTROL=Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,PVCProtection,ResourceQuota # Optional: Enable node logging. ENABLE_NODE_LOGGING=false @@ -120,4 +120,3 @@ E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false} # Default fallback NETWORK_IF_NAME, will be used in case when no 'VAGRANT-BEGIN' comments were defined in network-script export DEFAULT_NETWORK_IF_NAME="eth0" - diff --git a/cmd/kube-apiserver/app/options/BUILD b/cmd/kube-apiserver/app/options/BUILD index 2ad00991da..7c46be368b 100644 --- a/cmd/kube-apiserver/app/options/BUILD +++ b/cmd/kube-apiserver/app/options/BUILD @@ -40,6 +40,7 @@ go_library( "//plugin/pkg/admission/noderestriction:go_default_library", "//plugin/pkg/admission/persistentvolume/label:go_default_library", "//plugin/pkg/admission/persistentvolume/resize:go_default_library", + "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library", "//plugin/pkg/admission/podnodeselector:go_default_library", "//plugin/pkg/admission/podpreset:go_default_library", "//plugin/pkg/admission/podtolerationrestriction:go_default_library", diff --git a/cmd/kube-apiserver/app/options/plugins.go b/cmd/kube-apiserver/app/options/plugins.go index 30ed306485..a0d2502e7f 100644 --- a/cmd/kube-apiserver/app/options/plugins.go +++ b/cmd/kube-apiserver/app/options/plugins.go @@ -42,6 +42,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/admission/noderestriction" "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label" "k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize" + "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection" "k8s.io/kubernetes/plugin/pkg/admission/podnodeselector" "k8s.io/kubernetes/plugin/pkg/admission/podpreset" "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction" @@ -81,4 +82,5 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) { serviceaccount.Register(plugins) setdefault.Register(plugins) resize.Register(plugins) + pvcprotection.Register(plugins) } diff --git a/pkg/volume/util/finalizer.go b/pkg/volume/util/finalizer.go index db22ff2578..57a34951a4 100644 --- a/pkg/volume/util/finalizer.go +++ b/pkg/volume/util/finalizer.go @@ -20,6 +20,11 @@ import ( "k8s.io/api/core/v1" ) +const ( + // Name of finalizer on PVCs that have a running pod. + PVCProtectionFinalizer = "kubernetes.io/pvc-protection" +) + // IsPVCBeingDeleted returns: // true: in case PVC is being deleted, i.e. ObjectMeta.DeletionTimestamp is set // false: in case PVC is not being deleted, i.e. ObjectMeta.DeletionTimestamp is nil diff --git a/plugin/BUILD b/plugin/BUILD index 764a8e3283..03b7485610 100644 --- a/plugin/BUILD +++ b/plugin/BUILD @@ -29,6 +29,7 @@ filegroup( "//plugin/pkg/admission/noderestriction:all-srcs", "//plugin/pkg/admission/persistentvolume/label:all-srcs", "//plugin/pkg/admission/persistentvolume/resize:all-srcs", + "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:all-srcs", "//plugin/pkg/admission/podnodeselector:all-srcs", "//plugin/pkg/admission/podpreset:all-srcs", "//plugin/pkg/admission/podtolerationrestriction:all-srcs", diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD new file mode 100644 index 0000000000..e13f63e593 --- /dev/null +++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/BUILD @@ -0,0 +1,51 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["admission.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection", + visibility = ["//visibility:public"], + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/client/listers/core/internalversion:go_default_library", + "//pkg/features:go_default_library", + "//pkg/kubeapiserver/admission:go_default_library", + "//pkg/volume/util:go_default_library", + "//vendor/github.com/golang/glog:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["admission_test.go"], + importpath = "k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection", + library = ":go_default_library", + deps = [ + "//pkg/apis/core:go_default_library", + "//pkg/client/informers/informers_generated/internalversion:go_default_library", + "//pkg/controller:go_default_library", + "//pkg/volume/util:go_default_library", + "//vendor/github.com/davecgh/go-spew/spew:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", + "//vendor/k8s.io/apiserver/pkg/admission:go_default_library", + "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", + ], +) + +filegroup( + name = "package-srcs", + srcs = glob(["**"]), + tags = ["automanaged"], + visibility = ["//visibility:private"], +) + +filegroup( + name = "all-srcs", + srcs = [":package-srcs"], + tags = ["automanaged"], + visibility = ["//visibility:public"], +) diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go new file mode 100644 index 0000000000..218bca4b82 --- /dev/null +++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission.go @@ -0,0 +1,111 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pvcprotection + +import ( + "fmt" + "io" + + "github.com/golang/glog" + + admission "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + corelisters "k8s.io/kubernetes/pkg/client/listers/core/internalversion" + "k8s.io/kubernetes/pkg/features" + kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +const ( + // PluginName is the name of this admission controller plugin + PluginName = "PVCProtection" +) + +// Register registers a plugin +func Register(plugins *admission.Plugins) { + plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { + plugin := newPlugin() + return plugin, nil + }) +} + +// pvcProtectionPlugin holds state for and implements the admission plugin. +type pvcProtectionPlugin struct { + *admission.Handler + lister corelisters.PersistentVolumeClaimLister +} + +var _ admission.Interface = &pvcProtectionPlugin{} +var _ = kubeapiserveradmission.WantsInternalKubeInformerFactory(&pvcProtectionPlugin{}) + +// newPlugin creates a new admission plugin. +func newPlugin() *pvcProtectionPlugin { + return &pvcProtectionPlugin{ + Handler: admission.NewHandler(admission.Create), + } +} + +func (c *pvcProtectionPlugin) SetInternalKubeInformerFactory(f informers.SharedInformerFactory) { + informer := f.Core().InternalVersion().PersistentVolumeClaims() + c.lister = informer.Lister() + c.SetReadyFunc(informer.Informer().HasSynced) +} + +// ValidateInitialization ensures lister is set. +func (c *pvcProtectionPlugin) ValidateInitialization() error { + if c.lister == nil { + return fmt.Errorf("missing lister") + } + return nil +} + +// Admit sets finalizer on all PVCs. The finalizer is removed by +// PVCProtectionController when it's not referenced by any pod. +// +// This prevents users from deleting a PVC that's used by a running pod. +func (c *pvcProtectionPlugin) Admit(a admission.Attributes) error { + if !feature.DefaultFeatureGate.Enabled(features.PVCProtection) { + return nil + } + + if a.GetResource().GroupResource() != api.Resource("persistentvolumeclaims") { + return nil + } + + if len(a.GetSubresource()) != 0 { + return nil + } + + pvc, ok := a.GetObject().(*api.PersistentVolumeClaim) + // if we can't convert then we don't handle this object so just return + if !ok { + return nil + } + + for _, f := range pvc.Finalizers { + if f == volumeutil.PVCProtectionFinalizer { + // Finalizer is already present, nothing to do + return nil + } + } + + glog.V(4).Infof("adding PVC protection finalizer to %s/%s", pvc.Namespace, pvc.Name) + pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer) + return nil +} diff --git a/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go new file mode 100644 index 0000000000..0815c40615 --- /dev/null +++ b/plugin/pkg/admission/persistentvolumeclaim/pvcprotection/admission_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pvcprotection + +import ( + "fmt" + "reflect" + "testing" + + "github.com/davecgh/go-spew/spew" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/admission" + "k8s.io/apiserver/pkg/util/feature" + api "k8s.io/kubernetes/pkg/apis/core" + informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion" + "k8s.io/kubernetes/pkg/controller" + volumeutil "k8s.io/kubernetes/pkg/volume/util" +) + +func TestAdmit(t *testing.T) { + claim := &api.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolumeClaim", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "claim", + Namespace: "ns", + }, + } + claimWithFinalizer := claim.DeepCopy() + claimWithFinalizer.Finalizers = []string{volumeutil.PVCProtectionFinalizer} + + tests := []struct { + name string + object runtime.Object + expectedObject runtime.Object + featureEnabled bool + }{ + { + "create -> add finalizer", + claim, + claimWithFinalizer, + true, + }, + { + "finalizer already exists -> no new finalizer", + claimWithFinalizer, + claimWithFinalizer, + true, + }, + { + "disabled feature -> no finalizer", + claim, + claim, + false, + }, + } + + ctrl := newPlugin() + informerFactory := informers.NewSharedInformerFactory(nil, controller.NoResyncPeriodFunc()) + ctrl.SetInternalKubeInformerFactory(informerFactory) + + for _, test := range tests { + feature.DefaultFeatureGate.Set(fmt.Sprintf("PVCProtection=%v", test.featureEnabled)) + obj := test.object.DeepCopyObject() + attrs := admission.NewAttributesRecord( + obj, // new object + obj.DeepCopyObject(), // old object, copy to be sure it's not modified + api.Kind("PersistentVolumeClaim").WithVersion("version"), + claim.Namespace, + claim.Name, + api.Resource("persistentvolumeclaims").WithVersion("version"), + "", // subresource + admission.Create, + nil, // userInfo + ) + + err := ctrl.Admit(attrs) + if err != nil { + t.Errorf("Test %q: got unexpected error: %v", test.name, err) + } + if !reflect.DeepEqual(test.expectedObject, obj) { + t.Errorf("Test %q: Expected object:\n%s\ngot:\n%s", test.name, spew.Sdump(test.expectedObject), spew.Sdump(obj)) + } + } + + // Disable the feature for rest of the tests. + // TODO: remove after alpha + feature.DefaultFeatureGate.Set("PVCProtection=false") +}