mirror of https://github.com/k3s-io/k3s
Merge pull request #58123 from hzxuzhonghu/refactor-admission-flag
Automatic merge from submit-queue (batch tested with PRs 58496, 58078, 58123). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>. refactor admission flag **What this PR does / why we need it**: Refactor admission control flag, finally make cluster admins not care about orders in this flag. **Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*: Fixes # **Special notes for your reviewer**: **Release note**: ```release-note Add `--enable-admission-plugin` `--disable-admission-plugin` flags and deprecate `--admission-control`. Afterwards, don't care about the orders specified in the flags. ```pull/6/head
commit
f9bb978ad6
|
@ -10,49 +10,19 @@ go_library(
|
|||
name = "go_default_library",
|
||||
srcs = [
|
||||
"options.go",
|
||||
"plugins.go",
|
||||
"validation.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/options",
|
||||
deps = [
|
||||
"//pkg/apis/core:go_default_library",
|
||||
"//pkg/apis/core/validation:go_default_library",
|
||||
"//pkg/cloudprovider/providers:go_default_library",
|
||||
"//pkg/features:go_default_library",
|
||||
"//pkg/kubeapiserver/options:go_default_library",
|
||||
"//pkg/kubelet/client:go_default_library",
|
||||
"//pkg/master/ports:go_default_library",
|
||||
"//pkg/master/reconcilers:go_default_library",
|
||||
"//plugin/pkg/admission/admit:go_default_library",
|
||||
"//plugin/pkg/admission/alwayspullimages:go_default_library",
|
||||
"//plugin/pkg/admission/antiaffinity:go_default_library",
|
||||
"//plugin/pkg/admission/defaulttolerationseconds:go_default_library",
|
||||
"//plugin/pkg/admission/deny:go_default_library",
|
||||
"//plugin/pkg/admission/eventratelimit:go_default_library",
|
||||
"//plugin/pkg/admission/exec:go_default_library",
|
||||
"//plugin/pkg/admission/extendedresourcetoleration:go_default_library",
|
||||
"//plugin/pkg/admission/gc:go_default_library",
|
||||
"//plugin/pkg/admission/imagepolicy:go_default_library",
|
||||
"//plugin/pkg/admission/initialresources:go_default_library",
|
||||
"//plugin/pkg/admission/limitranger:go_default_library",
|
||||
"//plugin/pkg/admission/namespace/autoprovision:go_default_library",
|
||||
"//plugin/pkg/admission/namespace/exists:go_default_library",
|
||||
"//plugin/pkg/admission/noderestriction:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolume/label:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolume/resize:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library",
|
||||
"//plugin/pkg/admission/podnodeselector:go_default_library",
|
||||
"//plugin/pkg/admission/podpreset:go_default_library",
|
||||
"//plugin/pkg/admission/podtolerationrestriction:go_default_library",
|
||||
"//plugin/pkg/admission/priority:go_default_library",
|
||||
"//plugin/pkg/admission/resourcequota:go_default_library",
|
||||
"//plugin/pkg/admission/security/podsecuritypolicy:go_default_library",
|
||||
"//plugin/pkg/admission/securitycontext/scdeny:go_default_library",
|
||||
"//plugin/pkg/admission/serviceaccount:go_default_library",
|
||||
"//plugin/pkg/admission/storageclass/setdefault:go_default_library",
|
||||
"//vendor/github.com/spf13/pflag:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
|
||||
],
|
||||
|
|
|
@ -46,7 +46,7 @@ type ServerRunOptions struct {
|
|||
InsecureServing *kubeoptions.InsecureServingOptions
|
||||
Audit *genericoptions.AuditOptions
|
||||
Features *genericoptions.FeatureOptions
|
||||
Admission *genericoptions.AdmissionOptions
|
||||
Admission *kubeoptions.AdmissionOptions
|
||||
Authentication *kubeoptions.BuiltInAuthenticationOptions
|
||||
Authorization *kubeoptions.BuiltInAuthorizationOptions
|
||||
CloudProvider *kubeoptions.CloudProviderOptions
|
||||
|
@ -82,7 +82,7 @@ func NewServerRunOptions() *ServerRunOptions {
|
|||
InsecureServing: kubeoptions.NewInsecureServingOptions(),
|
||||
Audit: genericoptions.NewAuditOptions(),
|
||||
Features: genericoptions.NewFeatureOptions(),
|
||||
Admission: genericoptions.NewAdmissionOptions(),
|
||||
Admission: kubeoptions.NewAdmissionOptions(),
|
||||
Authentication: kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
|
||||
Authorization: kubeoptions.NewBuiltInAuthorizationOptions(),
|
||||
CloudProvider: kubeoptions.NewCloudProviderOptions(),
|
||||
|
@ -116,10 +116,6 @@ func NewServerRunOptions() *ServerRunOptions {
|
|||
// Overwrite the default for storage data format.
|
||||
s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
|
||||
|
||||
// register all admission plugins
|
||||
RegisterAllAdmissionPlugins(s.Admission.Plugins)
|
||||
// Set the default for admission plugins names
|
||||
s.Admission.PluginNames = []string{"AlwaysAdmit"}
|
||||
return &s
|
||||
}
|
||||
|
||||
|
|
|
@ -110,12 +110,14 @@ func TestAddFlags(t *testing.T) {
|
|||
RequestTimeout: time.Duration(2) * time.Minute,
|
||||
MinRequestTimeout: 1800,
|
||||
},
|
||||
Admission: &apiserveroptions.AdmissionOptions{
|
||||
RecommendedPluginOrder: []string{"NamespaceLifecycle", "Initializers", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook"},
|
||||
DefaultOffPlugins: []string{"Initializers", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook"},
|
||||
PluginNames: []string{"AlwaysDeny"},
|
||||
ConfigFile: "/admission-control-config",
|
||||
Plugins: s.Admission.Plugins,
|
||||
Admission: &kubeoptions.AdmissionOptions{
|
||||
PluginNames: []string{"AlwaysDeny"},
|
||||
GenericAdmission: &apiserveroptions.AdmissionOptions{
|
||||
RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
|
||||
DefaultOffPlugins: s.Admission.GenericAdmission.DefaultOffPlugins,
|
||||
ConfigFile: "/admission-control-config",
|
||||
Plugins: s.Admission.GenericAdmission.Plugins,
|
||||
},
|
||||
},
|
||||
Etcd: &apiserveroptions.EtcdOptions{
|
||||
StorageConfig: storagebackend.Config{
|
||||
|
|
|
@ -9,11 +9,13 @@ load(
|
|||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"admission.go",
|
||||
"api_enablement.go",
|
||||
"authentication.go",
|
||||
"authorization.go",
|
||||
"cloudprovider.go",
|
||||
"options.go",
|
||||
"plugins.go",
|
||||
"serving.go",
|
||||
"storage_versions.go",
|
||||
],
|
||||
|
@ -21,19 +23,55 @@ go_library(
|
|||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/client/informers/informers_generated/internalversion:go_default_library",
|
||||
"//pkg/cloudprovider/providers:go_default_library",
|
||||
"//pkg/kubeapiserver/authenticator:go_default_library",
|
||||
"//pkg/kubeapiserver/authorizer:go_default_library",
|
||||
"//pkg/kubeapiserver/authorizer/modes:go_default_library",
|
||||
"//pkg/kubeapiserver/server:go_default_library",
|
||||
"//plugin/pkg/admission/admit:go_default_library",
|
||||
"//plugin/pkg/admission/alwayspullimages:go_default_library",
|
||||
"//plugin/pkg/admission/antiaffinity:go_default_library",
|
||||
"//plugin/pkg/admission/defaulttolerationseconds:go_default_library",
|
||||
"//plugin/pkg/admission/deny:go_default_library",
|
||||
"//plugin/pkg/admission/eventratelimit:go_default_library",
|
||||
"//plugin/pkg/admission/exec:go_default_library",
|
||||
"//plugin/pkg/admission/extendedresourcetoleration:go_default_library",
|
||||
"//plugin/pkg/admission/gc:go_default_library",
|
||||
"//plugin/pkg/admission/imagepolicy:go_default_library",
|
||||
"//plugin/pkg/admission/initialresources:go_default_library",
|
||||
"//plugin/pkg/admission/limitranger:go_default_library",
|
||||
"//plugin/pkg/admission/namespace/autoprovision:go_default_library",
|
||||
"//plugin/pkg/admission/namespace/exists:go_default_library",
|
||||
"//plugin/pkg/admission/noderestriction:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolume/label:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolume/resize:go_default_library",
|
||||
"//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library",
|
||||
"//plugin/pkg/admission/podnodeselector:go_default_library",
|
||||
"//plugin/pkg/admission/podpreset:go_default_library",
|
||||
"//plugin/pkg/admission/podtolerationrestriction:go_default_library",
|
||||
"//plugin/pkg/admission/priority:go_default_library",
|
||||
"//plugin/pkg/admission/resourcequota:go_default_library",
|
||||
"//plugin/pkg/admission/security/podsecuritypolicy:go_default_library",
|
||||
"//plugin/pkg/admission/securitycontext/scdeny:go_default_library",
|
||||
"//plugin/pkg/admission/serviceaccount:go_default_library",
|
||||
"//plugin/pkg/admission/storageclass/setdefault:go_default_library",
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/github.com/pborman/uuid:go_default_library",
|
||||
"//vendor/github.com/spf13/pflag:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission/plugin/initialization:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/validating:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
|
||||
"//vendor/k8s.io/client-go/informers:go_default_library",
|
||||
"//vendor/k8s.io/client-go/rest:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
@ -52,7 +90,10 @@ filegroup(
|
|||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["storage_versions_test.go"],
|
||||
srcs = [
|
||||
"admission_test.go",
|
||||
"storage_versions_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
importpath = "k8s.io/kubernetes/pkg/kubeapiserver/options",
|
||||
deps = ["//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library"],
|
||||
|
|
|
@ -0,0 +1,123 @@
|
|||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package options
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/apiserver/pkg/server"
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/rest"
|
||||
)
|
||||
|
||||
// AdmissionOptions holds the admission options.
|
||||
// It is a wrap of generic AdmissionOptions.
|
||||
type AdmissionOptions struct {
|
||||
// GenericAdmission holds the generic admission options.
|
||||
GenericAdmission *genericoptions.AdmissionOptions
|
||||
// DEPRECATED flag, should use EnabledAdmissionPlugins and DisabledAdmissionPlugins.
|
||||
// They are mutually exclusive, specify both will lead to an error.
|
||||
PluginNames []string
|
||||
}
|
||||
|
||||
// NewAdmissionOptions creates a new instance of AdmissionOptions
|
||||
// Note:
|
||||
// In addition it calls RegisterAllAdmissionPlugins to register
|
||||
// all kube-apiserver admission plugins.
|
||||
//
|
||||
// Provides the list of RecommendedPluginOrder that holds sane values
|
||||
// that can be used by servers that don't care about admission chain.
|
||||
// Servers that do care can overwrite/append that field after creation.
|
||||
func NewAdmissionOptions() *AdmissionOptions {
|
||||
options := genericoptions.NewAdmissionOptions()
|
||||
// register all admission plugins
|
||||
RegisterAllAdmissionPlugins(options.Plugins)
|
||||
// set RecommendedPluginOrder
|
||||
options.RecommendedPluginOrder = AllOrderedPlugins
|
||||
// set DefaultOffPlugins
|
||||
options.DefaultOffPlugins = DefaultOffAdmissionPlugins()
|
||||
|
||||
return &AdmissionOptions{
|
||||
GenericAdmission: options,
|
||||
}
|
||||
}
|
||||
|
||||
// AddFlags adds flags related to admission for kube-apiserver to the specified FlagSet
|
||||
func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.StringSliceVar(&a.PluginNames, "admission-control", a.PluginNames, ""+
|
||||
"Admission is divided into two phases. "+
|
||||
"In the first phase, only mutating admission plugins run. "+
|
||||
"In the second phase, only validating admission plugins run. "+
|
||||
"The names in the below list may represent a validating plugin, a mutating plugin, or both. "+
|
||||
"The order of plugins in which they are passed to this flag does not matter. "+
|
||||
"Comma-delimited list of: "+strings.Join(a.GenericAdmission.Plugins.Registered(), ", ")+".")
|
||||
fs.MarkDeprecated("admission-control", "Use --enable-admission-plugins or --disable-admission-plugins instead. Will be removed in a future version.")
|
||||
|
||||
a.GenericAdmission.AddFlags(fs)
|
||||
}
|
||||
|
||||
// Validate verifies flags passed to kube-apiserver AdmissionOptions.
|
||||
// Kube-apiserver verifies PluginNames and then call generic AdmissionOptions.Validate.
|
||||
func (a *AdmissionOptions) Validate() []error {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
errs := []error{}
|
||||
if a.PluginNames != nil &&
|
||||
(a.GenericAdmission.EnablePlugins != nil || a.GenericAdmission.DisablePlugins != nil) {
|
||||
errs = append(errs, fmt.Errorf("admission-control and enable-admission-plugins/disable-admission-plugins flags are mutually exclusive"))
|
||||
}
|
||||
|
||||
registeredPlugins := sets.NewString(a.GenericAdmission.Plugins.Registered()...)
|
||||
for _, name := range a.PluginNames {
|
||||
if !registeredPlugins.Has(name) {
|
||||
errs = append(errs, fmt.Errorf("admission-control plugin %q is unknown", name))
|
||||
}
|
||||
}
|
||||
|
||||
errs = append(errs, a.GenericAdmission.Validate()...)
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// ApplyTo adds the admission chain to the server configuration.
|
||||
// Kube-apiserver just call generic AdmissionOptions.ApplyTo.
|
||||
func (a *AdmissionOptions) ApplyTo(
|
||||
c *server.Config,
|
||||
informers informers.SharedInformerFactory,
|
||||
kubeAPIServerClientConfig *rest.Config,
|
||||
scheme *runtime.Scheme,
|
||||
pluginInitializers ...admission.PluginInitializer,
|
||||
) error {
|
||||
if a == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if a.PluginNames != nil {
|
||||
// pass PluginNames to generic AdmissionOptions
|
||||
a.GenericAdmission.EnablePlugins = a.PluginNames
|
||||
}
|
||||
|
||||
return a.GenericAdmission.ApplyTo(c, informers, kubeAPIServerClientConfig, scheme, pluginInitializers...)
|
||||
}
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package options
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
// 1. Both `--admission-control` and `--enable-admission-plugins` are specified
|
||||
options := NewAdmissionOptions()
|
||||
options.PluginNames = []string{"ServiceAccount"}
|
||||
options.GenericAdmission.EnablePlugins = []string{"Initializers"}
|
||||
if len(options.Validate()) == 0 {
|
||||
t.Errorf("Expect error, but got none")
|
||||
}
|
||||
|
||||
// 2. Both `--admission-control` and `--disable-admission-plugins` are specified
|
||||
options = NewAdmissionOptions()
|
||||
options.PluginNames = []string{"ServiceAccount"}
|
||||
options.GenericAdmission.DisablePlugins = []string{"Initializers"}
|
||||
if len(options.Validate()) == 0 {
|
||||
t.Errorf("Expect error, but got none")
|
||||
}
|
||||
|
||||
// 3. PluginNames is not registered
|
||||
options = NewAdmissionOptions()
|
||||
options.PluginNames = []string{"pluginA"}
|
||||
if len(options.Validate()) == 0 {
|
||||
t.Errorf("Expect error, but got none")
|
||||
}
|
||||
|
||||
// 4. PluginNames is not valid
|
||||
options = NewAdmissionOptions()
|
||||
options.PluginNames = []string{"ServiceAccount"}
|
||||
if errs := options.Validate(); len(errs) > 0 {
|
||||
t.Errorf("Unexpected err: %v", errs)
|
||||
}
|
||||
}
|
|
@ -24,7 +24,6 @@ import (
|
|||
_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
|
||||
|
||||
// Admission policies
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/admit"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/alwayspullimages"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/antiaffinity"
|
||||
|
@ -52,15 +51,59 @@ import (
|
|||
"k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
|
||||
"k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
"k8s.io/apiserver/pkg/admission/plugin/initialization"
|
||||
"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
|
||||
mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
|
||||
validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
|
||||
)
|
||||
|
||||
// RegisterAllAdmissionPlugins registers all admission plugins
|
||||
// AllOrderedPlugins is the list of all the plugins in order.
|
||||
var AllOrderedPlugins = []string{
|
||||
admit.PluginName, // AlwaysAdmit
|
||||
autoprovision.PluginName, // NamespaceAutoProvision
|
||||
lifecycle.PluginName, // NamespaceLifecycle
|
||||
exists.PluginName, // NamespaceExists
|
||||
scdeny.PluginName, // SecurityContextDeny
|
||||
antiaffinity.PluginName, // LimitPodHardAntiAffinity
|
||||
initialresources.PluginName, // InitialResources
|
||||
podpreset.PluginName, // PodPreset
|
||||
limitranger.PluginName, // LimitRanger
|
||||
serviceaccount.PluginName, // ServiceAccount
|
||||
noderestriction.PluginName, // NodeRestriction
|
||||
alwayspullimages.PluginName, // AlwaysPullImages
|
||||
imagepolicy.PluginName, // ImagePolicyWebhook
|
||||
podsecuritypolicy.PluginName, // PodSecurityPolicy
|
||||
podnodeselector.PluginName, // PodNodeSelector
|
||||
podpriority.PluginName, // Priority
|
||||
defaulttolerationseconds.PluginName, // DefaultTolerationSeconds
|
||||
podtolerationrestriction.PluginName, // PodTolerationRestriction
|
||||
exec.DenyEscalatingExec, // DenyEscalatingExec
|
||||
exec.DenyExecOnPrivileged, // DenyExecOnPrivileged
|
||||
eventratelimit.PluginName, // EventRateLimit
|
||||
extendedresourcetoleration.PluginName, // ExtendedResourceToleration
|
||||
label.PluginName, // PersistentVolumeLabel
|
||||
setdefault.PluginName, // DefaultStorageClass
|
||||
pvcprotection.PluginName, // PVCProtection
|
||||
gc.PluginName, // OwnerReferencesPermissionEnforcement
|
||||
resize.PluginName, // PersistentVolumeClaimResize
|
||||
mutatingwebhook.PluginName, // MutatingAdmissionWebhook
|
||||
initialization.PluginName, // Initializers
|
||||
validatingwebhook.PluginName, // ValidatingAdmissionWebhook
|
||||
resourcequota.PluginName, // ResourceQuota
|
||||
deny.PluginName, // AlwaysDeny
|
||||
}
|
||||
|
||||
// RegisterAllAdmissionPlugins registers all admission plugins and
|
||||
// sets the recommended plugins order.
|
||||
func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
|
||||
admit.Register(plugins)
|
||||
admit.Register(plugins) // DEPRECATED as no real meaning
|
||||
alwayspullimages.Register(plugins)
|
||||
antiaffinity.Register(plugins)
|
||||
defaulttolerationseconds.Register(plugins)
|
||||
deny.Register(plugins)
|
||||
deny.Register(plugins) // DEPRECATED as no real meaning
|
||||
eventratelimit.Register(plugins)
|
||||
exec.Register(plugins)
|
||||
extendedresourcetoleration.Register(plugins)
|
||||
|
@ -84,3 +127,11 @@ func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
|
|||
resize.Register(plugins)
|
||||
pvcprotection.Register(plugins)
|
||||
}
|
||||
|
||||
// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver.
|
||||
func DefaultOffAdmissionPlugins() sets.String {
|
||||
defaultOffPlugins := sets.NewString(AllOrderedPlugins...)
|
||||
defaultOffPlugins.Delete(lifecycle.PluginName)
|
||||
|
||||
return defaultOffPlugins
|
||||
}
|
|
@ -10,7 +10,10 @@ go_library(
|
|||
name = "go_default_library",
|
||||
srcs = ["admission.go"],
|
||||
importpath = "k8s.io/kubernetes/plugin/pkg/admission/admit",
|
||||
deps = ["//vendor/k8s.io/apiserver/pkg/admission:go_default_library"],
|
||||
deps = [
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
|
|
|
@ -19,40 +19,46 @@ package admit
|
|||
import (
|
||||
"io"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "AlwaysAdmit"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("AlwaysAdmit", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewAlwaysAdmit(), nil
|
||||
})
|
||||
}
|
||||
|
||||
// AlwaysAdmit is an implementation of admission.Interface which always says yes to an admit request.
|
||||
// It is useful in tests and when using kubernetes in an open manner.
|
||||
type AlwaysAdmit struct{}
|
||||
// alwaysAdmit is an implementation of admission.Interface which always says yes to an admit request.
|
||||
type alwaysAdmit struct{}
|
||||
|
||||
var _ admission.MutationInterface = AlwaysAdmit{}
|
||||
var _ admission.ValidationInterface = AlwaysAdmit{}
|
||||
var _ admission.MutationInterface = alwaysAdmit{}
|
||||
var _ admission.ValidationInterface = alwaysAdmit{}
|
||||
|
||||
// Admit makes an admission decision based on the request attributes
|
||||
func (AlwaysAdmit) Admit(a admission.Attributes) (err error) {
|
||||
func (alwaysAdmit) Admit(a admission.Attributes) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Validate makes an admission decision based on the request attributes. It is NOT allowed to mutate.
|
||||
func (AlwaysAdmit) Validate(a admission.Attributes) (err error) {
|
||||
func (alwaysAdmit) Validate(a admission.Attributes) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handles returns true if this admission controller can handle the given operation
|
||||
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
|
||||
func (AlwaysAdmit) Handles(operation admission.Operation) bool {
|
||||
func (alwaysAdmit) Handles(operation admission.Operation) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// NewAlwaysAdmit creates a new always admit admission handler
|
||||
func NewAlwaysAdmit() *AlwaysAdmit {
|
||||
return new(AlwaysAdmit)
|
||||
func NewAlwaysAdmit() admission.Interface {
|
||||
// DEPRECATED: AlwaysAdmit admit all admission request, it is no use.
|
||||
glog.Warningf("%s admission controller is deprecated. "+
|
||||
"Please remove this controller from your configuration files and scripts.", PluginName)
|
||||
return new(alwaysAdmit)
|
||||
}
|
||||
|
|
|
@ -25,7 +25,7 @@ import (
|
|||
|
||||
func TestAdmissionNonNilAttribute(t *testing.T) {
|
||||
handler := NewAlwaysAdmit()
|
||||
err := handler.Admit(admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "namespace", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, nil))
|
||||
err := handler.(*alwaysAdmit).Admit(admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "namespace", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, nil))
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error returned from admission handler")
|
||||
}
|
||||
|
@ -33,7 +33,7 @@ func TestAdmissionNonNilAttribute(t *testing.T) {
|
|||
|
||||
func TestAdmissionNilAttribute(t *testing.T) {
|
||||
handler := NewAlwaysAdmit()
|
||||
err := handler.Admit(nil)
|
||||
err := handler.(*alwaysAdmit).Admit(nil)
|
||||
if err != nil {
|
||||
t.Errorf("Unexpected error returned from admission handler")
|
||||
}
|
||||
|
|
|
@ -33,9 +33,12 @@ import (
|
|||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "AlwaysPullImages"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("AlwaysPullImages", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewAlwaysPullImages(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -26,9 +26,11 @@ import (
|
|||
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
)
|
||||
|
||||
const PluginName = "LimitPodHardAntiAffinity"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("LimitPodHardAntiAffinityTopology", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewInterPodAntiAffinity(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -28,6 +28,9 @@ import (
|
|||
"k8s.io/kubernetes/pkg/scheduler/algorithm"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "DefaultTolerationSeconds"
|
||||
|
||||
var (
|
||||
defaultNotReadyTolerationSeconds = flag.Int64("default-not-ready-toleration-seconds", 300,
|
||||
"Indicates the tolerationSeconds of the toleration for notReady:NoExecute"+
|
||||
|
@ -40,7 +43,7 @@ var (
|
|||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("DefaultTolerationSeconds", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewDefaultTolerationSeconds(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -10,7 +10,10 @@ go_library(
|
|||
name = "go_default_library",
|
||||
srcs = ["admission.go"],
|
||||
importpath = "k8s.io/kubernetes/plugin/pkg/admission/deny",
|
||||
deps = ["//vendor/k8s.io/apiserver/pkg/admission:go_default_library"],
|
||||
deps = [
|
||||
"//vendor/github.com/golang/glog:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_test(
|
||||
|
|
|
@ -20,40 +20,47 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "AlwaysDeny"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("AlwaysDeny", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewAlwaysDeny(), nil
|
||||
})
|
||||
}
|
||||
|
||||
// AlwaysDeny is an implementation of admission.Interface which always says no to an admission request.
|
||||
// It is useful in unit tests to force an operation to be forbidden.
|
||||
type AlwaysDeny struct{}
|
||||
// alwaysDeny is an implementation of admission.Interface which always says no to an admission request.
|
||||
type alwaysDeny struct{}
|
||||
|
||||
var _ admission.MutationInterface = AlwaysDeny{}
|
||||
var _ admission.ValidationInterface = AlwaysDeny{}
|
||||
var _ admission.MutationInterface = alwaysDeny{}
|
||||
var _ admission.ValidationInterface = alwaysDeny{}
|
||||
|
||||
// Admit makes an admission decision based on the request attributes.
|
||||
func (AlwaysDeny) Admit(a admission.Attributes) (err error) {
|
||||
return admission.NewForbidden(a, errors.New("Admission control is denying all modifications"))
|
||||
func (alwaysDeny) Admit(a admission.Attributes) (err error) {
|
||||
return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
|
||||
}
|
||||
|
||||
// Validate makes an admission decision based on the request attributes. It is NOT allowed to mutate.
|
||||
func (AlwaysDeny) Validate(a admission.Attributes) (err error) {
|
||||
return admission.NewForbidden(a, errors.New("Admission control is denying all modifications"))
|
||||
func (alwaysDeny) Validate(a admission.Attributes) (err error) {
|
||||
return admission.NewForbidden(a, errors.New("admission control is denying all modifications"))
|
||||
}
|
||||
|
||||
// Handles returns true if this admission controller can handle the given operation
|
||||
// where operation can be one of CREATE, UPDATE, DELETE, or CONNECT
|
||||
func (AlwaysDeny) Handles(operation admission.Operation) bool {
|
||||
func (alwaysDeny) Handles(operation admission.Operation) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// NewAlwaysDeny creates an always deny admission handler
|
||||
func NewAlwaysDeny() *AlwaysDeny {
|
||||
return new(AlwaysDeny)
|
||||
func NewAlwaysDeny() admission.Interface {
|
||||
// DEPRECATED: AlwaysDeny denys all admission request, it is no use.
|
||||
glog.Warningf("%s admission controller is deprecated. "+
|
||||
"Please remove this controller from your configuration files and scripts.", PluginName)
|
||||
return new(alwaysDeny)
|
||||
}
|
||||
|
|
|
@ -25,7 +25,7 @@ import (
|
|||
|
||||
func TestAdmission(t *testing.T) {
|
||||
handler := NewAlwaysDeny()
|
||||
err := handler.Admit(admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "namespace", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, nil))
|
||||
err := handler.(*alwaysDeny).Admit(admission.NewAttributesRecord(nil, nil, api.Kind("kind").WithVersion("version"), "namespace", "name", api.Resource("resource").WithVersion("version"), "subresource", admission.Create, nil))
|
||||
if err == nil {
|
||||
t.Error("Expected error returned from admission handler")
|
||||
}
|
||||
|
|
|
@ -26,9 +26,12 @@ import (
|
|||
"k8s.io/kubernetes/plugin/pkg/admission/eventratelimit/apis/eventratelimit/validation"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "EventRateLimit"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("EventRateLimit",
|
||||
plugins.Register(PluginName,
|
||||
func(config io.Reader) (admission.Interface, error) {
|
||||
// load the configuration provided (if any)
|
||||
configuration, err := LoadConfiguration(config)
|
||||
|
|
|
@ -29,15 +29,23 @@ import (
|
|||
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
|
||||
)
|
||||
|
||||
const (
|
||||
// DenyEscalatingExec indicates name of admission plugin.
|
||||
DenyEscalatingExec = "DenyEscalatingExec"
|
||||
// DenyExecOnPrivileged indicates name of admission plugin.
|
||||
// Deprecated, should use DenyEscalatingExec instead.
|
||||
DenyExecOnPrivileged = "DenyExecOnPrivileged"
|
||||
)
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("DenyEscalatingExec", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(DenyEscalatingExec, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewDenyEscalatingExec(), nil
|
||||
})
|
||||
|
||||
// This is for legacy support of the DenyExecOnPrivileged admission controller. Most
|
||||
// of the time DenyEscalatingExec should be preferred.
|
||||
plugins.Register("DenyExecOnPrivileged", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(DenyExecOnPrivileged, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewDenyExecOnPrivileged(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -27,9 +27,12 @@ import (
|
|||
"k8s.io/kubernetes/pkg/apis/core/helper"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "ExtendedResourceToleration"
|
||||
|
||||
// Register is called by the apiserver to register the plugin factory.
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("ExtendedResourceToleration", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return newExtendedResourceToleration(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -30,9 +30,12 @@ import (
|
|||
"k8s.io/apiserver/pkg/authorization/authorizer"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "OwnerReferencesPermissionEnforcement"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("OwnerReferencesPermissionEnforcement", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
// the pods/status endpoint is ignored by this plugin since old kubelets
|
||||
// corrupt them. the pod status strategy ensures status updates cannot mutate
|
||||
// ownerRef.
|
||||
|
|
|
@ -43,13 +43,16 @@ import (
|
|||
_ "k8s.io/kubernetes/pkg/apis/imagepolicy/install"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "ImagePolicyWebhook"
|
||||
|
||||
var (
|
||||
groupVersions = []schema.GroupVersion{v1alpha1.SchemeGroupVersion}
|
||||
)
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("ImagePolicyWebhook", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
newImagePolicyWebhook, err := NewImagePolicyWebhook(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -42,12 +42,14 @@ const (
|
|||
samplesThreshold = 30
|
||||
week = 7 * 24 * time.Hour
|
||||
month = 30 * 24 * time.Hour
|
||||
// PluginName indicates name of admission plugin.
|
||||
PluginName = "InitialResources"
|
||||
)
|
||||
|
||||
// Register registers a plugin
|
||||
// WARNING: this feature is experimental and will definitely change.
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("InitialResources", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
// TODO: remove the usage of flags in favor of reading versioned configuration
|
||||
s, err := newDataSource(*source)
|
||||
if err != nil {
|
||||
|
|
|
@ -41,11 +41,13 @@ import (
|
|||
|
||||
const (
|
||||
limitRangerAnnotation = "kubernetes.io/limit-ranger"
|
||||
// PluginName indicates name of admission plugin.
|
||||
PluginName = "LimitRanger"
|
||||
)
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("LimitRanger", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewLimitRanger(&DefaultLimitRangerActions{})
|
||||
})
|
||||
}
|
||||
|
|
|
@ -30,9 +30,12 @@ import (
|
|||
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "NamespaceAutoProvision"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("NamespaceAutoProvision", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewProvision(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -30,9 +30,12 @@ import (
|
|||
kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "NamespaceExists"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("NamespaceExists", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewExists(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -33,9 +33,11 @@ import (
|
|||
vol "k8s.io/kubernetes/pkg/volume"
|
||||
)
|
||||
|
||||
const PluginName = "PersistentVolumeLabel"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("PersistentVolumeLabel", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
persistentVolumeLabelAdmission := NewPersistentVolumeLabel()
|
||||
return persistentVolumeLabelAdmission, nil
|
||||
})
|
||||
|
|
|
@ -40,9 +40,11 @@ import (
|
|||
// node selectors labels to namespaces
|
||||
var NamespaceNodeSelectors = []string{"scheduler.alpha.kubernetes.io/node-selector"}
|
||||
|
||||
const PluginName = "PodNodeSelector"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("PodNodeSelector", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
// TODO move this to a versioned configuration file format.
|
||||
pluginConfig := readConfig(config)
|
||||
plugin := NewPodNodeSelector(pluginConfig.PodNodeSelectorPluginConfig)
|
||||
|
|
|
@ -41,12 +41,12 @@ import (
|
|||
|
||||
const (
|
||||
annotationPrefix = "podpreset.admission.kubernetes.io"
|
||||
pluginName = "PodPreset"
|
||||
PluginName = "PodPreset"
|
||||
)
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register(pluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewPlugin(), nil
|
||||
})
|
||||
}
|
||||
|
@ -72,10 +72,10 @@ func NewPlugin() *podPresetPlugin {
|
|||
|
||||
func (plugin *podPresetPlugin) ValidateInitialization() error {
|
||||
if plugin.client == nil {
|
||||
return fmt.Errorf("%s requires a client", pluginName)
|
||||
return fmt.Errorf("%s requires a client", PluginName)
|
||||
}
|
||||
if plugin.lister == nil {
|
||||
return fmt.Errorf("%s requires a lister", pluginName)
|
||||
return fmt.Errorf("%s requires a lister", PluginName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -40,9 +40,11 @@ import (
|
|||
pluginapi "k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction/apis/podtolerationrestriction"
|
||||
)
|
||||
|
||||
const PluginName = "PodTolerationRestriction"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("PodTolerationRestriction", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
pluginConfig, err := loadConfiguration(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
@ -34,7 +34,8 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
pluginName = "Priority"
|
||||
// PluginName indicates name of admission plugin.
|
||||
PluginName = "Priority"
|
||||
|
||||
// HighestUserDefinablePriority is the highest priority for user defined priority classes. Priority values larger than 1 billion are reserved for Kubernetes system use.
|
||||
HighestUserDefinablePriority = 1000000000
|
||||
|
@ -50,7 +51,7 @@ var SystemPriorityClasses = map[string]int32{
|
|||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register(pluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewPlugin(), nil
|
||||
})
|
||||
}
|
||||
|
@ -79,10 +80,10 @@ func NewPlugin() *PriorityPlugin {
|
|||
// ValidateInitialization implements the InitializationValidator interface.
|
||||
func (p *PriorityPlugin) ValidateInitialization() error {
|
||||
if p.client == nil {
|
||||
return fmt.Errorf("%s requires a client", pluginName)
|
||||
return fmt.Errorf("%s requires a client", PluginName)
|
||||
}
|
||||
if p.lister == nil {
|
||||
return fmt.Errorf("%s requires a lister", pluginName)
|
||||
return fmt.Errorf("%s requires a lister", PluginName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -31,9 +31,11 @@ import (
|
|||
"k8s.io/kubernetes/plugin/pkg/admission/resourcequota/apis/resourcequota/validation"
|
||||
)
|
||||
|
||||
const PluginName = "ResourceQuota"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("ResourceQuota",
|
||||
plugins.Register(PluginName,
|
||||
func(config io.Reader) (admission.Interface, error) {
|
||||
// load the configuration provided (if any)
|
||||
configuration, err := LoadConfiguration(config)
|
||||
|
|
|
@ -25,9 +25,12 @@ import (
|
|||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
)
|
||||
|
||||
// PluginName indicates name of admission plugin.
|
||||
const PluginName = "SecurityContextDeny"
|
||||
|
||||
// Register registers a plugin
|
||||
func Register(plugins *admission.Plugins) {
|
||||
plugins.Register("SecurityContextDeny", func(config io.Reader) (admission.Interface, error) {
|
||||
plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) {
|
||||
return NewSecurityContextDeny(), nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -110,7 +110,7 @@ func ConnectResource(connecter rest.Connecter, scope RequestScope, admit admissi
|
|||
scope.err(err, w, req)
|
||||
return
|
||||
}
|
||||
if admit.Handles(admission.Connect) {
|
||||
if admit != nil && admit.Handles(admission.Connect) {
|
||||
connectRequest := &rest.ConnectRequest{
|
||||
Name: name,
|
||||
Options: opts,
|
||||
|
|
|
@ -78,7 +78,9 @@ go_test(
|
|||
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
|
||||
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
|
||||
|
|
|
@ -21,6 +21,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
|
@ -49,11 +50,16 @@ func init() {
|
|||
type AdmissionOptions struct {
|
||||
// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
|
||||
RecommendedPluginOrder []string
|
||||
// DefaultOffPlugins a list of plugin names that should be disabled by default
|
||||
DefaultOffPlugins []string
|
||||
PluginNames []string
|
||||
ConfigFile string
|
||||
// DefaultOffPlugins is a set of plugin names that is disabled by default
|
||||
DefaultOffPlugins sets.String
|
||||
|
||||
// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
|
||||
EnablePlugins []string
|
||||
// DisablePlugins indicates plugins to be disabled passed through `--disable-admission-plugins`.
|
||||
DisablePlugins []string
|
||||
// ConfigFile is the file path with admission control configuration.
|
||||
ConfigFile string
|
||||
// Plugins contains all registered plugins.
|
||||
Plugins *admission.Plugins
|
||||
}
|
||||
|
||||
|
@ -67,14 +73,13 @@ type AdmissionOptions struct {
|
|||
// Servers that do care can overwrite/append that field after creation.
|
||||
func NewAdmissionOptions() *AdmissionOptions {
|
||||
options := &AdmissionOptions{
|
||||
Plugins: admission.NewPlugins(),
|
||||
PluginNames: []string{},
|
||||
Plugins: admission.NewPlugins(),
|
||||
// This list is mix of mutating admission plugins and validating
|
||||
// admission plugins. The apiserver always runs the validating ones
|
||||
// after all the mutating ones, so their relative order in this list
|
||||
// doesn't matter.
|
||||
RecommendedPluginOrder: []string{lifecycle.PluginName, initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
|
||||
DefaultOffPlugins: []string{initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName},
|
||||
DefaultOffPlugins: sets.NewString(initialization.PluginName, mutatingwebhook.PluginName, validatingwebhook.PluginName),
|
||||
}
|
||||
server.RegisterAllAdmissionPlugins(options.Plugins)
|
||||
return options
|
||||
|
@ -82,14 +87,14 @@ func NewAdmissionOptions() *AdmissionOptions {
|
|||
|
||||
// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
|
||||
func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
fs.StringSliceVar(&a.PluginNames, "admission-control", a.PluginNames, ""+
|
||||
"Admission is divided into two phases. "+
|
||||
"In the first phase, only mutating admission plugins run. "+
|
||||
"In the second phase, only validating admission plugins run. "+
|
||||
"The names in the below list may represent a validating plugin, a mutating plugin, or both. "+
|
||||
"Within each phase, the plugins will run in the order in which they are passed to this flag. "+
|
||||
"Comma-delimited list of: "+strings.Join(a.Plugins.Registered(), ", ")+".")
|
||||
|
||||
fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+
|
||||
"admission plugins that should be enabled in addition to default enabled ones. "+
|
||||
"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
|
||||
"The order of plugins in this flag does not matter.")
|
||||
fs.StringSliceVar(&a.DisablePlugins, "disable-admission-plugins", a.DisablePlugins, ""+
|
||||
"admission plugins that should be disabled although they are in the default enabled plugins list. "+
|
||||
"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
|
||||
"The order of plugins in this flag does not matter.")
|
||||
fs.StringVar(&a.ConfigFile, "admission-control-config-file", a.ConfigFile,
|
||||
"File with admission control configuration.")
|
||||
}
|
||||
|
@ -120,10 +125,7 @@ func (a *AdmissionOptions) ApplyTo(
|
|||
return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
|
||||
}
|
||||
|
||||
pluginNames := a.PluginNames
|
||||
if len(a.PluginNames) == 0 {
|
||||
pluginNames = a.enabledPluginNames()
|
||||
}
|
||||
pluginNames := a.enabledPluginNames()
|
||||
|
||||
pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
|
||||
if err != nil {
|
||||
|
@ -148,6 +150,7 @@ func (a *AdmissionOptions) ApplyTo(
|
|||
return nil
|
||||
}
|
||||
|
||||
// Validate verifies flags passed to AdmissionOptions.
|
||||
func (a *AdmissionOptions) Validate() []error {
|
||||
if a == nil {
|
||||
return nil
|
||||
|
@ -156,41 +159,57 @@ func (a *AdmissionOptions) Validate() []error {
|
|||
errs := []error{}
|
||||
|
||||
registeredPlugins := sets.NewString(a.Plugins.Registered()...)
|
||||
for _, name := range a.PluginNames {
|
||||
for _, name := range a.EnablePlugins {
|
||||
if !registeredPlugins.Has(name) {
|
||||
errs = append(errs, fmt.Errorf("admission-control plugin %q is invalid", name))
|
||||
errs = append(errs, fmt.Errorf("enable-admission-plugins plugin %q is unknown", name))
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range a.DisablePlugins {
|
||||
if !registeredPlugins.Has(name) {
|
||||
errs = append(errs, fmt.Errorf("disable-admission-plugins plugin %q is unknown", name))
|
||||
}
|
||||
}
|
||||
|
||||
enablePlugins := sets.NewString(a.EnablePlugins...)
|
||||
disablePlugins := sets.NewString(a.DisablePlugins...)
|
||||
if len(enablePlugins.Intersection(disablePlugins).List()) > 0 {
|
||||
errs = append(errs, fmt.Errorf("%v in enable-admission-plugins and disable-admission-plugins "+
|
||||
"overlapped", enablePlugins.Intersection(disablePlugins).List()))
|
||||
}
|
||||
|
||||
// Verify RecommendedPluginOrder.
|
||||
recommendPlugins := sets.NewString(a.RecommendedPluginOrder...)
|
||||
intersections := registeredPlugins.Intersection(recommendPlugins)
|
||||
if !intersections.Equal(recommendPlugins) {
|
||||
// Developer error, this should never run in.
|
||||
errs = append(errs, fmt.Errorf("plugins %v in RecommendedPluginOrder are not registered",
|
||||
recommendPlugins.Difference(intersections).List()))
|
||||
}
|
||||
if !intersections.Equal(registeredPlugins) {
|
||||
// Developer error, this should never run in.
|
||||
errs = append(errs, fmt.Errorf("plugins %v registered are not in RecommendedPluginOrder",
|
||||
registeredPlugins.Difference(intersections).List()))
|
||||
}
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// enabledPluginNames makes use of RecommendedPluginOrder and DefaultOffPlugins fields
|
||||
// to prepare a list of plugin names that are enabled.
|
||||
//
|
||||
// TODO(p0lyn0mial): In the end we will introduce two new flags:
|
||||
// --disable-admission-plugin this would be a list of admission plugins that a cluster-admin wants to explicitly disable.
|
||||
// --enable-admission-plugin this would be a list of admission plugins that a cluster-admin wants to explicitly enable.
|
||||
// both flags are going to be handled by this method
|
||||
// enabledPluginNames makes use of RecommendedPluginOrder, DefaultOffPlugins,
|
||||
// EnablePlugins, DisablePlugins fields
|
||||
// to prepare a list of ordered plugin names that are enabled.
|
||||
func (a *AdmissionOptions) enabledPluginNames() []string {
|
||||
//TODO(p0lyn0mial): first subtract plugins that a user wants to explicitly enable from allOffPlugins (DefaultOffPlugins)
|
||||
//TODO(p0lyn0miial): then add/append plugins that a user wants to explicitly disable to allOffPlugins
|
||||
//TODO(p0lyn0mial): so that --off=three --on=one,three default-off=one,two results in "one" being enabled.
|
||||
allOffPlugins := a.DefaultOffPlugins
|
||||
onlyEnabledPluginNames := []string{}
|
||||
for _, pluginName := range a.RecommendedPluginOrder {
|
||||
disablePlugin := false
|
||||
for _, disabledPluginName := range allOffPlugins {
|
||||
if pluginName == disabledPluginName {
|
||||
disablePlugin = true
|
||||
break
|
||||
}
|
||||
allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
|
||||
disabledPlugins := sets.NewString(allOffPlugins...)
|
||||
enabledPlugins := sets.NewString(a.EnablePlugins...)
|
||||
disabledPlugins = disabledPlugins.Difference(enabledPlugins)
|
||||
|
||||
orderedPlugins := []string{}
|
||||
for _, plugin := range a.RecommendedPluginOrder {
|
||||
if !disabledPlugins.Has(plugin) {
|
||||
orderedPlugins = append(orderedPlugins, plugin)
|
||||
}
|
||||
if disablePlugin {
|
||||
continue
|
||||
}
|
||||
onlyEnabledPluginNames = append(onlyEnabledPluginNames, pluginName)
|
||||
}
|
||||
|
||||
return onlyEnabledPluginNames
|
||||
return orderedPlugins
|
||||
}
|
||||
|
|
|
@ -18,26 +18,80 @@ package options
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/apiserver/pkg/admission"
|
||||
)
|
||||
|
||||
func TestEnabledPluginNamesMethod(t *testing.T) {
|
||||
func TestEnabledPluginNames(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
expectedPluginNames []string
|
||||
setDefaultOffPluginNames []string
|
||||
setDefaultOffPlugins sets.String
|
||||
setRecommendedPluginOrder []string
|
||||
setEnablePlugins []string
|
||||
setDisablePlugins []string
|
||||
setAdmissionControl []string
|
||||
}{
|
||||
// scenario 1: check if a call to enabledPluginNames sets expected values.
|
||||
// scenario 0: check if a call to enabledPluginNames sets expected values.
|
||||
{
|
||||
expectedPluginNames: []string{"NamespaceLifecycle"},
|
||||
},
|
||||
|
||||
// scenario 2: overwrite RecommendedPluginOrder and set DefaultOffPluginNames
|
||||
// make sure that plugins which are on DefaultOffPluginNames list do not get to PluginNames list.
|
||||
// scenario 1: use default off plugins if no specified
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB"},
|
||||
setDefaultOffPluginNames: []string{"pluginB"},
|
||||
expectedPluginNames: []string{"pluginB"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginA", "pluginC", "pluginD"),
|
||||
},
|
||||
|
||||
// scenario 2: use default off plugins and with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString(),
|
||||
},
|
||||
|
||||
// scenario 3: use default off plugins and specified by enable-admission-plugin with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginC", "pluginD"),
|
||||
setEnablePlugins: []string{"pluginD", "pluginC"},
|
||||
},
|
||||
|
||||
// scenario 4: use default off plugins and specified by disable-admission-plugin with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginB"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginC", "pluginD"),
|
||||
setDisablePlugins: []string{"pluginA"},
|
||||
},
|
||||
|
||||
// scenario 5: use default off plugins and specified by enable-admission-plugin and disable-admission-plugin with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA", "pluginC"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginC", "pluginD"),
|
||||
setEnablePlugins: []string{"pluginC"},
|
||||
setDisablePlugins: []string{"pluginB"},
|
||||
},
|
||||
|
||||
// scenario 6: use default off plugins and specified by admission-control with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA", "pluginB", "pluginC"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginD"),
|
||||
setAdmissionControl: []string{"pluginA", "pluginB"},
|
||||
},
|
||||
|
||||
// scenario 7: use default off plugins and specified by admission-control with RecommendedPluginOrder
|
||||
{
|
||||
expectedPluginNames: []string{"pluginA", "pluginB", "pluginC"},
|
||||
setRecommendedPluginOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
setDefaultOffPlugins: sets.NewString("pluginC", "pluginD"),
|
||||
setAdmissionControl: []string{"pluginA", "pluginB", "pluginC"},
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -46,12 +100,21 @@ func TestEnabledPluginNamesMethod(t *testing.T) {
|
|||
t.Run(fmt.Sprintf("scenario %d", index), func(t *testing.T) {
|
||||
target := NewAdmissionOptions()
|
||||
|
||||
if len(scenario.setDefaultOffPluginNames) > 0 {
|
||||
target.DefaultOffPlugins = scenario.setDefaultOffPluginNames
|
||||
if scenario.setDefaultOffPlugins != nil {
|
||||
target.DefaultOffPlugins = scenario.setDefaultOffPlugins
|
||||
}
|
||||
if len(scenario.setRecommendedPluginOrder) > 0 {
|
||||
if scenario.setRecommendedPluginOrder != nil {
|
||||
target.RecommendedPluginOrder = scenario.setRecommendedPluginOrder
|
||||
}
|
||||
if scenario.setEnablePlugins != nil {
|
||||
target.EnablePlugins = scenario.setEnablePlugins
|
||||
}
|
||||
if scenario.setDisablePlugins != nil {
|
||||
target.DisablePlugins = scenario.setDisablePlugins
|
||||
}
|
||||
if scenario.setAdmissionControl != nil {
|
||||
target.EnablePlugins = scenario.setAdmissionControl
|
||||
}
|
||||
|
||||
actualPluginNames := target.enabledPluginNames()
|
||||
|
||||
|
@ -66,3 +129,111 @@ func TestEnabledPluginNamesMethod(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
setEnablePlugins []string
|
||||
setDisablePlugins []string
|
||||
setRecommendedPluginsOrder []string
|
||||
expectedResult bool
|
||||
}{
|
||||
// scenario 0: not set any flag
|
||||
{
|
||||
expectedResult: true,
|
||||
},
|
||||
|
||||
// scenario 1: set both `--enable-admission-plugin` `--disable-admission-plugin`
|
||||
{
|
||||
setEnablePlugins: []string{"pluginA", "pluginB"},
|
||||
setDisablePlugins: []string{"pluginC"},
|
||||
expectedResult: true,
|
||||
},
|
||||
|
||||
// scenario 2: set invalid `--enable-admission-plugin` `--disable-admission-plugin`
|
||||
{
|
||||
setEnablePlugins: []string{"pluginA", "pluginB"},
|
||||
setDisablePlugins: []string{"pluginB"},
|
||||
expectedResult: false,
|
||||
},
|
||||
|
||||
// scenario 3: set only invalid `--enable-admission-plugin`
|
||||
{
|
||||
setEnablePlugins: []string{"pluginA", "pluginE"},
|
||||
expectedResult: false,
|
||||
},
|
||||
|
||||
// scenario 4: set only invalid `--disable-admission-plugin`
|
||||
{
|
||||
setDisablePlugins: []string{"pluginA", "pluginE"},
|
||||
expectedResult: false,
|
||||
},
|
||||
|
||||
// scenario 5: set valid `--enable-admission-plugin`
|
||||
{
|
||||
setEnablePlugins: []string{"pluginA", "pluginB"},
|
||||
expectedResult: true,
|
||||
},
|
||||
|
||||
// scenario 6: set valid `--disable-admission-plugin`
|
||||
{
|
||||
setDisablePlugins: []string{"pluginA"},
|
||||
expectedResult: true,
|
||||
},
|
||||
|
||||
// scenario 7: RecommendedPluginOrder has duplicated plugin
|
||||
{
|
||||
setRecommendedPluginsOrder: []string{"pluginA", "pluginB", "pluginB", "pluginC"},
|
||||
expectedResult: false,
|
||||
},
|
||||
|
||||
// scenario 8: RecommendedPluginOrder not equal to registered
|
||||
{
|
||||
setRecommendedPluginsOrder: []string{"pluginA", "pluginB", "pluginC"},
|
||||
expectedResult: false,
|
||||
},
|
||||
|
||||
// scenario 9: RecommendedPluginOrder equal to registered
|
||||
{
|
||||
setRecommendedPluginsOrder: []string{"pluginA", "pluginB", "pluginC", "pluginD"},
|
||||
expectedResult: true,
|
||||
},
|
||||
|
||||
// scenario 10: RecommendedPluginOrder not equal to registered
|
||||
{
|
||||
setRecommendedPluginsOrder: []string{"pluginA", "pluginB", "pluginC", "pluginE"},
|
||||
expectedResult: false,
|
||||
},
|
||||
}
|
||||
|
||||
for index, scenario := range scenarios {
|
||||
t.Run(fmt.Sprintf("scenario %d", index), func(t *testing.T) {
|
||||
options := NewAdmissionOptions()
|
||||
options.DefaultOffPlugins = sets.NewString("pluginC", "pluginD")
|
||||
options.RecommendedPluginOrder = []string{"pluginA", "pluginB", "pluginC", "pluginD"}
|
||||
options.Plugins = &admission.Plugins{}
|
||||
for _, plugin := range options.RecommendedPluginOrder {
|
||||
options.Plugins.Register(plugin, func(config io.Reader) (admission.Interface, error) {
|
||||
return nil, nil
|
||||
})
|
||||
}
|
||||
|
||||
if scenario.setEnablePlugins != nil {
|
||||
options.EnablePlugins = scenario.setEnablePlugins
|
||||
}
|
||||
if scenario.setDisablePlugins != nil {
|
||||
options.DisablePlugins = scenario.setDisablePlugins
|
||||
}
|
||||
if scenario.setRecommendedPluginsOrder != nil {
|
||||
options.RecommendedPluginOrder = scenario.setRecommendedPluginsOrder
|
||||
}
|
||||
|
||||
err := options.Validate()
|
||||
if len(err) > 0 && scenario.expectedResult {
|
||||
t.Errorf("Unexpected err: %v", err)
|
||||
}
|
||||
if len(err) == 0 && !scenario.expectedResult {
|
||||
t.Errorf("Expect error, but got none")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue