github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update core quota framework

pull/6/head
Derek Carr 2017-10-27 11:07:01 -04:00
parent b00c15f1a4
commit 13294a0abe
21 changed files with 399 additions and 735 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/quota/BUILD
View File

@ -21,6 +21,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)

10
pkg/quota/evaluator/core/BUILD
View File

@ -9,14 +9,10 @@ load(
go_library(
name = "go_default_library",
srcs = [
"configmap.go",
"doc.go",
"persistent_volume_claims.go",
"pods.go",
"registry.go",
"replication_controllers.go",
"resource_quotas.go",
"secrets.go",
"services.go",
],
importpath = "k8s.io/kubernetes/pkg/quota/evaluator/core",
@ -32,7 +28,6 @@ go_library(
"//pkg/quota/generic:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
@ -43,8 +38,6 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/apiserver/pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)
@ -60,11 +53,12 @@ go_test(
deps = [
"//pkg/api:go_default_library",
"//pkg/quota:go_default_library",
"//pkg/quota/generic:go_default_library",
"//pkg/util/node:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

61
pkg/quota/evaluator/core/configmap.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package core
import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// listConfigMapsByNamespaceFuncUsingClient returns a configMap listing function based on the provided client.
func listConfigMapsByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().ConfigMaps(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewConfigMapEvaluator returns an evaluator that can evaluate configMaps
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewConfigMapEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listConfigMapsByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("configmaps"))
}
return &generic.ObjectCountEvaluator{
AllowCreateOnUpdate: false,
InternalGroupKind: api.Kind("ConfigMap"),
ResourceName: api.ResourceConfigMaps,
ListFuncByNamespace: listFuncByNamespace,
}
}

91
pkg/quota/evaluator/core/persistent_volume_claims.go
View File

@ -22,17 +22,13 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/initialization"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/helper"
k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1"
@ -42,6 +38,9 @@ import (
"k8s.io/kubernetes/pkg/quota/generic"
)
// the name used for object count quota
var pvcObjectCountName = generic.ObjectCountQuotaResourceNameFor(v1.SchemeGroupVersion.WithResource("persistentvolumeclaims").GroupResource())
// pvcResources are the set of static resources managed by quota associated with pvcs.
// for each resouce in this list, it may be refined dynamically based on storage class.
var pvcResources = []api.ResourceName{
@ -67,34 +66,11 @@ func V1ResourceByStorageClass(storageClass string, resourceName v1.ResourceName)
return v1.ResourceName(string(storageClass + storageClassSuffix + string(resourceName)))
}
// listPersistentVolumeClaimsByNamespaceFuncUsingClient returns a pvc listing function based on the provided client.
func listPersistentVolumeClaimsByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().PersistentVolumeClaims(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewPersistentVolumeClaimEvaluator returns an evaluator that can evaluate persistent volume claims
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewPersistentVolumeClaimEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listPersistentVolumeClaimsByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("persistentvolumeclaims"))
}
return &pvcEvaluator{
listFuncByNamespace: listFuncByNamespace,
}
func NewPersistentVolumeClaimEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, v1.SchemeGroupVersion.WithResource("persistentvolumeclaims"))
pvcEvaluator := &pvcEvaluator{listFuncByNamespace: listFuncByNamespace}
return pvcEvaluator
}
// pvcEvaluator knows how to evaluate quota usage for persistent volume claims
@ -105,45 +81,13 @@ type pvcEvaluator struct {
// Constraints verifies that all required resources are present on the item.
func (p *pvcEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error {
pvc, ok := item.(*api.PersistentVolumeClaim)
if !ok {
return fmt.Errorf("unexpected input object %v", item)
}
// these are the items that we will be handling based on the objects actual storage-class
pvcRequiredSet := append([]api.ResourceName{}, pvcResources...)
if storageClassRef := helper.GetPersistentVolumeClaimClass(pvc); len(storageClassRef) > 0 {
pvcRequiredSet = append(pvcRequiredSet, ResourceByStorageClass(storageClassRef, api.ResourcePersistentVolumeClaims))
pvcRequiredSet = append(pvcRequiredSet, ResourceByStorageClass(storageClassRef, api.ResourceRequestsStorage))
}
// in effect, this will remove things from the required set that are not tied to this pvcs storage class
// for example, if a quota has bronze and gold storage class items defined, we should not error a bronze pvc for not being gold.
// but we should error a bronze pvc if it doesn't make a storage request size...
requiredResources := quota.Intersection(required, pvcRequiredSet)
requiredSet := quota.ToSet(requiredResources)
// usage for this pvc will only include global pvc items + this storage class specific items
pvcUsage, err := p.Usage(item)
if err != nil {
return err
}
// determine what required resources were not tracked by usage.
missingSet := sets.NewString()
pvcSet := quota.ToSet(quota.ResourceNames(pvcUsage))
if diff := requiredSet.Difference(pvcSet); len(diff) > 0 {
missingSet.Insert(diff.List()...)
}
if len(missingSet) == 0 {
return nil
}
return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ","))
// no-op for persistent volume claims
return nil
}
// GroupKind that this evaluator tracks
func (p *pvcEvaluator) GroupKind() schema.GroupKind {
return api.Kind("PersistentVolumeClaim")
// GroupResource that this evaluator tracks
func (p *pvcEvaluator) GroupResource() schema.GroupResource {
return v1.SchemeGroupVersion.WithResource("persistentvolumeclaims").GroupResource()
}
// Handles returns true if the evaluator should handle the specified operation.
@ -183,6 +127,12 @@ func (p *pvcEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Ob
func (p *pvcEvaluator) MatchingResources(items []api.ResourceName) []api.ResourceName {
result := []api.ResourceName{}
for _, item := range items {
// match object count quota fields
if quota.Contains([]api.ResourceName{pvcObjectCountName}, item) {
result = append(result, item)
continue
}
// match pvc resources
if quota.Contains(pvcResources, item) {
result = append(result, item)
continue
@ -208,7 +158,8 @@ func (p *pvcEvaluator) Usage(item runtime.Object) (api.ResourceList, error) {
}
// charge for claim
result[api.ResourcePersistentVolumeClaims] = resource.MustParse("1")
result[api.ResourcePersistentVolumeClaims] = *(resource.NewQuantity(1, resource.DecimalSI))
result[pvcObjectCountName] = *(resource.NewQuantity(1, resource.DecimalSI))
if utilfeature.DefaultFeatureGate.Enabled(features.Initializers) {
if !initialization.IsInitialized(pvc.Initializers) {
// Only charge pvc count for uninitialized pvc.
@ -218,7 +169,7 @@ func (p *pvcEvaluator) Usage(item runtime.Object) (api.ResourceList, error) {
storageClassRef := helper.GetPersistentVolumeClaimClass(pvc)
if len(storageClassRef) > 0 {
storageClassClaim := api.ResourceName(storageClassRef + storageClassSuffix + string(api.ResourcePersistentVolumeClaims))
result[storageClassClaim] = resource.MustParse("1")
result[storageClassClaim] = *(resource.NewQuantity(1, resource.DecimalSI))
}
// charge for storage

182
pkg/quota/evaluator/core/persistent_volume_claims_test.go
View File

@ -21,9 +21,10 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
func testVolumeClaim(name string, namespace string, spec api.PersistentVolumeClaimSpec) *api.PersistentVolumeClaim {
@ -33,168 +34,6 @@ func testVolumeClaim(name string, namespace string, spec api.PersistentVolumeCla
}
}
func TestPersistentVolumeClaimsConstraintsFunc(t *testing.T) {
classGold := "gold"
classBronze := "bronze"
validClaim := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10G"),
},
},
})
validClaimGoldStorageClass := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10Gi"),
},
},
StorageClassName: &classGold,
})
validClaimBronzeStorageClass := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceName(api.ResourceStorage): resource.MustParse("10Gi"),
},
},
StorageClassName: &classBronze,
})
missingStorage := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{},
},
})
missingGoldStorage := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
Selector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "key2",
Operator: "Exists",
},
},
},
AccessModes: []api.PersistentVolumeAccessMode{
api.ReadWriteOnce,
api.ReadOnlyMany,
},
Resources: api.ResourceRequirements{
Requests: api.ResourceList{},
},
StorageClassName: &classGold,
})
testCases := map[string]struct {
pvc *api.PersistentVolumeClaim
required []api.ResourceName
err string
}{
"missing storage": {
pvc: missingStorage,
required: []api.ResourceName{api.ResourceRequestsStorage},
err: `must specify requests.storage`,
},
"missing gold storage": {
pvc: missingGoldStorage,
required: []api.ResourceName{ResourceByStorageClass(classGold, api.ResourceRequestsStorage)},
err: `must specify gold.storageclass.storage.k8s.io/requests.storage`,
},
"valid-claim-quota-storage": {
pvc: validClaim,
required: []api.ResourceName{api.ResourceRequestsStorage},
},
"valid-claim-quota-pvc": {
pvc: validClaim,
required: []api.ResourceName{api.ResourcePersistentVolumeClaims},
},
"valid-claim-quota-storage-and-pvc": {
pvc: validClaim,
required: []api.ResourceName{api.ResourceRequestsStorage, api.ResourcePersistentVolumeClaims},
},
"valid-claim-gold-quota-gold": {
pvc: validClaimGoldStorageClass,
required: []api.ResourceName{
api.ResourceRequestsStorage,
api.ResourcePersistentVolumeClaims,
ResourceByStorageClass(classGold, api.ResourceRequestsStorage),
ResourceByStorageClass(classGold, api.ResourcePersistentVolumeClaims),
},
},
"valid-claim-bronze-with-quota-gold": {
pvc: validClaimBronzeStorageClass,
required: []api.ResourceName{
api.ResourceRequestsStorage,
api.ResourcePersistentVolumeClaims,
ResourceByStorageClass(classGold, api.ResourceRequestsStorage),
ResourceByStorageClass(classGold, api.ResourcePersistentVolumeClaims),
},
},
}
kubeClient := fake.NewSimpleClientset()
evaluator := NewPersistentVolumeClaimEvaluator(kubeClient, nil)
for testName, test := range testCases {
err := evaluator.Constraints(test.required, test.pvc)
switch {
case err != nil && len(test.err) == 0,
err == nil && len(test.err) != 0,
err != nil && test.err != err.Error():
t.Errorf("%s unexpected error: %v", testName, err)
}
}
}
func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) {
classGold := "gold"
validClaim := testVolumeClaim("foo", "ns", api.PersistentVolumeClaimSpec{
@ -237,8 +76,7 @@ func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) {
StorageClassName: &classGold,
})
kubeClient := fake.NewSimpleClientset()
evaluator := NewPersistentVolumeClaimEvaluator(kubeClient, nil)
evaluator := NewPersistentVolumeClaimEvaluator(nil)
testCases := map[string]struct {
pvc *api.PersistentVolumeClaim
usage api.ResourceList
@ -246,17 +84,19 @@ func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) {
"pvc-usage": {
pvc: validClaim,
usage: api.ResourceList{
api.ResourceRequestsStorage: resource.MustParse("10Gi"),
api.ResourcePersistentVolumeClaims: resource.MustParse("1"),
api.ResourceRequestsStorage: resource.MustParse("10Gi"),
api.ResourcePersistentVolumeClaims: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"),
},
},
"pvc-usage-by-class": {
pvc: validClaimByStorageClass,
usage: api.ResourceList{
api.ResourceRequestsStorage: resource.MustParse("10Gi"),
api.ResourcePersistentVolumeClaims: resource.MustParse("1"),
ResourceByStorageClass(classGold, api.ResourceRequestsStorage): resource.MustParse("10Gi"),
ResourceByStorageClass(classGold, api.ResourcePersistentVolumeClaims): resource.MustParse("1"),
api.ResourceRequestsStorage: resource.MustParse("10Gi"),
api.ResourcePersistentVolumeClaims: resource.MustParse("1"),
ResourceByStorageClass(classGold, api.ResourceRequestsStorage): resource.MustParse("10Gi"),
ResourceByStorageClass(classGold, api.ResourcePersistentVolumeClaims): resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "persistentvolumeclaims"}): resource.MustParse("1"),
},
},
}

101
pkg/quota/evaluator/core/pods.go
View File

@ -23,20 +23,14 @@ import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/initialization"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/helper/qos"
k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1"
@ -46,8 +40,12 @@ import (
"k8s.io/kubernetes/pkg/quota/generic"
)
// the name used for object count quota
var podObjectCountName = generic.ObjectCountQuotaResourceNameFor(v1.SchemeGroupVersion.WithResource("pods").GroupResource())
// podResources are the set of resources managed by quota associated with pods.
var podResources = []api.ResourceName{
podObjectCountName,
api.ResourceCPU,
api.ResourceMemory,
api.ResourceEphemeralStorage,
@ -60,35 +58,24 @@ var podResources = []api.ResourceName{
api.ResourcePods,
}
// listPodsByNamespaceFuncUsingClient returns a pod listing function based on the provided client.
func listPodsByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().Pods(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NOTE: it was a mistake, but if a quota tracks cpu or memory related resources,
// the incoming pod is required to have those values set. we should not repeat
// this mistake for other future resources (gpus, ephemeral-storage,etc).
// do not add more resources to this list!
var validationSet = sets.NewString(
string(api.ResourceCPU),
string(api.ResourceMemory),
string(api.ResourceRequestsCPU),
string(api.ResourceRequestsMemory),
string(api.ResourceLimitsCPU),
string(api.ResourceLimitsMemory),
)
// NewPodEvaluator returns an evaluator that can evaluate pods
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewPodEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory, clock clock.Clock) quota.Evaluator {
listFuncByNamespace := listPodsByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("pods"))
}
return &podEvaluator{
listFuncByNamespace: listFuncByNamespace,
clock: clock,
}
func NewPodEvaluator(f quota.ListerForResourceFunc, clock clock.Clock) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, v1.SchemeGroupVersion.WithResource("pods"))
podEvaluator := &podEvaluator{listFuncByNamespace: listFuncByNamespace, clock: clock}
return podEvaluator
}
// podEvaluator knows how to measure usage of pods.
@ -110,6 +97,7 @@ func (p *podEvaluator) Constraints(required []api.ResourceName, item runtime.Obj
// Pod level resources are often set during admission control
// As a consequence, we want to verify that resources are valid prior
// to ever charging quota prematurely in case they are not.
// TODO remove this entire section when we have a validation step in admission.
allErrs := field.ErrorList{}
fldPath := field.NewPath("spec").Child("containers")
for i, ctr := range pod.Spec.Containers {
@ -123,10 +111,11 @@ func (p *podEvaluator) Constraints(required []api.ResourceName, item runtime.Obj
return allErrs.ToAggregate()
}
// TODO: fix this when we have pod level resource requirements
// since we do not yet pod level requests/limits, we need to ensure each
// container makes an explict request or limit for a quota tracked resource
requiredSet := quota.ToSet(required)
// BACKWARD COMPATIBILITY REQUIREMENT: if we quota cpu or memory, then each container
// must make an explicit request for the resource. this was a mistake. it coupled
// validation with resource counting, but we did this before QoS was even defined.
// let's not make that mistake again with other resources now that QoS is defined.
requiredSet := quota.ToSet(required).Intersection(validationSet)
missingSet := sets.NewString()
for i := range pod.Spec.Containers {
enforcePodContainerConstraints(&pod.Spec.Containers[i], requiredSet, missingSet)
@ -140,9 +129,9 @@ func (p *podEvaluator) Constraints(required []api.ResourceName, item runtime.Obj
return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ","))
}
// GroupKind that this evaluator tracks
func (p *podEvaluator) GroupKind() schema.GroupKind {
return api.Kind("Pod")
// GroupResource that this evaluator tracks
func (p *podEvaluator) GroupResource() schema.GroupResource {
return v1.SchemeGroupVersion.WithResource("pods").GroupResource()
}
// Handles returns true if the evaluator should handle the specified attributes.
@ -190,7 +179,7 @@ var _ quota.Evaluator = &podEvaluator{}
func enforcePodContainerConstraints(container *api.Container, requiredSet, missingSet sets.String) {
requests := container.Resources.Requests
limits := container.Resources.Limits
containerUsage := podUsageHelper(requests, limits)
containerUsage := podComputeUsageHelper(requests, limits)
containerSet := quota.ToSet(quota.ResourceNames(containerUsage))
if !containerSet.Equal(requiredSet) {
difference := requiredSet.Difference(containerSet)
@ -198,8 +187,8 @@ func enforcePodContainerConstraints(container *api.Container, requiredSet, missi
}
}
// podUsageHelper can summarize the pod quota usage based on requests and limits
func podUsageHelper(requests api.ResourceList, limits api.ResourceList) api.ResourceList {
// podComputeUsageHelper can summarize the pod compute quota usage based on requests and limits
func podComputeUsageHelper(requests api.ResourceList, limits api.ResourceList) api.ResourceList {
result := api.ResourceList{}
result[api.ResourcePods] = resource.MustParse("1")
if request, found := requests[api.ResourceCPU]; found {
@ -269,18 +258,21 @@ func PodUsageFunc(obj runtime.Object, clock clock.Clock) (api.ResourceList, erro
if err != nil {
return api.ResourceList{}, err
}
// by convention, we do not quota pods that have reached end-of life
// always quota the object count (even if the pod is end of life)
// object count quotas track all objects that are in storage.
// where "pods" tracks all pods that have not reached a terminal state,
// count/pods tracks all pods independent of state.
result := api.ResourceList{
podObjectCountName: *(resource.NewQuantity(1, resource.DecimalSI)),
}
// by convention, we do not quota compute resources that have reached end-of life
// note: the "pods" resource is considered a compute resource since it is tied to life-cycle.
if !QuotaPod(pod, clock) {
return api.ResourceList{}, nil
}
// Only charge pod count for uninitialized pod.
if utilfeature.DefaultFeatureGate.Enabled(features.Initializers) {
if !initialization.IsInitialized(pod.Initializers) {
result := api.ResourceList{}
result[api.ResourcePods] = resource.MustParse("1")
return result, nil
}
return result, nil
}
requests := api.ResourceList{}
limits := api.ResourceList{}
// TODO: ideally, we have pod level requests and limits in the future.
@ -296,7 +288,8 @@ func PodUsageFunc(obj runtime.Object, clock clock.Clock) (api.ResourceList, erro
limits = quota.Max(limits, pod.Spec.InitContainers[i].Resources.Limits)
}
return podUsageHelper(requests, limits), nil
result = quota.Add(result, podComputeUsageHelper(requests, limits))
return result, nil
}
func isBestEffort(pod *api.Pod) bool {

21
pkg/quota/evaluator/core/pods_test.go
View File

@ -22,10 +22,11 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
"k8s.io/kubernetes/pkg/util/node"
)
@ -90,8 +91,7 @@ func TestPodConstraintsFunc(t *testing.T) {
err: `must specify memory`,
},
}
kubeClient := fake.NewSimpleClientset()
evaluator := NewPodEvaluator(kubeClient, nil, clock.RealClock{})
evaluator := NewPodEvaluator(nil, clock.RealClock{})
for testName, test := range testCases {
err := evaluator.Constraints(test.required, test.pod)
switch {
@ -104,9 +104,8 @@ func TestPodConstraintsFunc(t *testing.T) {
}
func TestPodEvaluatorUsage(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
fakeClock := clock.NewFakeClock(time.Now())
evaluator := NewPodEvaluator(kubeClient, nil, fakeClock)
evaluator := NewPodEvaluator(nil, fakeClock)
// fields use to simulate a pod undergoing termination
// note: we set the deletion time in the past
@ -135,6 +134,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceLimitsCPU: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("1m"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"init container MEM": {
@ -153,6 +153,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceLimitsMemory: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("1m"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"init container local ephemeral storage": {
@ -171,6 +172,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceRequestsEphemeralStorage: resource.MustParse("32Mi"),
api.ResourceLimitsEphemeralStorage: resource.MustParse("64Mi"),
api.ResourcePods: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"container CPU": {
@ -189,6 +191,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceLimitsCPU: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("1m"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"container MEM": {
@ -207,6 +210,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceLimitsMemory: resource.MustParse("2m"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("1m"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"container local ephemeral storage": {
@ -225,6 +229,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceRequestsEphemeralStorage: resource.MustParse("32Mi"),
api.ResourceLimitsEphemeralStorage: resource.MustParse("64Mi"),
api.ResourcePods: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"init container maximums override sum of containers": {
@ -292,6 +297,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"pod deletion timestamp exceeded": {
@ -321,7 +327,9 @@ func TestPodEvaluatorUsage(t *testing.T) {
},
},
},
usage: api.ResourceList{},
usage: api.ResourceList{
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"pod deletion timestamp not exceeded": {
pod: &api.Pod{
@ -352,6 +360,7 @@ func TestPodEvaluatorUsage(t *testing.T) {
api.ResourceLimitsCPU: resource.MustParse("2"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
}

47
pkg/quota/evaluator/core/registry.go
View File

@ -17,33 +17,34 @@ limitations under the License.
package core
import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// NewRegistry returns a registry that knows how to deal with core kubernetes resources
// If an informer factory is provided, evaluators will use them.
func NewRegistry(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Registry {
pod := NewPodEvaluator(kubeClient, f, clock.RealClock{})
service := NewServiceEvaluator(kubeClient, f)
replicationController := NewReplicationControllerEvaluator(kubeClient, f)
resourceQuota := NewResourceQuotaEvaluator(kubeClient, f)
secret := NewSecretEvaluator(kubeClient, f)
configMap := NewConfigMapEvaluator(kubeClient, f)
persistentVolumeClaim := NewPersistentVolumeClaimEvaluator(kubeClient, f)
return &generic.GenericRegistry{
InternalEvaluators: map[schema.GroupKind]quota.Evaluator{
pod.GroupKind(): pod,
service.GroupKind(): service,
replicationController.GroupKind(): replicationController,
secret.GroupKind(): secret,
configMap.GroupKind(): configMap,
resourceQuota.GroupKind(): resourceQuota,
persistentVolumeClaim.GroupKind(): persistentVolumeClaim,
},
}
// legacyObjectCountAliases are what we used to do simple object counting quota with mapped to alias
var legacyObjectCountAliases = map[schema.GroupVersionResource]api.ResourceName{
v1.SchemeGroupVersion.WithResource("configmaps"): api.ResourceConfigMaps,
v1.SchemeGroupVersion.WithResource("resourcequotas"): api.ResourceQuotas,
v1.SchemeGroupVersion.WithResource("replicationcontrollers"): api.ResourceReplicationControllers,
v1.SchemeGroupVersion.WithResource("secrets"): api.ResourceSecrets,
}
// NewEvaluators returns the list of static evaluators that manage more than counts
func NewEvaluators(f quota.ListerForResourceFunc) []quota.Evaluator {
// these evaluators have special logic
result := []quota.Evaluator{
NewPodEvaluator(f, clock.RealClock{}),
NewServiceEvaluator(f),
NewPersistentVolumeClaimEvaluator(f),
}
// these evaluators require an alias for backwards compatibility
for gvr, alias := range legacyObjectCountAliases {
result = append(result,
generic.NewObjectCountEvaluator(false, gvr.GroupResource(), generic.ListResourceUsingListerFunc(f, gvr), alias))
}
return result
}

61
pkg/quota/evaluator/core/replication_controllers.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package core
import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// listReplicationControllersByNamespaceFuncUsingClient returns a replicationController listing function based on the provided client.
func listReplicationControllersByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().ReplicationControllers(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewReplicationControllerEvaluator returns an evaluator that can evaluate replicationControllers
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewReplicationControllerEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listReplicationControllersByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("replicationcontrollers"))
}
return &generic.ObjectCountEvaluator{
AllowCreateOnUpdate: false,
InternalGroupKind: api.Kind("ReplicationController"),
ResourceName: api.ResourceReplicationControllers,
ListFuncByNamespace: listFuncByNamespace,
}
}

61
pkg/quota/evaluator/core/resource_quotas.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package core
import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// listResourceQuotasByNamespaceFuncUsingClient returns a resourceQuota listing function based on the provided client.
func listResourceQuotasByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().ResourceQuotas(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewResourceQuotaEvaluator returns an evaluator that can evaluate resourceQuotas
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewResourceQuotaEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listResourceQuotasByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("resourcequotas"))
}
return &generic.ObjectCountEvaluator{
AllowCreateOnUpdate: false,
InternalGroupKind: api.Kind("ResourceQuota"),
ResourceName: api.ResourceQuotas,
ListFuncByNamespace: listFuncByNamespace,
}
}

61
pkg/quota/evaluator/core/secrets.go
View File

@ -1,61 +0,0 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package core
import (
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// listSecretsByNamespaceFuncUsingClient returns a secret listing function based on the provided client.
func listSecretsByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().Secrets(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewSecretEvaluator returns an evaluator that can evaluate secrets
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewSecretEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listSecretsByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("secrets"))
}
return &generic.ObjectCountEvaluator{
AllowCreateOnUpdate: false,
InternalGroupKind: api.Kind("Secret"),
ResourceName: api.ResourceSecrets,
ListFuncByNamespace: listFuncByNamespace,
}
}

71
pkg/quota/evaluator/core/services.go
View File

@ -18,58 +18,34 @@ package core
import (
"fmt"
"strings"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api"
k8s_api_v1 "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
// the name used for object count quota
var serviceObjectCountName = generic.ObjectCountQuotaResourceNameFor(v1.SchemeGroupVersion.WithResource("services").GroupResource())
// serviceResources are the set of resources managed by quota associated with services.
var serviceResources = []api.ResourceName{
serviceObjectCountName,
api.ResourceServices,
api.ResourceServicesNodePorts,
api.ResourceServicesLoadBalancers,
}
// listServicesByNamespaceFuncUsingClient returns a service listing function based on the provided client.
func listServicesByNamespaceFuncUsingClient(kubeClient clientset.Interface) generic.ListFuncByNamespace {
// TODO: ideally, we could pass dynamic client pool down into this code, and have one way of doing this.
// unfortunately, dynamic client works with Unstructured objects, and when we calculate Usage, we require
// structured objects.
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
itemList, err := kubeClient.CoreV1().Services(namespace).List(options)
if err != nil {
return nil, err
}
results := make([]runtime.Object, 0, len(itemList.Items))
for i := range itemList.Items {
results = append(results, &itemList.Items[i])
}
return results, nil
}
}
// NewServiceEvaluator returns an evaluator that can evaluate services
// if the specified shared informer factory is not nil, evaluator may use it to support listing functions.
func NewServiceEvaluator(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Evaluator {
listFuncByNamespace := listServicesByNamespaceFuncUsingClient(kubeClient)
if f != nil {
listFuncByNamespace = generic.ListResourceUsingInformerFunc(f, v1.SchemeGroupVersion.WithResource("services"))
}
return &serviceEvaluator{
listFuncByNamespace: listFuncByNamespace,
}
// NewServiceEvaluator returns an evaluator that can evaluate services.
func NewServiceEvaluator(f quota.ListerForResourceFunc) quota.Evaluator {
listFuncByNamespace := generic.ListResourceUsingListerFunc(f, v1.SchemeGroupVersion.WithResource("services"))
serviceEvaluator := &serviceEvaluator{listFuncByNamespace: listFuncByNamespace}
return serviceEvaluator
}
// serviceEvaluator knows how to measure usage for services.
@ -80,31 +56,13 @@ type serviceEvaluator struct {
// Constraints verifies that all required resources are present on the item
func (p *serviceEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error {
service, ok := item.(*api.Service)
if !ok {
return fmt.Errorf("unexpected input object %v", item)
}
requiredSet := quota.ToSet(required)
missingSet := sets.NewString()
serviceUsage, err := p.Usage(service)
if err != nil {
return err
}
serviceSet := quota.ToSet(quota.ResourceNames(serviceUsage))
if diff := requiredSet.Difference(serviceSet); len(diff) > 0 {
missingSet.Insert(diff.List()...)
}
if len(missingSet) == 0 {
return nil
}
return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ","))
// this is a no-op for services
return nil
}
// GroupKind that this evaluator tracks
func (p *serviceEvaluator) GroupKind() schema.GroupKind {
return api.Kind("Service")
// GroupResource that this evaluator tracks
func (p *serviceEvaluator) GroupResource() schema.GroupResource {
return v1.SchemeGroupVersion.WithResource("services").GroupResource()
}
// Handles returns true of the evaluator should handle the specified operation.
@ -149,6 +107,7 @@ func (p *serviceEvaluator) Usage(item runtime.Object) (api.ResourceList, error)
}
ports := len(svc.Spec.Ports)
// default service usage
result[serviceObjectCountName] = *(resource.NewQuantity(1, resource.DecimalSI))
result[api.ResourceServices] = *(resource.NewQuantity(1, resource.DecimalSI))
result[api.ResourceServicesLoadBalancers] = resource.Quantity{Format: resource.DecimalSI}
result[api.ResourceServicesNodePorts] = resource.Quantity{Format: resource.DecimalSI}

35
pkg/quota/evaluator/core/services_test.go
View File

@ -20,14 +20,14 @@ import (
"testing"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/generic"
)
func TestServiceEvaluatorMatchesResources(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
evaluator := NewServiceEvaluator(kubeClient, nil)
evaluator := NewServiceEvaluator(nil)
// we give a lot of resources
input := []api.ResourceName{
api.ResourceConfigMaps,
@ -49,8 +49,7 @@ func TestServiceEvaluatorMatchesResources(t *testing.T) {
}
func TestServiceEvaluatorUsage(t *testing.T) {
kubeClient := fake.NewSimpleClientset()
evaluator := NewServiceEvaluator(kubeClient, nil)
evaluator := NewServiceEvaluator(nil)
testCases := map[string]struct {
service *api.Service
usage api.ResourceList
@ -65,6 +64,7 @@ func TestServiceEvaluatorUsage(t *testing.T) {
api.ResourceServicesNodePorts: resource.MustParse("0"),
api.ResourceServicesLoadBalancers: resource.MustParse("1"),
api.ResourceServices: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "services"}): resource.MustParse("1"),
},
},
"loadbalancer_ports": {
@ -82,6 +82,7 @@ func TestServiceEvaluatorUsage(t *testing.T) {
api.ResourceServicesNodePorts: resource.MustParse("1"),
api.ResourceServicesLoadBalancers: resource.MustParse("1"),
api.ResourceServices: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "services"}): resource.MustParse("1"),
},
},
"clusterip": {
@ -91,9 +92,10 @@ func TestServiceEvaluatorUsage(t *testing.T) {
},
},
usage: api.ResourceList{
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("0"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("0"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "services"}): resource.MustParse("1"),
},
},
"nodeports": {
@ -108,9 +110,10 @@ func TestServiceEvaluatorUsage(t *testing.T) {
},
},
usage: api.ResourceList{
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("1"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("1"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "services"}): resource.MustParse("1"),
},
},
"multi-nodeports": {
@ -128,9 +131,10 @@ func TestServiceEvaluatorUsage(t *testing.T) {
},
},
usage: api.ResourceList{
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("2"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
api.ResourceServices: resource.MustParse("1"),
api.ResourceServicesNodePorts: resource.MustParse("2"),
api.ResourceServicesLoadBalancers: resource.MustParse("0"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "services"}): resource.MustParse("1"),
},
},
}
@ -198,8 +202,7 @@ func TestServiceConstraintsFunc(t *testing.T) {
},
}
kubeClient := fake.NewSimpleClientset()
evaluator := NewServiceEvaluator(kubeClient, nil)
evaluator := NewServiceEvaluator(nil)
for testName, test := range testCases {
err := evaluator.Constraints(test.required, test.service)
switch {

3
pkg/quota/generic/BUILD
View File

@ -8,6 +8,7 @@ load(
go_library(
name = "go_default_library",
srcs = [
"configuration.go",
"evaluator.go",
"registry.go",
],
@ -16,12 +17,12 @@ go_library(
"//pkg/api:go_default_library",
"//pkg/quota:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)

44
pkg/quota/generic/configuration.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generic
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/quota"
)
// implements a basic configuration
type simpleConfiguration struct {
evaluators []quota.Evaluator
ignoredResources map[schema.GroupResource]struct{}
}
// NewConfiguration creates a quota configuration
func NewConfiguration(evaluators []quota.Evaluator, ignoredResources map[schema.GroupResource]struct{}) quota.Configuration {
return &simpleConfiguration{
evaluators: evaluators,
ignoredResources: ignoredResources,
}
}
func (c *simpleConfiguration) IgnoredResources() map[schema.GroupResource]struct{} {
return c.ignoredResources
}
func (c *simpleConfiguration) Evaluators() []quota.Evaluator {
return c.evaluators
}

127
pkg/quota/generic/evaluator.go
View File

@ -20,33 +20,51 @@ import (
"fmt"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/informers"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/quota"
)
// ListResourceUsingInformerFunc returns a listing function based on the shared informer factory for the specified resource.
func ListResourceUsingInformerFunc(f informers.SharedInformerFactory, resource schema.GroupVersionResource) ListFuncByNamespace {
return func(namespace string, options metav1.ListOptions) ([]runtime.Object, error) {
labelSelector, err := labels.Parse(options.LabelSelector)
// InformerForResourceFunc knows how to provision an informer
type InformerForResourceFunc func(schema.GroupVersionResource) (informers.GenericInformer, error)
// ListerFuncForResourceFunc knows how to provision a lister from an informer func
func ListerFuncForResourceFunc(f InformerForResourceFunc) quota.ListerForResourceFunc {
return func(gvr schema.GroupVersionResource) (cache.GenericLister, error) {
informer, err := f(gvr)
if err != nil {
return nil, err
}
informer, err := f.ForResource(resource)
if err != nil {
return nil, err
}
return informer.Lister().ByNamespace(namespace).List(labelSelector)
return informer.Lister(), nil
}
}
// ListResourceUsingListerFunc returns a listing function based on the shared informer factory for the specified resource.
func ListResourceUsingListerFunc(l quota.ListerForResourceFunc, resource schema.GroupVersionResource) ListFuncByNamespace {
return func(namespace string) ([]runtime.Object, error) {
lister, err := l(resource)
if err != nil {
return nil, err
}
return lister.ByNamespace(namespace).List(labels.Everything())
}
}
// ObjectCountQuotaResourceNameFor returns the object count quota name for specified groupResource
func ObjectCountQuotaResourceNameFor(groupResource schema.GroupResource) api.ResourceName {
if len(groupResource.Group) == 0 {
return api.ResourceName("count/" + groupResource.Resource)
}
return api.ResourceName("count/" + groupResource.Resource + "." + groupResource.Group)
}
// ListFuncByNamespace knows how to list resources in a namespace
type ListFuncByNamespace func(namespace string, options metav1.ListOptions) ([]runtime.Object, error)
type ListFuncByNamespace func(namespace string) ([]runtime.Object, error)
// MatchesScopeFunc knows how to evaluate if an object matches a scope
type MatchesScopeFunc func(scope api.ResourceQuotaScope, object runtime.Object) (bool, error)
@ -91,9 +109,7 @@ func CalculateUsageStats(options quota.UsageStatsOptions,
for _, resourceName := range options.Resources {
result.Used[resourceName] = resource.Quantity{Format: resource.DecimalSI}
}
items, err := listFunc(options.Namespace, metav1.ListOptions{
LabelSelector: labels.Everything().String(),
})
items, err := listFunc(options.Namespace)
if err != nil {
return result, fmt.Errorf("failed to list content: %v", err)
}
@ -121,63 +137,86 @@ func CalculateUsageStats(options quota.UsageStatsOptions,
return result, nil
}
// ObjectCountEvaluator provides an implementation for quota.Evaluator
// objectCountEvaluator provides an implementation for quota.Evaluator
// that associates usage of the specified resource based on the number of items
// returned by the specified listing function.
type ObjectCountEvaluator struct {
// AllowCreateOnUpdate if true will ensure the evaluator tracks create
type objectCountEvaluator struct {
// allowCreateOnUpdate if true will ensure the evaluator tracks create
// and update operations.
AllowCreateOnUpdate bool
// GroupKind that this evaluator tracks.
InternalGroupKind schema.GroupKind
allowCreateOnUpdate bool
// GroupResource that this evaluator tracks.
// It is used to construct a generic object count quota name
groupResource schema.GroupResource
// A function that knows how to list resources by namespace.
// TODO move to dynamic client in future
ListFuncByNamespace ListFuncByNamespace
// Name associated with this resource in the quota.
ResourceName api.ResourceName
listFuncByNamespace ListFuncByNamespace
// Names associated with this resource in the quota for generic counting.
resourceNames []api.ResourceName
}
// Constraints returns an error if the configured resource name is not in the required set.
func (o *ObjectCountEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error {
if !quota.Contains(required, o.ResourceName) {
return fmt.Errorf("missing %s", o.ResourceName)
}
func (o *objectCountEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error {
// no-op for object counting
return nil
}
// GroupKind that this evaluator tracks
func (o *ObjectCountEvaluator) GroupKind() schema.GroupKind {
return o.InternalGroupKind
}
// Handles returns true if the object count evaluator needs to track this attributes.
func (o *ObjectCountEvaluator) Handles(a admission.Attributes) bool {
func (o *objectCountEvaluator) Handles(a admission.Attributes) bool {
operation := a.GetOperation()
return operation == admission.Create || (o.AllowCreateOnUpdate && operation == admission.Update)
return operation == admission.Create || (o.allowCreateOnUpdate && operation == admission.Update)
}
// Matches returns true if the evaluator matches the specified quota with the provided input item
func (o *ObjectCountEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Object) (bool, error) {
func (o *objectCountEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Object) (bool, error) {
return Matches(resourceQuota, item, o.MatchingResources, MatchesNoScopeFunc)
}
// MatchingResources takes the input specified list of resources and returns the set of resources it matches.
func (o *ObjectCountEvaluator) MatchingResources(input []api.ResourceName) []api.ResourceName {
return quota.Intersection(input, []api.ResourceName{o.ResourceName})
func (o *objectCountEvaluator) MatchingResources(input []api.ResourceName) []api.ResourceName {
return quota.Intersection(input, o.resourceNames)
}
// Usage returns the resource usage for the specified object
func (o *ObjectCountEvaluator) Usage(object runtime.Object) (api.ResourceList, error) {
func (o *objectCountEvaluator) Usage(object runtime.Object) (api.ResourceList, error) {
quantity := resource.NewQuantity(1, resource.DecimalSI)
return api.ResourceList{
o.ResourceName: *quantity,
}, nil
resourceList := api.ResourceList{}
for _, resourceName := range o.resourceNames {
resourceList[resourceName] = *quantity
}
return resourceList, nil
}
// GroupResource tracked by this evaluator
func (o *objectCountEvaluator) GroupResource() schema.GroupResource {
return o.groupResource
}
// UsageStats calculates aggregate usage for the object.
func (o *ObjectCountEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
return CalculateUsageStats(options, o.ListFuncByNamespace, MatchesNoScopeFunc, o.Usage)
func (o *objectCountEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
return CalculateUsageStats(options, o.listFuncByNamespace, MatchesNoScopeFunc, o.Usage)
}
// Verify implementation of interface at compile time.
var _ quota.Evaluator = &ObjectCountEvaluator{}
var _ quota.Evaluator = &objectCountEvaluator{}
// NewObjectCountEvaluator returns an evaluator that can perform generic
// object quota counting. It allows an optional alias for backwards compatibilty
// purposes for the legacy object counting names in quota. Unless its supporting
// backward compatibility, alias should not be used.
func NewObjectCountEvaluator(
allowCreateOnUpdate bool,
groupResource schema.GroupResource, listFuncByNamespace ListFuncByNamespace,
alias api.ResourceName) quota.Evaluator {
resourceNames := []api.ResourceName{ObjectCountQuotaResourceNameFor(groupResource)}
if len(alias) > 0 {
resourceNames = append(resourceNames, alias)
}
return &objectCountEvaluator{
allowCreateOnUpdate: allowCreateOnUpdate,
groupResource: groupResource,
listFuncByNamespace: listFuncByNamespace,
resourceNames: resourceNames,
}
}

65
pkg/quota/generic/registry.go
View File

@ -17,20 +17,65 @@ limitations under the License.
package generic
import (
"sync"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/quota"
)
// Ensure it implements the required interface
var _ quota.Registry = &GenericRegistry{}
// GenericRegistry implements Registry
type GenericRegistry struct {
// internal evaluators by group kind
InternalEvaluators map[schema.GroupKind]quota.Evaluator
// implements a basic registry
type simpleRegistry struct {
lock sync.RWMutex
// evaluators tracked by the registry
evaluators map[schema.GroupResource]quota.Evaluator
}
// Evaluators returns the map of evaluators by groupKind
func (r *GenericRegistry) Evaluators() map[schema.GroupKind]quota.Evaluator {
return r.InternalEvaluators
// NewRegistry creates a simple registry with initial list of evaluators
func NewRegistry(evaluators []quota.Evaluator) quota.Registry {
return &simpleRegistry{
evaluators: evaluatorsByGroupResource(evaluators),
}
}
func (r *simpleRegistry) Add(e quota.Evaluator) {
r.lock.Lock()
defer r.lock.Unlock()
r.evaluators[e.GroupResource()] = e
}
func (r *simpleRegistry) Remove(e quota.Evaluator) {
r.lock.Lock()
defer r.lock.Unlock()
delete(r.evaluators, e.GroupResource())
}
func (r *simpleRegistry) Get(gr schema.GroupResource) quota.Evaluator {
r.lock.RLock()
defer r.lock.RUnlock()
return r.evaluators[gr]
}
func (r *simpleRegistry) List() []quota.Evaluator {
r.lock.RLock()
defer r.lock.RUnlock()
return evaluatorsList(r.evaluators)
}
// evaluatorsByGroupResource converts a list of evaluators to a map by group resource.
func evaluatorsByGroupResource(items []quota.Evaluator) map[schema.GroupResource]quota.Evaluator {
result := map[schema.GroupResource]quota.Evaluator{}
for _, item := range items {
result[item.GroupResource()] = item
}
return result
}
// evaluatorsList converts a map of evaluators to list
func evaluatorsList(input map[schema.GroupResource]quota.Evaluator) []quota.Evaluator {
var result []quota.Evaluator
for _, item := range input {
result = append(result, item)
}
return result
}

4
pkg/quota/install/BUILD
View File

@ -12,8 +12,8 @@ go_library(
deps = [
"//pkg/quota:go_default_library",
"//pkg/quota/evaluator/core:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//pkg/quota/generic:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
],
)

41
pkg/quota/install/registry.go
View File

@ -17,15 +17,42 @@ limitations under the License.
package install
import (
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/kubernetes/pkg/quota"
"k8s.io/kubernetes/pkg/quota/evaluator/core"
"k8s.io/kubernetes/pkg/quota/generic"
)
// NewRegistry returns a registry of quota evaluators.
// If a shared informer factory is provided, it is used by evaluators rather than performing direct queries.
func NewRegistry(kubeClient clientset.Interface, f informers.SharedInformerFactory) quota.Registry {
// TODO: when quota supports resources in other api groups, we will need to merge
return core.NewRegistry(kubeClient, f)
// NewQuotaConfigurationForAdmission returns a quota configuration for admission control.
func NewQuotaConfigurationForAdmission() quota.Configuration {
evaluators := core.NewEvaluators(nil)
return generic.NewConfiguration(evaluators, DefaultIgnoredResources())
}
// NewQuotaConfigurationForControllers returns a quota configuration for controllers.
func NewQuotaConfigurationForControllers(f quota.ListerForResourceFunc) quota.Configuration {
evaluators := core.NewEvaluators(f)
return generic.NewConfiguration(evaluators, DefaultIgnoredResources())
}
// ignoredResources are ignored by quota by default
var ignoredResources = map[schema.GroupResource]struct{}{
{Group: "extensions", Resource: "replicationcontrollers"}: {},
{Group: "extensions", Resource: "networkpolicies"}: {},
{Group: "", Resource: "bindings"}: {},
{Group: "", Resource: "componentstatuses"}: {},
{Group: "", Resource: "events"}: {},
{Group: "authentication.k8s.io", Resource: "tokenreviews"}: {},
{Group: "authorization.k8s.io", Resource: "subjectaccessreviews"}: {},
{Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"}: {},
{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"}: {},
{Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"}: {},
{Group: "apiregistration.k8s.io", Resource: "apiservices"}: {},
{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}: {},
}
// DefaultIgnoredResources returns the default set of resources that quota system
// should ignore. This is exposed so downstream integrators can have access to them.
func DefaultIgnoredResources() map[schema.GroupResource]struct{} {
return ignoredResources
}

45
pkg/quota/interfaces.go
View File

@ -20,6 +20,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/admission"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/api"
)
@ -39,12 +40,12 @@ type UsageStats struct {
Used api.ResourceList
}
// Evaluator knows how to evaluate quota usage for a particular group kind
// Evaluator knows how to evaluate quota usage for a particular group resource
type Evaluator interface {
// Constraints ensures that each required resource is present on item
Constraints(required []api.ResourceName, item runtime.Object) error
// GroupKind returns the groupKind that this object knows how to evaluate
GroupKind() schema.GroupKind
// GroupResource returns the groupResource that this object knows how to evaluate
GroupResource() schema.GroupResource
// Handles determines if quota could be impacted by the specified attribute.
// If true, admission control must perform quota processing for the operation, otherwise it is safe to ignore quota.
Handles(operation admission.Attributes) bool
@ -58,25 +59,25 @@ type Evaluator interface {
UsageStats(options UsageStatsOptions) (UsageStats, error)
}
// Registry holds the list of evaluators associated to a particular group kind
// Configuration defines how the quota system is configured.
type Configuration interface {
// IgnoredResources are ignored by quota.
IgnoredResources() map[schema.GroupResource]struct{}
// Evaluators for quota evaluation.
Evaluators() []Evaluator
}
// Registry maintains a list of evaluators
type Registry interface {
// Evaluators returns the set Evaluator objects registered to a groupKind
Evaluators() map[schema.GroupKind]Evaluator
// Add to registry
Add(e Evaluator)
// Remove from registry
Remove(e Evaluator)
// Get by group resource
Get(gr schema.GroupResource) Evaluator
// List from registry
List() []Evaluator
}
// UnionRegistry combines multiple registries. Order matters because first registry to claim a GroupKind
// is the "winner"
type UnionRegistry []Registry
// Evaluators returns a mapping of evaluators by group kind.
func (r UnionRegistry) Evaluators() map[schema.GroupKind]Evaluator {
ret := map[schema.GroupKind]Evaluator{}
for i := len(r) - 1; i >= 0; i-- {
for k, v := range r[i].Evaluators() {
ret[k] = v
}
}
return ret
}
// ListerForResourceFunc knows how to get a lister for a specific resource
type ListerForResourceFunc func(schema.GroupVersionResource) (cache.GenericLister, error)

2
pkg/quota/resources.go
View File

@ -247,7 +247,7 @@ func CalculateUsage(namespaceName string, scopes []api.ResourceQuotaScope, hardL
// look to measure updated usage stats for
hardResources := ResourceNames(hardLimits)
potentialResources := []api.ResourceName{}
evaluators := registry.Evaluators()
evaluators := registry.List()
for _, evaluator := range evaluators {
potentialResources = append(potentialResources, evaluator.MatchingResources(hardResources)...)
}