From 553c4701af8b2f869272fd649cdee6c439e5faed Mon Sep 17 00:00:00 2001 From: derekwaynecarr Date: Mon, 22 Feb 2016 11:14:25 -0500 Subject: [PATCH] Add quota evaluator framework --- pkg/quota/evaluator/core/doc.go | 18 ++ .../core/persistent_volume_claims.go | 45 ++++ pkg/quota/evaluator/core/pods.go | 183 ++++++++++++++ pkg/quota/evaluator/core/registry.go | 44 ++++ .../evaluator/core/replication_controllers.go | 45 ++++ pkg/quota/evaluator/core/resource_quotas.go | 45 ++++ pkg/quota/evaluator/core/secrets.go | 45 ++++ pkg/quota/evaluator/core/services.go | 45 ++++ pkg/quota/generic/evaluator.go | 199 ++++++++++++++++ pkg/quota/generic/registry.go | 36 +++ pkg/quota/install/registry.go | 30 +++ pkg/quota/interfaces.go | 66 ++++++ pkg/quota/resources.go | 159 +++++++++++++ pkg/quota/resources_test.go | 223 ++++++++++++++++++ 14 files changed, 1183 insertions(+) create mode 100644 pkg/quota/evaluator/core/doc.go create mode 100644 pkg/quota/evaluator/core/persistent_volume_claims.go create mode 100644 pkg/quota/evaluator/core/pods.go create mode 100644 pkg/quota/evaluator/core/registry.go create mode 100644 pkg/quota/evaluator/core/replication_controllers.go create mode 100644 pkg/quota/evaluator/core/resource_quotas.go create mode 100644 pkg/quota/evaluator/core/secrets.go create mode 100644 pkg/quota/evaluator/core/services.go create mode 100644 pkg/quota/generic/evaluator.go create mode 100644 pkg/quota/generic/registry.go create mode 100644 pkg/quota/install/registry.go create mode 100644 pkg/quota/interfaces.go create mode 100644 pkg/quota/resources.go create mode 100644 pkg/quota/resources_test.go diff --git a/pkg/quota/evaluator/core/doc.go b/pkg/quota/evaluator/core/doc.go new file mode 100644 index 0000000000..3fdfaa773b --- /dev/null +++ b/pkg/quota/evaluator/core/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// core contains modules that interface with the core api group +package core diff --git a/pkg/quota/evaluator/core/persistent_volume_claims.go b/pkg/quota/evaluator/core/persistent_volume_claims.go new file mode 100644 index 0000000000..4edfffdd0d --- /dev/null +++ b/pkg/quota/evaluator/core/persistent_volume_claims.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewPersistentVolumeClaimEvaluator returns an evaluator that can evaluate persistent volume claims +func NewPersistentVolumeClaimEvaluator(kubeClient clientset.Interface) quota.Evaluator { + allResources := []api.ResourceName{api.ResourcePersistentVolumeClaims} + return &generic.GenericEvaluator{ + Name: "Evaluator.PersistentVolumeClaim", + InternalGroupKind: api.Kind("PersistentVolumeClaim"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + }, + MatchedResourceNames: allResources, + MatchesScopeFunc: generic.MatchesNoScopeFunc, + ConstraintsFunc: generic.ObjectCountConstraintsFunc(api.ResourcePersistentVolumeClaims), + UsageFunc: generic.ObjectCountUsageFunc(api.ResourcePersistentVolumeClaims), + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().PersistentVolumeClaims(namespace).List(options) + }, + } +} diff --git a/pkg/quota/evaluator/core/pods.go b/pkg/quota/evaluator/core/pods.go new file mode 100644 index 0000000000..eedc86927c --- /dev/null +++ b/pkg/quota/evaluator/core/pods.go @@ -0,0 +1,183 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "fmt" + "strings" + + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/kubelet/qos/util" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" + "k8s.io/kubernetes/pkg/util/sets" +) + +// NewPodEvaluator returns an evaluator that can evaluate pods +func NewPodEvaluator(kubeClient clientset.Interface) quota.Evaluator { + computeResources := []api.ResourceName{ + api.ResourceCPU, + api.ResourceMemory, + api.ResourceRequestsCPU, + api.ResourceRequestsMemory, + api.ResourceLimitsCPU, + api.ResourceLimitsMemory, + } + allResources := append(computeResources, api.ResourcePods) + return &generic.GenericEvaluator{ + Name: "Evaluator.Pod", + InternalGroupKind: api.Kind("Pod"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + admission.Update: computeResources, + }, + GetFuncByNamespace: func(namespace, name string) (runtime.Object, error) { + return kubeClient.Core().Pods(namespace).Get(name) + }, + ConstraintsFunc: PodConstraintsFunc, + MatchedResourceNames: allResources, + MatchesScopeFunc: PodMatchesScopeFunc, + UsageFunc: PodUsageFunc, + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().Pods(namespace).List(options) + }, + } +} + +// PodConstraintsFunc verifies that all required resources are present on the pod +func PodConstraintsFunc(required []api.ResourceName, object runtime.Object) error { + pod, ok := object.(*api.Pod) + if !ok { + return fmt.Errorf("Unexpected input object %v", object) + } + + // TODO: fix this when we have pod level cgroups + // since we do not yet pod level requests/limits, we need to ensure each + // container makes an explict request or limit for a quota tracked resource + requiredSet := quota.ToSet(required) + missingSet := sets.NewString() + for i := range pod.Spec.Containers { + requests := pod.Spec.Containers[i].Resources.Requests + limits := pod.Spec.Containers[i].Resources.Limits + containerUsage := podUsageHelper(requests, limits) + containerSet := quota.ToSet(quota.ResourceNames(containerUsage)) + if !containerSet.Equal(requiredSet) { + difference := requiredSet.Difference(containerSet) + missingSet.Insert(difference.List()...) + } + } + if len(missingSet) == 0 { + return nil + } + return fmt.Errorf("must specify %s", strings.Join(missingSet.List(), ",")) +} + +// podUsageHelper can summarize the pod quota usage based on requests and limits +func podUsageHelper(requests api.ResourceList, limits api.ResourceList) api.ResourceList { + result := api.ResourceList{} + result[api.ResourcePods] = resource.MustParse("1") + if request, found := requests[api.ResourceCPU]; found { + result[api.ResourceCPU] = request + result[api.ResourceRequestsCPU] = request + } + if limit, found := limits[api.ResourceCPU]; found { + result[api.ResourceLimitsCPU] = limit + } + if request, found := requests[api.ResourceMemory]; found { + result[api.ResourceMemory] = request + result[api.ResourceRequestsMemory] = request + } + if limit, found := limits[api.ResourceMemory]; found { + result[api.ResourceLimitsMemory] = limit + } + return result +} + +// PodUsageFunc knows how to measure usage associated with pods +func PodUsageFunc(object runtime.Object) api.ResourceList { + pod, ok := object.(*api.Pod) + if !ok { + return api.ResourceList{} + } + + // by convention, we do not quota pods that have reached an end-of-life state + if !QuotaPod(pod) { + return api.ResourceList{} + } + + // TODO: fix this when we have pod level cgroups + // when we have pod level cgroups, we can just read pod level requests/limits + requests := api.ResourceList{} + limits := api.ResourceList{} + for i := range pod.Spec.Containers { + requests = quota.Add(requests, pod.Spec.Containers[i].Resources.Requests) + limits = quota.Add(limits, pod.Spec.Containers[i].Resources.Limits) + } + + return podUsageHelper(requests, limits) +} + +// PodMatchesScopeFunc is a function that knows how to evaluate if a pod matches a scope +func PodMatchesScopeFunc(scope api.ResourceQuotaScope, object runtime.Object) bool { + pod, ok := object.(*api.Pod) + if !ok { + return false + } + switch scope { + case api.ResourceQuotaScopeTerminating: + return isTerminating(pod) + case api.ResourceQuotaScopeNotTerminating: + return !isTerminating(pod) + case api.ResourceQuotaScopeBestEffort: + return isBestEffort(pod) + case api.ResourceQuotaScopeNotBestEffort: + return !isBestEffort(pod) + } + return false +} + +func isBestEffort(pod *api.Pod) bool { + // TODO: when we have request/limits on a pod scope, we need to revisit this + for _, container := range pod.Spec.Containers { + qosPerResource := util.GetQoS(&container) + for _, qos := range qosPerResource { + if util.BestEffort == qos { + return true + } + } + } + return false +} + +func isTerminating(pod *api.Pod) bool { + if pod.Spec.ActiveDeadlineSeconds != nil && *pod.Spec.ActiveDeadlineSeconds >= int64(0) { + return true + } + return false +} + +// QuotaPod returns true if the pod is eligible to track against a quota +// if it's not in a terminal state according to its phase. +func QuotaPod(pod *api.Pod) bool { + // see GetPhase in kubelet.go for details on how it covers all restart policy conditions + // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet.go#L3001 + return !(api.PodFailed == pod.Status.Phase || api.PodSucceeded == pod.Status.Phase) +} diff --git a/pkg/quota/evaluator/core/registry.go b/pkg/quota/evaluator/core/registry.go new file mode 100644 index 0000000000..a9b3c44a97 --- /dev/null +++ b/pkg/quota/evaluator/core/registry.go @@ -0,0 +1,44 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/api/unversioned" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" +) + +// NewRegistry returns a registry that knows how to deal with core kubernetes resources +func NewRegistry(kubeClient clientset.Interface) quota.Registry { + pod := NewPodEvaluator(kubeClient) + service := NewServiceEvaluator(kubeClient) + replicationController := NewReplicationControllerEvaluator(kubeClient) + resourceQuota := NewResourceQuotaEvaluator(kubeClient) + secret := NewSecretEvaluator(kubeClient) + persistentVolumeClaim := NewPersistentVolumeClaimEvaluator(kubeClient) + return &generic.GenericRegistry{ + InternalEvaluators: map[unversioned.GroupKind]quota.Evaluator{ + pod.GroupKind(): pod, + service.GroupKind(): service, + replicationController.GroupKind(): replicationController, + secret.GroupKind(): secret, + resourceQuota.GroupKind(): resourceQuota, + persistentVolumeClaim.GroupKind(): persistentVolumeClaim, + }, + } +} diff --git a/pkg/quota/evaluator/core/replication_controllers.go b/pkg/quota/evaluator/core/replication_controllers.go new file mode 100644 index 0000000000..7d4b44337c --- /dev/null +++ b/pkg/quota/evaluator/core/replication_controllers.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewReplicationControllerEvaluator returns an evaluator that can evaluate replication controllers +func NewReplicationControllerEvaluator(kubeClient clientset.Interface) quota.Evaluator { + allResources := []api.ResourceName{api.ResourceReplicationControllers} + return &generic.GenericEvaluator{ + Name: "Evaluator.ReplicationController", + InternalGroupKind: api.Kind("ReplicationController"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + }, + MatchedResourceNames: allResources, + MatchesScopeFunc: generic.MatchesNoScopeFunc, + ConstraintsFunc: generic.ObjectCountConstraintsFunc(api.ResourceReplicationControllers), + UsageFunc: generic.ObjectCountUsageFunc(api.ResourceReplicationControllers), + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().ReplicationControllers(namespace).List(options) + }, + } +} diff --git a/pkg/quota/evaluator/core/resource_quotas.go b/pkg/quota/evaluator/core/resource_quotas.go new file mode 100644 index 0000000000..6d52e70e14 --- /dev/null +++ b/pkg/quota/evaluator/core/resource_quotas.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewResourceQuotaEvaluator returns an evaluator that can evaluate resource quotas +func NewResourceQuotaEvaluator(kubeClient clientset.Interface) quota.Evaluator { + allResources := []api.ResourceName{api.ResourceQuotas} + return &generic.GenericEvaluator{ + Name: "Evaluator.ResourceQuota", + InternalGroupKind: api.Kind("ResourceQuota"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + }, + MatchedResourceNames: allResources, + MatchesScopeFunc: generic.MatchesNoScopeFunc, + ConstraintsFunc: generic.ObjectCountConstraintsFunc(api.ResourceQuotas), + UsageFunc: generic.ObjectCountUsageFunc(api.ResourceQuotas), + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().ResourceQuotas(namespace).List(options) + }, + } +} diff --git a/pkg/quota/evaluator/core/secrets.go b/pkg/quota/evaluator/core/secrets.go new file mode 100644 index 0000000000..d3d79f293d --- /dev/null +++ b/pkg/quota/evaluator/core/secrets.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewSecretEvaluator returns an evaluator that can evaluate secrets +func NewSecretEvaluator(kubeClient clientset.Interface) quota.Evaluator { + allResources := []api.ResourceName{api.ResourceSecrets} + return &generic.GenericEvaluator{ + Name: "Evaluator.Secret", + InternalGroupKind: api.Kind("Secret"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + }, + MatchedResourceNames: allResources, + MatchesScopeFunc: generic.MatchesNoScopeFunc, + ConstraintsFunc: generic.ObjectCountConstraintsFunc(api.ResourceSecrets), + UsageFunc: generic.ObjectCountUsageFunc(api.ResourceSecrets), + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().Secrets(namespace).List(options) + }, + } +} diff --git a/pkg/quota/evaluator/core/services.go b/pkg/quota/evaluator/core/services.go new file mode 100644 index 0000000000..112dd9c575 --- /dev/null +++ b/pkg/quota/evaluator/core/services.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/generic" + "k8s.io/kubernetes/pkg/runtime" +) + +// NewServiceEvaluator returns an evaluator that can evaluate service quotas +func NewServiceEvaluator(kubeClient clientset.Interface) quota.Evaluator { + allResources := []api.ResourceName{api.ResourceServices} + return &generic.GenericEvaluator{ + Name: "Evaluator.Service", + InternalGroupKind: api.Kind("Service"), + InternalOperationResources: map[admission.Operation][]api.ResourceName{ + admission.Create: allResources, + }, + MatchedResourceNames: allResources, + MatchesScopeFunc: generic.MatchesNoScopeFunc, + ConstraintsFunc: generic.ObjectCountConstraintsFunc(api.ResourceServices), + UsageFunc: generic.ObjectCountUsageFunc(api.ResourceServices), + ListFuncByNamespace: func(namespace string, options api.ListOptions) (runtime.Object, error) { + return kubeClient.Core().Services(namespace).List(options) + }, + } +} diff --git a/pkg/quota/generic/evaluator.go b/pkg/quota/generic/evaluator.go new file mode 100644 index 0000000000..b53057b7c3 --- /dev/null +++ b/pkg/quota/generic/evaluator.go @@ -0,0 +1,199 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "fmt" + + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/meta" + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/runtime" +) + +// ConstraintsFunc takes a list of required resources that must match on the input item +type ConstraintsFunc func(required []api.ResourceName, item runtime.Object) error + +// GetFuncByNamespace knows how to get a resource with specified namespace and name +type GetFuncByNamespace func(namespace, name string) (runtime.Object, error) + +// ListFuncByNamespace knows how to list resources in a namespace +type ListFuncByNamespace func(namespace string, options api.ListOptions) (runtime.Object, error) + +// MatchesScopeFunc knows how to evaluate if an object matches a scope +type MatchesScopeFunc func(scope api.ResourceQuotaScope, object runtime.Object) bool + +// UsageFunc knows how to measure usage associated with an object +type UsageFunc func(object runtime.Object) api.ResourceList + +// MatchesNoScopeFunc returns false on all match checks +func MatchesNoScopeFunc(scope api.ResourceQuotaScope, object runtime.Object) bool { + return false +} + +// ObjectCountConstraintsFunc returns true if the specified resource name is in +// the required set of resource names +func ObjectCountConstraintsFunc(resourceName api.ResourceName) ConstraintsFunc { + return func(required []api.ResourceName, item runtime.Object) error { + if !quota.Contains(required, resourceName) { + return fmt.Errorf("missing %s", resourceName) + } + return nil + } +} + +// ObjectCountUsageFunc is useful if you are only counting your object +// It always returns 1 as the usage for the named resource +func ObjectCountUsageFunc(resourceName api.ResourceName) UsageFunc { + return func(object runtime.Object) api.ResourceList { + return api.ResourceList{ + resourceName: resource.MustParse("1"), + } + } +} + +// GenericEvaluator provides an implementation for quota.Evaluator +type GenericEvaluator struct { + // Name used for logging + Name string + // The GroupKind that this evaluator tracks + InternalGroupKind unversioned.GroupKind + // The set of resources that are pertinent to the mapped operation + InternalOperationResources map[admission.Operation][]api.ResourceName + // The set of resource names this evaluator matches + MatchedResourceNames []api.ResourceName + // A function that knows how to evaluate a matches scope request + MatchesScopeFunc MatchesScopeFunc + // A function that knows how to return usage for an object + UsageFunc UsageFunc + // A function that knows how to list resources by namespace + ListFuncByNamespace ListFuncByNamespace + // A function that knows how to get resource in a namespace + // This function must be specified if the evaluator needs to handle UPDATE + GetFuncByNamespace GetFuncByNamespace + // A function that checks required constraints are satisfied + ConstraintsFunc ConstraintsFunc +} + +// Ensure that GenericEvaluator implements quota.Evaluator +var _ quota.Evaluator = &GenericEvaluator{} + +// Constraints checks required constraints are satisfied on the input object +func (g *GenericEvaluator) Constraints(required []api.ResourceName, item runtime.Object) error { + return g.ConstraintsFunc(required, item) +} + +// Get returns the object by namespace and name +func (g *GenericEvaluator) Get(namespace, name string) (runtime.Object, error) { + return g.GetFuncByNamespace(namespace, name) +} + +// OperationResources returns the set of resources that could be updated for the +// specified operation for this kind. If empty, admission control will ignore +// quota processing for the operation. +func (g *GenericEvaluator) OperationResources(operation admission.Operation) []api.ResourceName { + return g.InternalOperationResources[operation] +} + +// GroupKind that this evaluator tracks +func (g *GenericEvaluator) GroupKind() unversioned.GroupKind { + return g.InternalGroupKind +} + +// MatchesResources is the list of resources that this evaluator matches +func (g *GenericEvaluator) MatchesResources() []api.ResourceName { + return g.MatchedResourceNames +} + +// Matches returns true if the evaluator matches the specified quota with the provided input item +func (g *GenericEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Object) bool { + if resourceQuota == nil { + return false + } + + // verify the quota matches on resource, by default its false + matchResource := false + for resourceName := range resourceQuota.Status.Hard { + if g.MatchesResource(resourceName) { + matchResource = true + } + } + // by default, no scopes matches all + matchScope := true + for _, scope := range resourceQuota.Spec.Scopes { + matchScope = matchScope && g.MatchesScope(scope, item) + } + return matchResource && matchScope +} + +// MatchesResource returns true if this evaluator can match on the specified resource +func (g *GenericEvaluator) MatchesResource(resourceName api.ResourceName) bool { + for _, matchedResourceName := range g.MatchedResourceNames { + if resourceName == matchedResourceName { + return true + } + } + return false +} + +// MatchesScope returns true if the input object matches the specified scope +func (g *GenericEvaluator) MatchesScope(scope api.ResourceQuotaScope, object runtime.Object) bool { + return g.MatchesScopeFunc(scope, object) +} + +// Usage returns the resource usage for the specified object +func (g *GenericEvaluator) Usage(object runtime.Object) api.ResourceList { + return g.UsageFunc(object) +} + +// UsageStats calculates latest observed usage stats for all objects +func (g *GenericEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) { + // default each tracked resource to zero + result := quota.UsageStats{Used: api.ResourceList{}} + for _, resourceName := range g.MatchedResourceNames { + result.Used[resourceName] = resource.MustParse("0") + } + list, err := g.ListFuncByNamespace(options.Namespace, api.ListOptions{}) + if err != nil { + return result, fmt.Errorf("%s: Failed to list %v: %v", g.Name, g.GroupKind, err) + } + _, err = meta.Accessor(list) + if err != nil { + return result, fmt.Errorf("%s: Unable to understand list result %#v", g.Name, list) + } + items, err := meta.ExtractList(list) + if err != nil { + return result, fmt.Errorf("%s: Unable to understand list result %#v (%v)", g.Name, list, err) + } + for _, item := range items { + // need to verify that the item matches the set of scopes + matchesScopes := true + for _, scope := range options.Scopes { + if !g.MatchesScope(scope, item) { + matchesScopes = false + } + } + // only count usage if there was a match + if matchesScopes { + result.Used = quota.Add(result.Used, g.Usage(item)) + } + } + return result, nil +} diff --git a/pkg/quota/generic/registry.go b/pkg/quota/generic/registry.go new file mode 100644 index 0000000000..0609d73cfa --- /dev/null +++ b/pkg/quota/generic/registry.go @@ -0,0 +1,36 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generic + +import ( + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/quota" +) + +// Ensure it implements the required interface +var _ quota.Registry = &GenericRegistry{} + +// GenericRegistry implements Registry +type GenericRegistry struct { + // internal evaluators by group kind + InternalEvaluators map[unversioned.GroupKind]quota.Evaluator +} + +// Evaluators returns the map of evaluators by groupKind +func (r *GenericRegistry) Evaluators() map[unversioned.GroupKind]quota.Evaluator { + return r.InternalEvaluators +} diff --git a/pkg/quota/install/registry.go b/pkg/quota/install/registry.go new file mode 100644 index 0000000000..109b57484b --- /dev/null +++ b/pkg/quota/install/registry.go @@ -0,0 +1,30 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" + "k8s.io/kubernetes/pkg/quota" + "k8s.io/kubernetes/pkg/quota/evaluator/core" +) + +// NewRegistry returns a registry that knows how to deal kubernetes resources +// across API groups +func NewRegistry(kubeClient clientset.Interface) quota.Registry { + // TODO: when quota supports resources in other api groups, we will need to merge + return core.NewRegistry(kubeClient) +} diff --git a/pkg/quota/interfaces.go b/pkg/quota/interfaces.go new file mode 100644 index 0000000000..da7e4a18fe --- /dev/null +++ b/pkg/quota/interfaces.go @@ -0,0 +1,66 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "k8s.io/kubernetes/pkg/admission" + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/unversioned" + "k8s.io/kubernetes/pkg/runtime" +) + +// UsageStatsOptions is an options structs that describes how stats should be calculated +type UsageStatsOptions struct { + // Namespace where stats should be calculate + Namespace string + // Scopes that must match counted objects + Scopes []api.ResourceQuotaScope +} + +// UsageStats is result of measuring observed resource use in the system +type UsageStats struct { + // Used maps resource to quantity used + Used api.ResourceList +} + +// Evaluator knows how to evaluate quota usage for a particular group kind +type Evaluator interface { + // Constraints ensures that each required resource is present on item + Constraints(required []api.ResourceName, item runtime.Object) error + // Get returns the object with specified namespace and name + Get(namespace, name string) (runtime.Object, error) + // GroupKind returns the groupKind that this object knows how to evaluate + GroupKind() unversioned.GroupKind + // MatchesResources is the list of resources that this evaluator matches + MatchesResources() []api.ResourceName + // Matches returns true if the specified quota matches the input item + Matches(resourceQuota *api.ResourceQuota, item runtime.Object) bool + // OperationResources returns the set of resources that could be updated for the + // specified operation for this kind. If empty, admission control will ignore + // quota processing for the operation. + OperationResources(operation admission.Operation) []api.ResourceName + // Usage returns the resource usage for the specified object + Usage(object runtime.Object) api.ResourceList + // UsageStats calculates latest observed usage stats for all objects + UsageStats(options UsageStatsOptions) (UsageStats, error) +} + +// Registry holds the list of evaluators associated to a particular group kind +type Registry interface { + // Evaluators returns the set Evaluator objects registered to a groupKind + Evaluators() map[unversioned.GroupKind]Evaluator +} diff --git a/pkg/quota/resources.go b/pkg/quota/resources.go new file mode 100644 index 0000000000..0cb03a84af --- /dev/null +++ b/pkg/quota/resources.go @@ -0,0 +1,159 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" + "k8s.io/kubernetes/pkg/util/sets" +) + +// Equals returns true if the two lists are equivalent +func Equals(a api.ResourceList, b api.ResourceList) bool { + for key, value1 := range a { + value2, found := b[key] + if !found { + return false + } + if value1.Cmp(value2) != 0 { + return false + } + } + for key, value1 := range b { + value2, found := a[key] + if !found { + return false + } + if value1.Cmp(value2) != 0 { + return false + } + } + return true +} + +// LessThanOrEqual returns true if a < b for each key in b +// If false, it returns the keys in a that exceeded b +func LessThanOrEqual(a api.ResourceList, b api.ResourceList) (bool, []api.ResourceName) { + result := true + resourceNames := []api.ResourceName{} + for key, value := range b { + if other, found := a[key]; found { + if other.Cmp(value) > 0 { + result = false + resourceNames = append(resourceNames, key) + } + } + } + return result, resourceNames +} + +// Add returns the result of a + b for each named resource +func Add(a api.ResourceList, b api.ResourceList) api.ResourceList { + result := api.ResourceList{} + for key, value := range a { + quantity := *value.Copy() + if other, found := b[key]; found { + quantity.Add(other) + } + result[key] = quantity + } + for key, value := range b { + if _, found := result[key]; !found { + quantity := *value.Copy() + result[key] = quantity + } + } + return result +} + +// Subtract returns the result of a - b for each named resource +func Subtract(a api.ResourceList, b api.ResourceList) api.ResourceList { + result := api.ResourceList{} + for key, value := range a { + quantity := *value.Copy() + if other, found := b[key]; found { + quantity.Sub(other) + } + result[key] = quantity + } + for key, value := range b { + if _, found := result[key]; !found { + quantity := *value.Copy() + quantity.Neg(value) + result[key] = quantity + } + } + return result +} + +// Mask returns a new resource list that only has the values with the specified names +func Mask(resources api.ResourceList, names []api.ResourceName) api.ResourceList { + nameSet := ToSet(names) + result := api.ResourceList{} + for key, value := range resources { + if nameSet.Has(string(key)) { + result[key] = *value.Copy() + } + } + return result +} + +// ResourceNames returns a list of all resource names in the ResourceList +func ResourceNames(resources api.ResourceList) []api.ResourceName { + result := []api.ResourceName{} + for resourceName := range resources { + result = append(result, resourceName) + } + return result +} + +// Contains returns true if the specified item is in the list of items +func Contains(items []api.ResourceName, item api.ResourceName) bool { + return ToSet(items).Has(string(item)) +} + +// Intersection returns the intersection of both list of resources +func Intersection(a []api.ResourceName, b []api.ResourceName) []api.ResourceName { + setA := ToSet(a) + setB := ToSet(b) + setC := setA.Intersection(setB) + result := []api.ResourceName{} + for _, resourceName := range setC.List() { + result = append(result, api.ResourceName(resourceName)) + } + return result +} + +// IsZero returns true if each key maps to the quantity value 0 +func IsZero(a api.ResourceList) bool { + zero := resource.MustParse("0") + for _, v := range a { + if v.Cmp(zero) != 0 { + return false + } + } + return true +} + +// ToSet takes a list of resource names and converts to a string set +func ToSet(resourceNames []api.ResourceName) sets.String { + result := sets.NewString() + for _, resourceName := range resourceNames { + result.Insert(string(resourceName)) + } + return result +} diff --git a/pkg/quota/resources_test.go b/pkg/quota/resources_test.go new file mode 100644 index 0000000000..5e2b3196aa --- /dev/null +++ b/pkg/quota/resources_test.go @@ -0,0 +1,223 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "testing" + + "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/resource" +) + +func TestEquals(t *testing.T) { + testCases := map[string]struct { + a api.ResourceList + b api.ResourceList + expected bool + }{ + "isEqual": { + a: api.ResourceList{}, + b: api.ResourceList{}, + expected: true, + }, + "isEqualWithKeys": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + b: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: true, + }, + "isNotEqualSameKeys": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("200m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + b: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: false, + }, + "isNotEqualDiffKeys": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + b: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + api.ResourcePods: resource.MustParse("1"), + }, + expected: false, + }, + } + for testName, testCase := range testCases { + if result := Equals(testCase.a, testCase.b); result != testCase.expected { + t.Errorf("%s expected: %v, actual: %v, a=%v, b=%v", testName, testCase.expected, result, testCase.a, testCase.b) + } + } +} + +func TestAdd(t *testing.T) { + testCases := map[string]struct { + a api.ResourceList + b api.ResourceList + expected api.ResourceList + }{ + "noKeys": { + a: api.ResourceList{}, + b: api.ResourceList{}, + expected: api.ResourceList{}, + }, + "toEmpty": { + a: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + b: api.ResourceList{}, + expected: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + }, + "matching": { + a: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + b: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + expected: api.ResourceList{api.ResourceCPU: resource.MustParse("200m")}, + }, + } + for testName, testCase := range testCases { + sum := Add(testCase.a, testCase.b) + if result := Equals(testCase.expected, sum); !result { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, sum) + } + } +} + +func TestSubtract(t *testing.T) { + testCases := map[string]struct { + a api.ResourceList + b api.ResourceList + expected api.ResourceList + }{ + "noKeys": { + a: api.ResourceList{}, + b: api.ResourceList{}, + expected: api.ResourceList{}, + }, + "value-empty": { + a: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + b: api.ResourceList{}, + expected: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + }, + "empty-value": { + a: api.ResourceList{}, + b: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + expected: api.ResourceList{api.ResourceCPU: resource.MustParse("-100m")}, + }, + "value-value": { + a: api.ResourceList{api.ResourceCPU: resource.MustParse("200m")}, + b: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + expected: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")}, + }, + } + for testName, testCase := range testCases { + sub := Subtract(testCase.a, testCase.b) + if result := Equals(testCase.expected, sub); !result { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, sub) + } + } +} + +func TestResourceNames(t *testing.T) { + testCases := map[string]struct { + a api.ResourceList + expected []api.ResourceName + }{ + "empty": { + a: api.ResourceList{}, + expected: []api.ResourceName{}, + }, + "values": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("100m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: []api.ResourceName{api.ResourceMemory, api.ResourceCPU}, + }, + } + for testName, testCase := range testCases { + actualSet := ToSet(ResourceNames(testCase.a)) + expectedSet := ToSet(testCase.expected) + if !actualSet.Equal(expectedSet) { + t.Errorf("%s expected: %v, actual: %v", testName, expectedSet, actualSet) + } + } +} + +func TestContains(t *testing.T) { + testCases := map[string]struct { + a []api.ResourceName + b api.ResourceName + expected bool + }{ + "does-not-contain": { + a: []api.ResourceName{api.ResourceMemory}, + b: api.ResourceCPU, + expected: false, + }, + "does-contain": { + a: []api.ResourceName{api.ResourceMemory, api.ResourceCPU}, + b: api.ResourceCPU, + expected: true, + }, + } + for testName, testCase := range testCases { + if actual := Contains(testCase.a, testCase.b); actual != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected, actual) + } + } +} + +func TestIsZero(t *testing.T) { + testCases := map[string]struct { + a api.ResourceList + expected bool + }{ + "empty": { + a: api.ResourceList{}, + expected: true, + }, + "zero": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("0"), + api.ResourceMemory: resource.MustParse("0"), + }, + expected: true, + }, + "non-zero": { + a: api.ResourceList{ + api.ResourceCPU: resource.MustParse("200m"), + api.ResourceMemory: resource.MustParse("1Gi"), + }, + expected: false, + }, + } + for testName, testCase := range testCases { + if result := IsZero(testCase.a); result != testCase.expected { + t.Errorf("%s expected: %v, actual: %v", testName, testCase.expected) + } + } +}