github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

resourceQuota support for extended resources

pull/6/head
lichuqiang 2018-01-30 09:29:22 +08:00
parent e87d8511b1
commit fde4f6f9b0
11 changed files with 154 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/apis/core/helper/BUILD
View File

@ -31,6 +31,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
],
)

18
pkg/apis/core/helper/helpers.go
View File

@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/kubernetes/pkg/apis/core"
)
@ -146,10 +147,21 @@ func IsStandardContainerResourceName(str string) bool {
return standardContainerResources.Has(str) || IsHugePageResourceName(core.ResourceName(str))
}
// IsExtendedResourceName returns true if the resource name is not in the
// default namespace.
// IsExtendedResourceName returns true if:
// 1. the resource name is not in the default namespace;
// 2. resource name does not have "requests." prefix,
// to avoid confusion with the convention in quota
// 3. it satisfies the rules in IsQualifiedName() after converted into quota resource name
func IsExtendedResourceName(name core.ResourceName) bool {
return !IsDefaultNamespaceResource(name)
if IsDefaultNamespaceResource(name) || strings.HasPrefix(string(name), core.DefaultResourceRequestsPrefix) {
return false
}
// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
nameForQuota := fmt.Sprintf("%s%s", core.DefaultResourceRequestsPrefix, string(name))
if errs := validation.IsQualifiedName(string(nameForQuota)); len(errs) != 0 {
return false
}
return true
}
// IsDefaultNamespaceResource returns true if the resource name is in the

2
pkg/apis/core/types.go
View File

@ -4159,6 +4159,8 @@ const (
// HugePages request, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024)
// As burst is not supported for HugePages, we would only quota its request, and ignore the limit.
ResourceRequestsHugePagesPrefix = "requests.hugepages-"
// Default resource requests prefix
DefaultResourceRequestsPrefix = "requests."
)
// A ResourceQuotaScope defines a filter that must match each object tracked by a quota

1
pkg/apis/core/v1/helper/BUILD
View File

@ -31,6 +31,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/selection:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
],
)

18
pkg/apis/core/v1/helper/helpers.go
View File

@ -26,13 +26,25 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/kubernetes/pkg/apis/core/helper"
)
// IsExtendedResourceName returns true if the resource name is not in the
// default namespace.
// IsExtendedResourceName returns true if:
// 1. the resource name is not in the default namespace;
// 2. resource name does not have "requests." prefix,
// to avoid confusion with the convention in quota
// 3. it satisfies the rules in IsQualifiedName() after converted into quota resource name
func IsExtendedResourceName(name v1.ResourceName) bool {
return !IsDefaultNamespaceResource(name)
if IsDefaultNamespaceResource(name) || strings.HasPrefix(string(name), v1.DefaultResourceRequestsPrefix) {
return false
}
// Ensure it satisfies the rules in IsQualifiedName() after converted into quota resource name
nameForQuota := fmt.Sprintf("%s%s", v1.DefaultResourceRequestsPrefix, string(name))
if errs := validation.IsQualifiedName(string(nameForQuota)); len(errs) != 0 {
return false
}
return true
}
// IsDefaultNamespaceResource returns true if the resource name is in the

4
pkg/apis/core/v1/validation/validation.go
View File

@ -75,6 +75,10 @@ func validateContainerResourceName(value string, fldPath *field.Path) field.Erro
if !helper.IsStandardContainerResourceName(value) {
return append(allErrs, field.Invalid(fldPath, value, "must be a standard resource for containers"))
}
} else if !v1helper.IsDefaultNamespaceResource(v1.ResourceName(value)) {
if !v1helper.IsExtendedResourceName(v1.ResourceName(value)) {
return append(allErrs, field.Invalid(fldPath, value, "doesn't follow extended resource name standard"))
}
}
return allErrs
}

7
pkg/apis/core/validation/validation.go
View File

@ -3998,6 +3998,10 @@ func validateContainerResourceName(value string, fldPath *field.Path) field.Erro
if !helper.IsStandardContainerResourceName(value) {
return append(allErrs, field.Invalid(fldPath, value, "must be a standard resource for containers"))
}
} else if !helper.IsDefaultNamespaceResource(core.ResourceName(value)) {
if !helper.IsExtendedResourceName(core.ResourceName(value)) {
return append(allErrs, field.Invalid(fldPath, value, "doesn't follow extended resource name standard"))
}
}
return allErrs
}
@ -4179,7 +4183,8 @@ func ValidateLimitRange(limitRange *core.LimitRange) field.ErrorList {
}
}
// for GPU and hugepages, the default value and defaultRequest value must match if both are specified
// for GPU, hugepages and other resources that are not allowed to overcommit,
// the default value and defaultRequest value must match if both are specified
if !helper.IsOvercommitAllowed(core.ResourceName(k)) && defaultQuantityFound && defaultRequestQuantityFound && defaultQuantity.Cmp(defaultRequestQuantity) != 0 {
allErrs = append(allErrs, field.Invalid(idxPath.Child("defaultRequest").Key(string(k)), defaultRequestQuantity, fmt.Sprintf("default value %s must equal to defaultRequest value %s in %s", defaultQuantity.String(), defaultRequestQuantity.String(), k)))
}

24
pkg/controller/resourcequota/resource_quota_controller_test.go
View File

@ -337,6 +337,26 @@ func TestAddQuota(t *testing.T) {
},
},
},
{
name: "status, no usage(to validate it works for extended resources)",
expectedPriority: true,
quota: &v1.ResourceQuota{
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "rq",
},
Spec: v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
"requests.example/foobars.example.com": resource.MustParse("4"),
},
},
Status: v1.ResourceQuotaStatus{
Hard: v1.ResourceList{
"requests.example/foobars.example.com": resource.MustParse("4"),
},
},
},
},
{
name: "status, mismatch",
expectedPriority: true,
@ -370,12 +390,12 @@ func TestAddQuota(t *testing.T) {
},
Spec: v1.ResourceQuotaSpec{
Hard: v1.ResourceList{
"count/foobars.example.com": resource.MustParse("4"),
"foobars.example.com": resource.MustParse("4"),
},
},
Status: v1.ResourceQuotaStatus{
Hard: v1.ResourceList{
"count/foobars.example.com": resource.MustParse("4"),
"foobars.example.com": resource.MustParse("4"),
},
},
},

30
pkg/quota/evaluator/core/pods.go
View File

@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apiserver/pkg/admission"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/helper"
"k8s.io/kubernetes/pkg/apis/core/helper/qos"
k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1"
"k8s.io/kubernetes/pkg/apis/core/validation"
@ -71,17 +72,20 @@ var requestedResourcePrefixes = []string{
api.ResourceHugePagesPrefix,
}
const (
requestsPrefix = "requests."
limitsPrefix = "limits."
)
// maskResourceWithPrefix mask resource with certain prefix
// e.g. hugepages-XXX -> requests.hugepages-XXX
func maskResourceWithPrefix(resource api.ResourceName, prefix string) api.ResourceName {
return api.ResourceName(fmt.Sprintf("%s%s", prefix, string(resource)))
}
// isExtendedResourceNameForQuota returns true if the extended resource name
// has the quota related resource prefix.
func isExtendedResourceNameForQuota(name api.ResourceName) bool {
// As overcommit is not supported by extended resources for now,
// only quota objects in format of "requests.resourceName" is allowed.
return !helper.IsDefaultNamespaceResource(name) && strings.HasPrefix(string(name), api.DefaultResourceRequestsPrefix)
}
// NOTE: it was a mistake, but if a quota tracks cpu or memory related resources,
// the incoming pod is required to have those values set. we should not repeat
// this mistake for other future resources (gpus, ephemeral-storage,etc).
@ -183,9 +187,14 @@ func (p *podEvaluator) Matches(resourceQuota *api.ResourceQuota, item runtime.Ob
func (p *podEvaluator) MatchingResources(input []api.ResourceName) []api.ResourceName {
result := quota.Intersection(input, podResources)
for _, resource := range input {
// for resources with certain prefix, e.g. hugepages
if quota.ContainsPrefix(podResourcePrefixes, resource) {
result = append(result, resource)
}
// for extended resources
if isExtendedResourceNameForQuota(resource) {
result = append(result, resource)
}
}
return result
@ -244,14 +253,15 @@ func podComputeUsageHelper(requests api.ResourceList, limits api.ResourceList) a
result[api.ResourceLimitsEphemeralStorage] = limit
}
for resource, request := range requests {
// for resources with certain prefix, e.g. hugepages
if quota.ContainsPrefix(requestedResourcePrefixes, resource) {
result[resource] = request
result[maskResourceWithPrefix(resource, requestsPrefix)] = request
result[maskResourceWithPrefix(resource, api.DefaultResourceRequestsPrefix)] = request
}
}
for resource, limit := range limits {
if quota.ContainsPrefix(requestedResourcePrefixes, resource) {
result[maskResourceWithPrefix(resource, limitsPrefix)] = limit
// for extended resources
if helper.IsExtendedResourceName(resource) {
// only quota objects in format of "requests.resourceName" is allowed for extended resource.
result[maskResourceWithPrefix(resource, api.DefaultResourceRequestsPrefix)] = request
}
}

89
pkg/quota/evaluator/core/pods_test.go
View File

@ -192,6 +192,23 @@ func TestPodEvaluatorUsage(t *testing.T) {
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"init container extended resources": {
pod: &api.Pod{
Spec: api.PodSpec{
InitContainers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceName("example.com/dongle"): resource.MustParse("3")},
Limits: api.ResourceList{api.ResourceName("example.com/dongle"): resource.MustParse("3")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceName("requests.example.com/dongle"): resource.MustParse("3"),
api.ResourcePods: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"container CPU": {
pod: &api.Pod{
Spec: api.PodSpec{
@ -266,6 +283,23 @@ func TestPodEvaluatorUsage(t *testing.T) {
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"container extended resources": {
pod: &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{api.ResourceName("example.com/dongle"): resource.MustParse("3")},
Limits: api.ResourceList{api.ResourceName("example.com/dongle"): resource.MustParse("3")},
},
}},
},
},
usage: api.ResourceList{
api.ResourceName("requests.example.com/dongle"): resource.MustParse("3"),
api.ResourcePods: resource.MustParse("1"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},
"init container maximums override sum of containers": {
pod: &api.Pod{
Spec: api.PodSpec{
@ -273,24 +307,28 @@ func TestPodEvaluatorUsage(t *testing.T) {
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceName("example.com/dongle"): resource.MustParse("4"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("8"),
api.ResourceMemory: resource.MustParse("200M"),
api.ResourceCPU: resource.MustParse("8"),
api.ResourceMemory: resource.MustParse("200M"),
api.ResourceName("example.com/dongle"): resource.MustParse("4"),
},
},
},
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceName("example.com/dongle"): resource.MustParse("2"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceName("example.com/dongle"): resource.MustParse("2"),
},
},
},
@ -299,24 +337,28 @@ func TestPodEvaluatorUsage(t *testing.T) {
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceCPU: resource.MustParse("1"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceName("example.com/dongle"): resource.MustParse("1"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceName("example.com/dongle"): resource.MustParse("1"),
},
},
},
{
Resources: api.ResourceRequirements{
Requests: api.ResourceList{
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("25M"),
api.ResourceCPU: resource.MustParse("2"),
api.ResourceMemory: resource.MustParse("25M"),
api.ResourceName("example.com/dongle"): resource.MustParse("2"),
},
Limits: api.ResourceList{
api.ResourceCPU: resource.MustParse("5"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceCPU: resource.MustParse("5"),
api.ResourceMemory: resource.MustParse("50M"),
api.ResourceName("example.com/dongle"): resource.MustParse("2"),
},
},
},
@ -324,13 +366,14 @@ func TestPodEvaluatorUsage(t *testing.T) {
},
},
usage: api.ResourceList{
api.ResourceRequestsCPU: resource.MustParse("4"),
api.ResourceRequestsMemory: resource.MustParse("100M"),
api.ResourceLimitsCPU: resource.MustParse("8"),
api.ResourceLimitsMemory: resource.MustParse("200M"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceRequestsCPU: resource.MustParse("4"),
api.ResourceRequestsMemory: resource.MustParse("100M"),
api.ResourceLimitsCPU: resource.MustParse("8"),
api.ResourceLimitsMemory: resource.MustParse("200M"),
api.ResourcePods: resource.MustParse("1"),
api.ResourceCPU: resource.MustParse("4"),
api.ResourceMemory: resource.MustParse("100M"),
api.ResourceName("requests.example.com/dongle"): resource.MustParse("4"),
generic.ObjectCountQuotaResourceNameFor(schema.GroupResource{Resource: "pods"}): resource.MustParse("1"),
},
},

2
staging/src/k8s.io/api/core/v1/types.go
View File

@ -4684,6 +4684,8 @@ const (
// HugePages request, in bytes. (500Gi = 500GiB = 500 * 1024 * 1024 * 1024)
// As burst is not supported for HugePages, we would only quota its request, and ignore the limit.
ResourceRequestsHugePagesPrefix = "requests.hugepages-"
// Default resource requests prefix
DefaultResourceRequestsPrefix = "requests."
)
// A ResourceQuotaScope defines a filter that must match each object tracked by a quota