From deae071d78e164ae4a7e74a5d6e029125764315b Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Thu, 10 Jan 2019 13:01:23 -0500 Subject: [PATCH] Graduate HugePages feature to GA --- pkg/apis/core/validation/validation.go | 23 ++++------ pkg/apis/core/validation/validation_test.go | 22 +--------- pkg/features/kube_features.go | 3 +- pkg/kubelet/cadvisor/BUILD | 5 --- pkg/kubelet/cadvisor/util.go | 14 +++---- pkg/kubelet/cadvisor/util_test.go | 27 ------------ pkg/kubelet/cm/cgroup_manager_linux.go | 42 +++++++++---------- .../cm/node_container_manager_linux.go | 6 +-- pkg/kubelet/cm/qos_container_manager_linux.go | 10 ++--- 9 files changed, 41 insertions(+), 111 deletions(-) diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index 35ea4b9f0b..cd04b878f6 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -415,9 +415,6 @@ func validateVolumeSource(source *core.VolumeSource, fldPath *field.Path, volNam if source.EmptyDir.SizeLimit != nil && source.EmptyDir.SizeLimit.Cmp(resource.Quantity{}) < 0 { allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("sizeLimit"), "SizeLimit field must be a valid resource quantity")) } - if !utilfeature.DefaultFeatureGate.Enabled(features.HugePages) && source.EmptyDir.Medium == core.StorageMediumHugePages { - allErrs = append(allErrs, field.Forbidden(fldPath.Child("emptyDir").Child("medium"), "HugePages medium is disabled by feature-gate for EmptyDir volumes")) - } } if source.HostPath != nil { if numVolumes > 0 { @@ -2925,19 +2922,17 @@ func ValidatePod(pod *core.Pod) field.ErrorList { allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.Containers, specPath.Child("containers"))...) allErrs = append(allErrs, validateContainersOnlyForPod(pod.Spec.InitContainers, specPath.Child("initContainers"))...) - if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) { - hugePageResources := sets.NewString() - for i := range pod.Spec.Containers { - resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i]) - for resourceStr := range resourceSet { - if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) { - hugePageResources.Insert(resourceStr) - } + hugePageResources := sets.NewString() + for i := range pod.Spec.Containers { + resourceSet := toContainerResourcesSet(&pod.Spec.Containers[i]) + for resourceStr := range resourceSet { + if v1helper.IsHugePageResourceName(v1.ResourceName(resourceStr)) { + hugePageResources.Insert(resourceStr) } } - if len(hugePageResources) > 1 { - allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec")) - } + } + if len(hugePageResources) > 1 { + allErrs = append(allErrs, field.Invalid(specPath, hugePageResources, "must use a single hugepage size in a pod spec")) } return allErrs diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 531bf2ff2d..43b992ea04 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -3659,24 +3659,17 @@ func TestValidateVolumes(t *testing.T) { t.Errorf("expected error type %v, got %v", field.ErrorTypeDuplicate, errs[0].Type) } - // Validate HugePages medium type for EmptyDir when HugePages feature is enabled/disabled + // Validate HugePages medium type for EmptyDir hugePagesCase := core.VolumeSource{EmptyDir: &core.EmptyDirVolumeSource{Medium: core.StorageMediumHugePages}} // Enable HugePages - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "working"); len(errs) != 0 { t.Errorf("Unexpected error when HugePages feature is enabled.") } - // Disable feature HugePages - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)() - if errs := validateVolumeSource(&hugePagesCase, field.NewPath("field").Index(0), "failing"); len(errs) == 0 { - t.Errorf("Expected error when HugePages feature is disabled got nothing.") - } - } -func TestAlphaHugePagesIsolation(t *testing.T) { +func TestHugePagesIsolation(t *testing.T) { successCases := []core.Pod{ { // Basic fields. ObjectMeta: metav1.ObjectMeta{Name: "123", Namespace: "ns"}, @@ -3774,8 +3767,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) { }, }, } - // Enable feature HugePages - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() for i := range successCases { pod := &successCases[i] if errs := ValidatePod(pod); len(errs) != 0 { @@ -3788,15 +3779,6 @@ func TestAlphaHugePagesIsolation(t *testing.T) { t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name) } } - // Disable feature HugePages - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)() - // Disable feature HugePages and ensure all success cases fail - for i := range successCases { - pod := &successCases[i] - if errs := ValidatePod(pod); len(errs) == 0 { - t.Errorf("Expected error for case[%d], pod: %v", i, pod.Name) - } - } } func TestPVCVolumeMode(t *testing.T) { diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a3b523a8d7..c884dc8f23 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -163,6 +163,7 @@ const ( // owner: @derekwaynecarr // beta: v1.10 + // GA: v1.14 // // Enable pods to consume pre-allocated huge pages of varying page sizes HugePages utilfeature.Feature = "HugePages" @@ -424,7 +425,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS RotateKubeletClientCertificate: {Default: true, PreRelease: utilfeature.Beta}, PersistentLocalVolumes: {Default: true, PreRelease: utilfeature.Beta}, LocalStorageCapacityIsolation: {Default: true, PreRelease: utilfeature.Beta}, - HugePages: {Default: true, PreRelease: utilfeature.Beta}, + HugePages: {Default: true, PreRelease: utilfeature.GA, LockToDefault: true}, // remove in 1.16 Sysctls: {Default: true, PreRelease: utilfeature.Beta}, DebugContainers: {Default: false, PreRelease: utilfeature.Alpha}, PodShareProcessNamespace: {Default: true, PreRelease: utilfeature.Beta}, diff --git a/pkg/kubelet/cadvisor/BUILD b/pkg/kubelet/cadvisor/BUILD index cf30e5aa3f..12bb3b42c6 100644 --- a/pkg/kubelet/cadvisor/BUILD +++ b/pkg/kubelet/cadvisor/BUILD @@ -21,11 +21,9 @@ go_library( importpath = "k8s.io/kubernetes/pkg/kubelet/cadvisor", deps = [ "//pkg/apis/core/v1/helper:go_default_library", - "//pkg/features:go_default_library", "//pkg/kubelet/types:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/github.com/google/cadvisor/events:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library", @@ -51,11 +49,8 @@ go_test( embed = [":go_default_library"], deps = select({ "@io_bazel_rules_go//go/platform:linux": [ - "//pkg/features:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", - "//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library", "//vendor/github.com/google/cadvisor/container/crio:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", diff --git a/pkg/kubelet/cadvisor/util.go b/pkg/kubelet/cadvisor/util.go index 9a7e874643..49e35fb5d0 100644 --- a/pkg/kubelet/cadvisor/util.go +++ b/pkg/kubelet/cadvisor/util.go @@ -23,9 +23,7 @@ import ( cadvisorapi2 "github.com/google/cadvisor/info/v2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - utilfeature "k8s.io/apiserver/pkg/util/feature" v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" - "k8s.io/kubernetes/pkg/features" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -46,13 +44,11 @@ func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList { } // if huge pages are enabled, we report them as a schedulable resource on the node - if utilfeature.DefaultFeatureGate.Enabled(features.HugePages) { - for _, hugepagesInfo := range info.HugePages { - pageSizeBytes := int64(hugepagesInfo.PageSize * 1024) - hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages) - pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI) - c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI) - } + for _, hugepagesInfo := range info.HugePages { + pageSizeBytes := int64(hugepagesInfo.PageSize * 1024) + hugePagesBytes := pageSizeBytes * int64(hugepagesInfo.NumPages) + pageSizeQuantity := resource.NewQuantity(pageSizeBytes, resource.BinarySI) + c[v1helper.HugePageResourceName(*pageSizeQuantity)] = *resource.NewQuantity(hugePagesBytes, resource.BinarySI) } return c diff --git a/pkg/kubelet/cadvisor/util_test.go b/pkg/kubelet/cadvisor/util_test.go index bb65b9acea..9d0e7895e3 100644 --- a/pkg/kubelet/cadvisor/util_test.go +++ b/pkg/kubelet/cadvisor/util_test.go @@ -27,9 +27,6 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - utilfeature "k8s.io/apiserver/pkg/util/feature" - utilfeaturetesting "k8s.io/apiserver/pkg/util/feature/testing" - "k8s.io/kubernetes/pkg/features" ) func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) { @@ -49,36 +46,12 @@ func TestCapacityFromMachineInfoWithHugePagesEnable(t *testing.T) { v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI), "hugepages-5Ki": *resource.NewQuantity(int64(51200), resource.BinarySI), } - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, true)() actual := CapacityFromMachineInfo(machineInfo) if !reflect.DeepEqual(actual, expected) { t.Errorf("when set hugepages true, got resource list %v, want %v", actual, expected) } } -func TestCapacityFromMachineInfoWithHugePagesDisable(t *testing.T) { - machineInfo := &info.MachineInfo{ - NumCores: 2, - MemoryCapacity: 2048, - HugePages: []info.HugePagesInfo{ - { - PageSize: 5, - NumPages: 10, - }, - }, - } - - expected := v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(int64(2000), resource.DecimalSI), - v1.ResourceMemory: *resource.NewQuantity(int64(2048), resource.BinarySI), - } - defer utilfeaturetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.HugePages, false)() - actual := CapacityFromMachineInfo(machineInfo) - if !reflect.DeepEqual(actual, expected) { - t.Errorf("when set hugepages false, got resource list %v, want %v", actual, expected) - } -} - func TestCrioSocket(t *testing.T) { assert.EqualValues(t, CrioSocket, crio.CrioSocket, "CrioSocket in this package must equal the one in github.com/google/cadvisor/container/crio/client.go") } diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go index 3e8b60d9d1..2eb2a167f7 100644 --- a/pkg/kubelet/cm/cgroup_manager_linux.go +++ b/pkg/kubelet/cm/cgroup_manager_linux.go @@ -327,9 +327,7 @@ func getSupportedSubsystems() map[subsystem]bool { &cgroupfs.CpuGroup{}: true, } // not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity. - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { - supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false - } + supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) { supportedSubsystems[&cgroupfs.PidsGroup{}] = true } @@ -385,27 +383,25 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont } } // if huge pages are enabled, we set them in libcontainer - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { - // for each page size enumerated, set that value - pageSizes := sets.NewString() - for pageSize, limit := range resourceConfig.HugePageLimit { - sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList) - resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{ - Pagesize: sizeString, - Limit: uint64(limit), - }) - pageSizes.Insert(sizeString) - } - // for each page size omitted, limit to 0 - for _, pageSize := range cgroupfs.HugePageSizes { - if pageSizes.Has(pageSize) { - continue - } - resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{ - Pagesize: pageSize, - Limit: uint64(0), - }) + // for each page size enumerated, set that value + pageSizes := sets.NewString() + for pageSize, limit := range resourceConfig.HugePageLimit { + sizeString := units.CustomSize("%g%s", float64(pageSize), 1024.0, hugePageSizeList) + resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{ + Pagesize: sizeString, + Limit: uint64(limit), + }) + pageSizes.Insert(sizeString) + } + // for each page size omitted, limit to 0 + for _, pageSize := range cgroupfs.HugePageSizes { + if pageSizes.Has(pageSize) { + continue } + resources.HugetlbLimit = append(resources.HugetlbLimit, &libcontainerconfigs.HugepageLimit{ + Pagesize: pageSize, + Limit: uint64(0), + }) } return resources } diff --git a/pkg/kubelet/cm/node_container_manager_linux.go b/pkg/kubelet/cm/node_container_manager_linux.go index 8d7f16f1d1..0820e78381 100644 --- a/pkg/kubelet/cm/node_container_manager_linux.go +++ b/pkg/kubelet/cm/node_container_manager_linux.go @@ -26,9 +26,7 @@ import ( "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" - utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" - kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) @@ -159,9 +157,7 @@ func getCgroupConfig(rl v1.ResourceList) *ResourceConfig { val := MilliCPUToShares(q.MilliValue()) rc.CpuShares = &val } - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { - rc.HugePageLimit = HugePageLimits(rl) - } + rc.HugePageLimit = HugePageLimits(rl) return &rc } diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go index cebc9756e4..7f956a776a 100644 --- a/pkg/kubelet/cm/qos_container_manager_linux.go +++ b/pkg/kubelet/cm/qos_container_manager_linux.go @@ -108,9 +108,7 @@ func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceLis } // for each enumerated huge page size, the qos tiers are unbounded - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { - m.setHugePagesUnbounded(containerConfig) - } + m.setHugePagesUnbounded(containerConfig) // check if it exists if !cm.Exists(containerName) { @@ -290,10 +288,8 @@ func (m *qosContainerManagerImpl) UpdateCgroups() error { } // update the qos level cgroup settings for huge pages (ensure they remain unbounded) - if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { - if err := m.setHugePagesConfig(qosConfigs); err != nil { - return err - } + if err := m.setHugePagesConfig(qosConfigs); err != nil { + return err } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.QOSReserved) {