Add e2e test for quota-based eviction.

Positive test is skipped if quotas not available.

test/e2e_node/BUILD

@@ -19,6 +19,8 @@ go_library(
         "node_problem_detector_linux.go",
         "resource_collector.go",
         "util.go",
+        "util_xfs_linux.go",
+        "util_xfs_unsupported.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e_node",
     visibility = ["//visibility:public"],
@@ -41,7 +43,9 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
+        "//staging/src/k8s.io/component-base/featuregate:go_default_library",
         "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library",
         "//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
         "//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
@@ -62,6 +66,7 @@ go_library(
         "//vendor/k8s.io/klog:go_default_library",
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/mount:go_default_library",
             "//pkg/util/procfs:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
             "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
@@ -105,6 +110,7 @@ go_test(
         "node_perf_test.go",
         "pids_test.go",
         "pods_container_manager_test.go",
+        "quota_lsci_test.go",
         "resource_metrics_test.go",
         "resource_usage_test.go",
         "restart_test.go",
@@ -138,6 +144,8 @@ go_test(
         "//pkg/kubelet/metrics:go_default_library",
         "//pkg/kubelet/types:go_default_library",
         "//pkg/security/apparmor:go_default_library",
+        "//pkg/util/mount:go_default_library",
+        "//pkg/volume/util/quota:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/api/scheduling/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",

test/e2e_node/image_list.go

@@ -49,6 +49,7 @@ var NodeImageWhiteList = sets.NewString(
 	busyboxImage,
 	"k8s.gcr.io/busybox@sha256:4bdd623e848417d96127e16037743f0cd8b528c026e9175e22a84f639eca58ff",
 	imageutils.GetE2EImage(imageutils.Nginx),
+	imageutils.GetE2EImage(imageutils.Perl),
 	imageutils.GetE2EImage(imageutils.ServeHostname),
 	imageutils.GetE2EImage(imageutils.Netexec),
 	imageutils.GetE2EImage(imageutils.Nonewprivs),

test/e2e_node/quota_lsci_test.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/pkg/features"
+	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume/util/quota"
+	"k8s.io/kubernetes/test/e2e/framework"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	LSCIQuotaFeature = features.LocalStorageCapacityIsolationFSQuotaMonitoring
+)
+
+func runOneQuotaTest(f *framework.Framework, quotasRequested bool) {
+	evictionTestTimeout := 10 * time.Minute
+	sizeLimit := resource.MustParse("100Mi")
+	useOverLimit := 101 /* Mb */
+	useUnderLimit := 99 /* Mb */
+	// TODO: remove hardcoded kubelet volume directory path
+	// framework.TestContext.KubeVolumeDir is currently not populated for node e2e
+	// As for why we do this: see comment below at isXfs.
+	if isXfs("/var/lib/kubelet") {
+		useUnderLimit = 50 /* Mb */
+	}
+	priority := 0
+	if quotasRequested {
+		priority = 1
+	}
+	Context(fmt.Sprintf(testContextFmt, fmt.Sprintf("use quotas for LSCI monitoring (quotas enabled: %v)", quotasRequested)), func() {
+		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
+			defer withFeatureGate(LSCIQuotaFeature, quotasRequested)()
+			// TODO: remove hardcoded kubelet volume directory path
+			// framework.TestContext.KubeVolumeDir is currently not populated for node e2e
+			if quotasRequested && !supportsQuotas("/var/lib/kubelet") {
+				// No point in running this as a positive test if quotas are not
+				// enabled on the underlying filesystem.
+				framework.Skipf("Cannot run LocalStorageCapacityIsolationQuotaMonitoring on filesystem without project quota enabled")
+			}
+			// setting a threshold to 0% disables; non-empty map overrides default value (necessary due to omitempty)
+			initialConfig.EvictionHard = map[string]string{"memory.available": "0%"}
+			initialConfig.FeatureGates[string(LSCIQuotaFeature)] = quotasRequested
+		})
+		runEvictionTest(f, evictionTestTimeout, noPressure, noStarvedResource, logDiskMetrics, []podEvictSpec{
+			{
+				evictionPriority: priority, // This pod should be evicted because of emptyDir violation only if quotas are enabled
+				pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-over-sizelimit-quotas-%v", quotasRequested), useOverLimit, &v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit},
+				}, v1.ResourceRequirements{}),
+			},
+			{
+				evictionPriority: 0, // This pod should not be evicted because it uses less than its limit (test for quotas)
+				pod: diskConcealingPod(fmt.Sprintf("emptydir-concealed-disk-under-sizelimit-quotas-%v", quotasRequested), useUnderLimit, &v1.VolumeSource{
+					EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit},
+				}, v1.ResourceRequirements{}),
+			},
+		})
+	})
+}
+
+// LocalStorageCapacityIsolationQuotaMonitoring tests that quotas are
+// used for monitoring rather than du.  The mechanism is to create a
+// pod that creates a file, deletes it, and writes data to it.  If
+// quotas are used to monitor, it will detect this deleted-but-in-use
+// file; if du is used to monitor, it will not detect this.
+var _ = framework.KubeDescribe("LocalStorageCapacityIsolationQuotaMonitoring [Slow] [Serial] [Disruptive] [Feature:LocalStorageCapacityIsolationQuota][NodeFeature:LSCIQuotaMonitoring]", func() {
+	f := framework.NewDefaultFramework("localstorage-quota-monitoring-test")
+	runOneQuotaTest(f, true)
+	runOneQuotaTest(f, false)
+})
+
+const (
+	writeConcealedPodCommand = `
+my $file = "%s.bin";
+open OUT, ">$file" || die "Cannot open $file: $!\n";
+unlink "$file" || die "Cannot unlink $file: $!\n";
+my $a = "a";
+foreach (1..20) { $a = "$a$a"; }
+foreach (1..%d) { syswrite(OUT, $a); }
+sleep 999999;`
+)
+
+// This is needed for testing eviction of pods using disk space in concealed files; the shell has no convenient
+// way of performing I/O to a concealed file, and the busybox image doesn't contain Perl.
+func diskConcealingPod(name string, diskConsumedMB int, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod {
+	path := ""
+	volumeMounts := []v1.VolumeMount{}
+	volumes := []v1.Volume{}
+	if volumeSource != nil {
+		path = volumeMountPath
+		volumeMounts = []v1.VolumeMount{{MountPath: volumeMountPath, Name: volumeName}}
+		volumes = []v1.Volume{{Name: volumeName, VolumeSource: *volumeSource}}
+	}
+	return &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-pod", name)},
+		Spec: v1.PodSpec{
+			RestartPolicy: v1.RestartPolicyNever,
+			Containers: []v1.Container{
+				{
+					Image: imageutils.GetE2EImage(imageutils.Perl),
+					Name:  fmt.Sprintf("%s-container", name),
+					Command: []string{
+						"perl",
+						"-e",
+						fmt.Sprintf(writeConcealedPodCommand, filepath.Join(path, "file"), diskConsumedMB),
+					},
+					Resources:    resources,
+					VolumeMounts: volumeMounts,
+				},
+			},
+			Volumes: volumes,
+		},
+	}
+}
+
+// Don't bother returning an error; if something goes wrong,
+// simply treat it as "no".
+func supportsQuotas(dir string) bool {
+	supportsQuota, err := quota.SupportsQuotas(mount.New(""), dir)
+	return supportsQuota && err == nil
+}

test/e2e_node/util.go

@@ -34,7 +34,9 @@ import (
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/component-base/featuregate"
 	internalapi "k8s.io/cri-api/pkg/apis"
 	kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1"
 	"k8s.io/kubernetes/pkg/features"
@@ -62,6 +64,7 @@ var kubeletAddress = flag.String("kubelet-address", "http://127.0.0.1:10255", "H
 var startServices = flag.Bool("start-services", true, "If true, start local node services")
 var stopServices = flag.Bool("stop-services", true, "If true, stop local node services after running tests")
 var busyboxImage = imageutils.GetE2EImage(imageutils.BusyBox)
+var perlImage = imageutils.GetE2EImage(imageutils.Perl)
 
 const (
 	// Kubelet internal cgroup name for node allocatable cgroup.
@@ -440,3 +443,15 @@ func reduceAllocatableMemoryUsage() {
 	_, err := exec.Command("sudo", "sh", "-c", cmd).CombinedOutput()
 	framework.ExpectNoError(err)
 }
+
+// Equivalent of featuregatetesting.SetFeatureGateDuringTest
+// which can't be used here because we're not in a Testing context.
+// This must be in a non-"_test" file to pass
+// make verify WHAT=test-featuregates
+func withFeatureGate(feature featuregate.Feature, desired bool) func() {
+	current := utilfeature.DefaultFeatureGate.Enabled(feature)
+	utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), desired))
+	return func() {
+		utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=%v", string(feature), current))
+	}
+}

test/e2e_node/util_xfs_linux.go

@@ -0,0 +1,74 @@
+// +build linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+import (
+	"path/filepath"
+	"syscall"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+func detectMountpoint(m mount.Interface, path string) string {
+	path, err := filepath.Abs(path)
+	if err == nil {
+		path, err = filepath.EvalSymlinks(path)
+	}
+	if err != nil {
+		return ""
+	}
+	for path != "" && path != "/" {
+		isNotMount, err := m.IsLikelyNotMountPoint(path)
+		if err != nil {
+			return ""
+		}
+		if !isNotMount {
+			return path
+		}
+		path = filepath.Dir(path)
+	}
+	return "/"
+}
+
+const (
+	xfsMagic = 0x58465342
+)
+
+// XFS over-allocates and then eventually removes that excess allocation.
+// That can lead to a file growing beyond its eventual size, causing
+// an unnecessary eviction:
+//
+// % ls -ls
+// total 32704
+// 32704 -rw-r--r-- 1 rkrawitz rkrawitz 20971520 Jan 15 13:16 foo.bin
+//
+// This issue can be hit regardless of the means used to count storage.
+// It is not present in ext4fs.
+func isXfs(dir string) bool {
+	mountpoint := detectMountpoint(mount.New(""), dir)
+	if mountpoint == "" {
+		return false
+	}
+	var buf syscall.Statfs_t
+	err := syscall.Statfs(mountpoint, &buf)
+	if err != nil {
+		return false
+	}
+	return buf.Type == xfsMagic
+}

test/e2e_node/util_xfs_unsupported.go

@@ -0,0 +1,23 @@
+// +build !linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_node
+
+func isXfs(dir string) bool {
+	return false
+}

test/utils/image/manifest.go

@@ -171,6 +171,8 @@ const (
 	// Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go
 	// Pause image
 	Pause
+	// Perl image
+	Perl
 	// Porter image
 	Porter
 	// PortForwardTester image
@@ -236,6 +238,7 @@ func initImageConfigs() map[int]Config {
 	configs[NoSnatTestProxy] = Config{e2eRegistry, "no-snat-test-proxy", "1.0"}
 	// Pause - when these values are updated, also update cmd/kubelet/app/options/container_runtime.go
 	configs[Pause] = Config{gcRegistry, "pause", "3.1"}
+	configs[Perl] = Config{dockerLibraryRegistry, "perl", "5.26"}
 	configs[Porter] = Config{e2eRegistry, "porter", "1.0"}
 	configs[PortForwardTester] = Config{e2eRegistry, "port-forward-tester", "1.0"}
 	configs[Redis] = Config{e2eRegistry, "redis", "1.0"}