mirror of https://github.com/k3s-io/k3s
remove deprecated flags LowDiskSpaceThresholdMB and OutOfDiskTransitionFrequency
parent
c2dd291ecb
commit
7a23f8b018
|
@ -158,7 +158,7 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m
|
||||||
# ContentType used by all components to communicate with apiserver.
|
# ContentType used by all components to communicate with apiserver.
|
||||||
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
|
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
|
||||||
|
|
||||||
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0 ${TEST_CLUSTER_API_CONTENT_TYPE}"
|
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE}"
|
||||||
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
|
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
|
||||||
NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
|
NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -226,8 +226,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
|
||||||
fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'.")
|
fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'.")
|
||||||
fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run.")
|
fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run.")
|
||||||
fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
|
fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
|
||||||
fs.Int32Var(&c.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", c.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected.")
|
|
||||||
fs.MarkDeprecated("low-diskspace-threshold-mb", "Use --eviction-hard instead. Will be removed in a future version.")
|
|
||||||
fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.")
|
fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.")
|
||||||
fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
|
fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
|
||||||
fs.StringVar(&c.CloudProvider, "cloud-provider", c.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
|
fs.StringVar(&c.CloudProvider, "cloud-provider", c.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
|
||||||
|
@ -269,8 +267,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
|
||||||
fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
|
fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
|
||||||
fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
|
fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
|
||||||
fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.")
|
fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.")
|
||||||
fs.DurationVar(&c.OutOfDiskTransitionFrequency.Duration, "outofdisk-transition-frequency", c.OutOfDiskTransitionFrequency.Duration, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status.")
|
|
||||||
fs.MarkDeprecated("outofdisk-transition-frequency", "Use --eviction-pressure-transition-period instead. Will be removed in a future version.")
|
|
||||||
|
|
||||||
fs.BoolVar(&c.EnableCustomMetrics, "enable-custom-metrics", c.EnableCustomMetrics, "Support for gathering custom metrics.")
|
fs.BoolVar(&c.EnableCustomMetrics, "enable-custom-metrics", c.EnableCustomMetrics, "Support for gathering custom metrics.")
|
||||||
fs.StringVar(&c.RuntimeCgroups, "runtime-cgroups", c.RuntimeCgroups, "Optional absolute name of cgroups to create and run the runtime in.")
|
fs.StringVar(&c.RuntimeCgroups, "runtime-cgroups", c.RuntimeCgroups, "Optional absolute name of cgroups to create and run the runtime in.")
|
||||||
|
|
|
@ -4998,7 +4998,7 @@ The resulting set of endpoints can be viewed as:<br>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td class="tableblock halign-left valign-top"><p class="tableblock">reason</p></td>
|
<td class="tableblock halign-left valign-top"><p class="tableblock">reason</p></td>
|
||||||
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>OutOfDisk</em></p></td>
|
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>Evicted</em></p></td>
|
||||||
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
|
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
|
||||||
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
|
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
|
||||||
<td class="tableblock halign-left valign-top"></td>
|
<td class="tableblock halign-left valign-top"></td>
|
||||||
|
|
|
@ -449,7 +449,6 @@ log-flush-frequency
|
||||||
log-lines-total
|
log-lines-total
|
||||||
logexporter-gcs-path
|
logexporter-gcs-path
|
||||||
long-running-request-regexp
|
long-running-request-regexp
|
||||||
low-diskspace-threshold-mb
|
|
||||||
make-iptables-util-chains
|
make-iptables-util-chains
|
||||||
make-symlinks
|
make-symlinks
|
||||||
manifest-url
|
manifest-url
|
||||||
|
@ -521,7 +520,6 @@ oidc-issuer-url
|
||||||
oidc-username-claim
|
oidc-username-claim
|
||||||
only-idl
|
only-idl
|
||||||
oom-score-adj
|
oom-score-adj
|
||||||
outofdisk-transition-frequency
|
|
||||||
output-base
|
output-base
|
||||||
output-directory
|
output-directory
|
||||||
output-file-base
|
output-file-base
|
||||||
|
|
|
@ -2334,7 +2334,7 @@ type PodStatus struct {
|
||||||
// A human readable message indicating details about why the pod is in this state.
|
// A human readable message indicating details about why the pod is in this state.
|
||||||
// +optional
|
// +optional
|
||||||
Message string
|
Message string
|
||||||
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'
|
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'
|
||||||
// +optional
|
// +optional
|
||||||
Reason string
|
Reason string
|
||||||
|
|
||||||
|
|
|
@ -309,10 +309,6 @@ type KubeletConfiguration struct {
|
||||||
// image garbage collection is never run. Lowest disk usage to garbage
|
// image garbage collection is never run. Lowest disk usage to garbage
|
||||||
// collect to.
|
// collect to.
|
||||||
ImageGCLowThresholdPercent int32
|
ImageGCLowThresholdPercent int32
|
||||||
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
|
|
||||||
// maintain. When disk space falls below this threshold, new pods would
|
|
||||||
// be rejected.
|
|
||||||
LowDiskSpaceThresholdMB int32
|
|
||||||
// How frequently to calculate and cache volume disk usage for all pods
|
// How frequently to calculate and cache volume disk usage for all pods
|
||||||
VolumeStatsAggPeriod metav1.Duration
|
VolumeStatsAggPeriod metav1.Duration
|
||||||
// volumePluginDir is the full path of the directory in which to search
|
// volumePluginDir is the full path of the directory in which to search
|
||||||
|
@ -412,10 +408,6 @@ type KubeletConfiguration struct {
|
||||||
// run docker daemon with version < 1.9 or an Aufs storage backend.
|
// run docker daemon with version < 1.9 or an Aufs storage backend.
|
||||||
// Issue #10959 has more details.
|
// Issue #10959 has more details.
|
||||||
SerializeImagePulls bool
|
SerializeImagePulls bool
|
||||||
// outOfDiskTransitionFrequency is duration for which the kubelet has to
|
|
||||||
// wait before transitioning out of out-of-disk node condition status.
|
|
||||||
// +optional
|
|
||||||
OutOfDiskTransitionFrequency metav1.Duration
|
|
||||||
// nodeLabels to add when registering the node in the cluster.
|
// nodeLabels to add when registering the node in the cluster.
|
||||||
NodeLabels map[string]string
|
NodeLabels map[string]string
|
||||||
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
|
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
|
||||||
|
|
|
@ -363,9 +363,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
|
||||||
if obj.KubeAPIBurst == 0 {
|
if obj.KubeAPIBurst == 0 {
|
||||||
obj.KubeAPIBurst = 10
|
obj.KubeAPIBurst = 10
|
||||||
}
|
}
|
||||||
if obj.OutOfDiskTransitionFrequency == zeroDuration {
|
|
||||||
obj.OutOfDiskTransitionFrequency = metav1.Duration{Duration: 5 * time.Minute}
|
|
||||||
}
|
|
||||||
if string(obj.HairpinMode) == "" {
|
if string(obj.HairpinMode) == "" {
|
||||||
obj.HairpinMode = PromiscuousBridge
|
obj.HairpinMode = PromiscuousBridge
|
||||||
}
|
}
|
||||||
|
|
|
@ -386,10 +386,6 @@ type KubeletConfiguration struct {
|
||||||
// image garbage collection is never run. Lowest disk usage to garbage
|
// image garbage collection is never run. Lowest disk usage to garbage
|
||||||
// collect to. The percent is calculated as this field value out of 100.
|
// collect to. The percent is calculated as this field value out of 100.
|
||||||
ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent"`
|
ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent"`
|
||||||
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
|
|
||||||
// maintain. When disk space falls below this threshold, new pods would
|
|
||||||
// be rejected.
|
|
||||||
LowDiskSpaceThresholdMB int32 `json:"lowDiskSpaceThresholdMB"`
|
|
||||||
// How frequently to calculate and cache volume disk usage for all pods
|
// How frequently to calculate and cache volume disk usage for all pods
|
||||||
VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod"`
|
VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod"`
|
||||||
// volumePluginDir is the full path of the directory in which to search
|
// volumePluginDir is the full path of the directory in which to search
|
||||||
|
@ -483,9 +479,6 @@ type KubeletConfiguration struct {
|
||||||
// run docker daemon with version < 1.9 or an Aufs storage backend.
|
// run docker daemon with version < 1.9 or an Aufs storage backend.
|
||||||
// Issue #10959 has more details.
|
// Issue #10959 has more details.
|
||||||
SerializeImagePulls *bool `json:"serializeImagePulls"`
|
SerializeImagePulls *bool `json:"serializeImagePulls"`
|
||||||
// outOfDiskTransitionFrequency is duration for which the kubelet has to
|
|
||||||
// wait before transitioning out of out-of-disk node condition status.
|
|
||||||
OutOfDiskTransitionFrequency metav1.Duration `json:"outOfDiskTransitionFrequency"`
|
|
||||||
// nodeLabels to add when registering the node in the cluster.
|
// nodeLabels to add when registering the node in the cluster.
|
||||||
NodeLabels map[string]string `json:"nodeLabels"`
|
NodeLabels map[string]string `json:"nodeLabels"`
|
||||||
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
|
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.
|
||||||
|
|
|
@ -12,7 +12,6 @@ go_library(
|
||||||
name = "go_default_library",
|
name = "go_default_library",
|
||||||
srcs = [
|
srcs = [
|
||||||
"active_deadline.go",
|
"active_deadline.go",
|
||||||
"disk_manager.go",
|
|
||||||
"doc.go",
|
"doc.go",
|
||||||
"kubelet.go",
|
"kubelet.go",
|
||||||
"kubelet_cadvisor.go",
|
"kubelet_cadvisor.go",
|
||||||
|
@ -148,7 +147,6 @@ go_test(
|
||||||
name = "go_default_test",
|
name = "go_default_test",
|
||||||
srcs = [
|
srcs = [
|
||||||
"active_deadline_test.go",
|
"active_deadline_test.go",
|
||||||
"disk_manager_test.go",
|
|
||||||
"kubelet_cadvisor_test.go",
|
"kubelet_cadvisor_test.go",
|
||||||
"kubelet_getters_test.go",
|
"kubelet_getters_test.go",
|
||||||
"kubelet_network_test.go",
|
"kubelet_network_test.go",
|
||||||
|
|
|
@ -1,138 +0,0 @@
|
||||||
/*
|
|
||||||
Copyright 2015 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package kubelet
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/golang/glog"
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v2"
|
|
||||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Manages policy for diskspace management for disks holding docker images and root fs.
|
|
||||||
|
|
||||||
// mb is used to easily convert an int to an mb
|
|
||||||
const mb = 1024 * 1024
|
|
||||||
|
|
||||||
// Implementation is thread-safe.
|
|
||||||
type diskSpaceManager interface {
|
|
||||||
// Checks the available disk space
|
|
||||||
IsRootDiskSpaceAvailable() (bool, error)
|
|
||||||
IsRuntimeDiskSpaceAvailable() (bool, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// DiskSpacePolicy defines the free disk for Docker and Root.
|
|
||||||
type DiskSpacePolicy struct {
|
|
||||||
// free disk space threshold for filesystem holding docker images.
|
|
||||||
DockerFreeDiskMB int
|
|
||||||
// free disk space threshold for root filesystem. Host volumes are created on root fs.
|
|
||||||
RootFreeDiskMB int
|
|
||||||
}
|
|
||||||
|
|
||||||
type fsInfo struct {
|
|
||||||
Usage int64
|
|
||||||
Capacity int64
|
|
||||||
Available int64
|
|
||||||
Timestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
type realDiskSpaceManager struct {
|
|
||||||
cadvisor cadvisor.Interface
|
|
||||||
cachedInfo map[string]fsInfo // cache of filesystem info.
|
|
||||||
lock sync.Mutex // protecting cachedInfo.
|
|
||||||
policy DiskSpacePolicy // thresholds. Set at creation time.
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.FsInfo, error)) (fsInfo, error) {
|
|
||||||
dm.lock.Lock()
|
|
||||||
defer dm.lock.Unlock()
|
|
||||||
fsi := fsInfo{}
|
|
||||||
if info, ok := dm.cachedInfo[fsType]; ok {
|
|
||||||
timeLimit := time.Now().Add(-2 * time.Second)
|
|
||||||
if info.Timestamp.After(timeLimit) {
|
|
||||||
fsi = info
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fsi.Timestamp.IsZero() {
|
|
||||||
fs, err := f()
|
|
||||||
if err != nil {
|
|
||||||
return fsInfo{}, err
|
|
||||||
}
|
|
||||||
fsi.Timestamp = time.Now()
|
|
||||||
fsi.Usage = int64(fs.Usage)
|
|
||||||
fsi.Capacity = int64(fs.Capacity)
|
|
||||||
fsi.Available = int64(fs.Available)
|
|
||||||
dm.cachedInfo[fsType] = fsi
|
|
||||||
}
|
|
||||||
return fsi, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *realDiskSpaceManager) IsRuntimeDiskSpaceAvailable() (bool, error) {
|
|
||||||
return dm.isSpaceAvailable("runtime", dm.policy.DockerFreeDiskMB, dm.cadvisor.ImagesFsInfo)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *realDiskSpaceManager) IsRootDiskSpaceAvailable() (bool, error) {
|
|
||||||
return dm.isSpaceAvailable("root", dm.policy.RootFreeDiskMB, dm.cadvisor.RootFsInfo)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f func() (cadvisorapi.FsInfo, error)) (bool, error) {
|
|
||||||
fsInfo, err := dm.getFsInfo(fsType, f)
|
|
||||||
if err != nil {
|
|
||||||
return true, fmt.Errorf("failed to get fs info for %q: %v", fsType, err)
|
|
||||||
}
|
|
||||||
if fsInfo.Capacity == 0 {
|
|
||||||
return true, fmt.Errorf("could not determine capacity for %q fs. Info: %+v", fsType, fsInfo)
|
|
||||||
}
|
|
||||||
if fsInfo.Available < 0 {
|
|
||||||
return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo)
|
|
||||||
}
|
|
||||||
|
|
||||||
if fsInfo.Available < int64(threshold)*mb {
|
|
||||||
glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold)
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func validatePolicy(policy DiskSpacePolicy) error {
|
|
||||||
if policy.DockerFreeDiskMB < 0 {
|
|
||||||
return fmt.Errorf("free disk space should be non-negative; invalid value %d for docker disk space threshold", policy.DockerFreeDiskMB)
|
|
||||||
}
|
|
||||||
if policy.RootFreeDiskMB < 0 {
|
|
||||||
return fmt.Errorf("free disk space should be non-negative; invalid value %d for root disk space threshold", policy.RootFreeDiskMB)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func newDiskSpaceManager(cadvisorInterface cadvisor.Interface, policy DiskSpacePolicy) (diskSpaceManager, error) {
|
|
||||||
// validate policy
|
|
||||||
err := validatePolicy(policy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
dm := &realDiskSpaceManager{
|
|
||||||
cadvisor: cadvisorInterface,
|
|
||||||
policy: policy,
|
|
||||||
cachedInfo: map[string]fsInfo{},
|
|
||||||
}
|
|
||||||
|
|
||||||
return dm, nil
|
|
||||||
}
|
|
|
@ -1,295 +0,0 @@
|
||||||
/*
|
|
||||||
Copyright 2015 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package kubelet
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v2"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func testPolicy() DiskSpacePolicy {
|
|
||||||
return DiskSpacePolicy{
|
|
||||||
DockerFreeDiskMB: 250,
|
|
||||||
RootFreeDiskMB: 250,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func setUp(t *testing.T) (*assert.Assertions, DiskSpacePolicy, *cadvisortest.Mock) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
policy := testPolicy()
|
|
||||||
c := new(cadvisortest.Mock)
|
|
||||||
return assert, policy, c
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidPolicy(t *testing.T) {
|
|
||||||
assert, policy, c := setUp(t)
|
|
||||||
_, err := newDiskSpaceManager(c, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
policy = testPolicy()
|
|
||||||
policy.DockerFreeDiskMB = -1
|
|
||||||
_, err = newDiskSpaceManager(c, policy)
|
|
||||||
assert.Error(err)
|
|
||||||
|
|
||||||
policy = testPolicy()
|
|
||||||
policy.RootFreeDiskMB = -1
|
|
||||||
_, err = newDiskSpaceManager(c, policy)
|
|
||||||
assert.Error(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSpaceAvailable(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 400 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 600 * mb,
|
|
||||||
}, nil)
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 9 * mb,
|
|
||||||
Capacity: 10 * mb,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ok, err := dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
|
|
||||||
ok, err = dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.False(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when
|
|
||||||
// space is available.
|
|
||||||
func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// 500MB available
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 9500 * mb,
|
|
||||||
Capacity: 10000 * mb,
|
|
||||||
Available: 500 * mb,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ok, err := dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIsRuntimeDiskSpaceAvailableWithoutSpace verifies IsRuntimeDiskSpaceAvailable results when
|
|
||||||
// space is not available.
|
|
||||||
func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
|
|
||||||
// 1MB available
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 999 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 1 * mb,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ok, err := dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.False(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when
|
|
||||||
// space is available.
|
|
||||||
func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
policy.RootFreeDiskMB = 10
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
// 999MB available
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 1 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 999 * mb,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ok, err := dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIsRootDiskSpaceAvailableWithoutSpace verifies IsRootDiskSpaceAvailable results when
|
|
||||||
// space is not available.
|
|
||||||
func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
policy.RootFreeDiskMB = 10
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
// 9MB available
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 990 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 9 * mb,
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
ok, err := dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.False(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCache verifies that caching works properly with DiskSpaceAvailable calls
|
|
||||||
func TestCache(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 400 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 300 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 500 * mb,
|
|
||||||
Capacity: 1000 * mb,
|
|
||||||
Available: 500 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
|
|
||||||
// Initial calls which should be recorded in mockCadvisor
|
|
||||||
ok, err := dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
|
|
||||||
ok, err = dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
|
|
||||||
// Get the current count of calls to mockCadvisor
|
|
||||||
cadvisorCallCount := len(mockCadvisor.Calls)
|
|
||||||
|
|
||||||
// Checking for space again shouldn't need to mock as cache would serve it.
|
|
||||||
ok, err = dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
|
|
||||||
ok, err = dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.NoError(err)
|
|
||||||
assert.True(ok)
|
|
||||||
|
|
||||||
// Ensure no more calls to the mockCadvisor occurred
|
|
||||||
assert.Equal(cadvisorCallCount, len(mockCadvisor.Calls))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFsInfoError verifies errors are returned by DiskSpaceAvailable calls
|
|
||||||
// when FsInfo calls return an error
|
|
||||||
func TestFsInfoError(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
policy.RootFreeDiskMB = 10
|
|
||||||
dm, err := newDiskSpaceManager(mockCadvisor, policy)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("can't find fs"))
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("EBUSY"))
|
|
||||||
ok, err := dm.IsRuntimeDiskSpaceAvailable()
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(ok)
|
|
||||||
ok, err = dm.IsRootDiskSpaceAvailable()
|
|
||||||
assert.Error(err)
|
|
||||||
assert.True(ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test_getFSInfo verifies multiple possible cases for getFsInfo.
|
|
||||||
func Test_getFsInfo(t *testing.T) {
|
|
||||||
assert, policy, mockCadvisor := setUp(t)
|
|
||||||
|
|
||||||
// Sunny day case
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 10 * mb,
|
|
||||||
Capacity: 100 * mb,
|
|
||||||
Available: 90 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
|
|
||||||
dm := &realDiskSpaceManager{
|
|
||||||
cadvisor: mockCadvisor,
|
|
||||||
policy: policy,
|
|
||||||
cachedInfo: map[string]fsInfo{},
|
|
||||||
}
|
|
||||||
|
|
||||||
available, err := dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
|
|
||||||
assert.True(available)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
// Threshold case
|
|
||||||
mockCadvisor = new(cadvisortest.Mock)
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 9 * mb,
|
|
||||||
Capacity: 100 * mb,
|
|
||||||
Available: 9 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
|
|
||||||
dm = &realDiskSpaceManager{
|
|
||||||
cadvisor: mockCadvisor,
|
|
||||||
policy: policy,
|
|
||||||
cachedInfo: map[string]fsInfo{},
|
|
||||||
}
|
|
||||||
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
|
|
||||||
assert.False(available)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
// Frozen case
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 9 * mb,
|
|
||||||
Capacity: 10 * mb,
|
|
||||||
Available: 500 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
|
|
||||||
dm = &realDiskSpaceManager{
|
|
||||||
cadvisor: mockCadvisor,
|
|
||||||
policy: policy,
|
|
||||||
cachedInfo: map[string]fsInfo{},
|
|
||||||
}
|
|
||||||
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
|
|
||||||
assert.True(available)
|
|
||||||
assert.NoError(err)
|
|
||||||
|
|
||||||
// Capacity error case
|
|
||||||
mockCadvisor = new(cadvisortest.Mock)
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
|
|
||||||
Usage: 9 * mb,
|
|
||||||
Capacity: 0,
|
|
||||||
Available: 500 * mb,
|
|
||||||
}, nil).Once()
|
|
||||||
|
|
||||||
dm = &realDiskSpaceManager{
|
|
||||||
cadvisor: mockCadvisor,
|
|
||||||
policy: policy,
|
|
||||||
cachedInfo: map[string]fsInfo{},
|
|
||||||
}
|
|
||||||
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
|
|
||||||
assert.True(available)
|
|
||||||
assert.Error(err)
|
|
||||||
assert.Contains(fmt.Sprintf("%s", err), "could not determine capacity")
|
|
||||||
|
|
||||||
// Available error case skipped as v2.FSInfo uses uint64 and this
|
|
||||||
// can not be less than 0
|
|
||||||
}
|
|
|
@ -53,7 +53,6 @@ const (
|
||||||
NodeSelectorMismatching = "NodeSelectorMismatching"
|
NodeSelectorMismatching = "NodeSelectorMismatching"
|
||||||
InsufficientFreeCPU = "InsufficientFreeCPU"
|
InsufficientFreeCPU = "InsufficientFreeCPU"
|
||||||
InsufficientFreeMemory = "InsufficientFreeMemory"
|
InsufficientFreeMemory = "InsufficientFreeMemory"
|
||||||
OutOfDisk = "OutOfDisk"
|
|
||||||
HostNetworkNotSupported = "HostNetworkNotSupported"
|
HostNetworkNotSupported = "HostNetworkNotSupported"
|
||||||
UndefinedShaper = "NilShaper"
|
UndefinedShaper = "NilShaper"
|
||||||
NodeRebooted = "Rebooted"
|
NodeRebooted = "Rebooted"
|
||||||
|
|
|
@ -1363,12 +1363,12 @@ func TestHasNodeConditions(t *testing.T) {
|
||||||
result bool
|
result bool
|
||||||
}{
|
}{
|
||||||
"has-condition": {
|
"has-condition": {
|
||||||
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk, v1.NodeMemoryPressure},
|
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure, v1.NodeMemoryPressure},
|
||||||
item: v1.NodeMemoryPressure,
|
item: v1.NodeMemoryPressure,
|
||||||
result: true,
|
result: true,
|
||||||
},
|
},
|
||||||
"does-not-have-condition": {
|
"does-not-have-condition": {
|
||||||
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk},
|
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure},
|
||||||
item: v1.NodeMemoryPressure,
|
item: v1.NodeMemoryPressure,
|
||||||
result: false,
|
result: false,
|
||||||
},
|
},
|
||||||
|
|
|
@ -368,11 +368,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
|
||||||
LowThresholdPercent: int(kubeCfg.ImageGCLowThresholdPercent),
|
LowThresholdPercent: int(kubeCfg.ImageGCLowThresholdPercent),
|
||||||
}
|
}
|
||||||
|
|
||||||
diskSpacePolicy := DiskSpacePolicy{
|
|
||||||
DockerFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
|
|
||||||
RootFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
|
|
||||||
}
|
|
||||||
|
|
||||||
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
|
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
|
||||||
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
|
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
|
||||||
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
|
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
|
||||||
|
@ -416,10 +411,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
|
||||||
Namespace: "",
|
Namespace: "",
|
||||||
}
|
}
|
||||||
|
|
||||||
diskSpaceManager, err := newDiskSpaceManager(kubeDeps.CAdvisorInterface, diskSpacePolicy)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to initialize disk manager: %v", err)
|
|
||||||
}
|
|
||||||
containerRefManager := kubecontainer.NewRefManager()
|
containerRefManager := kubecontainer.NewRefManager()
|
||||||
|
|
||||||
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
|
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
|
||||||
|
@ -453,7 +444,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
|
||||||
streamingConnectionIdleTimeout: kubeCfg.StreamingConnectionIdleTimeout.Duration,
|
streamingConnectionIdleTimeout: kubeCfg.StreamingConnectionIdleTimeout.Duration,
|
||||||
recorder: kubeDeps.Recorder,
|
recorder: kubeDeps.Recorder,
|
||||||
cadvisor: kubeDeps.CAdvisorInterface,
|
cadvisor: kubeDeps.CAdvisorInterface,
|
||||||
diskSpaceManager: diskSpaceManager,
|
|
||||||
cloud: kubeDeps.Cloud,
|
cloud: kubeDeps.Cloud,
|
||||||
autoDetectCloudProvider: (componentconfigv1alpha1.AutoDetectCloudProvider == kubeCfg.CloudProvider),
|
autoDetectCloudProvider: (componentconfigv1alpha1.AutoDetectCloudProvider == kubeCfg.CloudProvider),
|
||||||
externalCloudProvider: cloudprovider.IsExternal(kubeCfg.CloudProvider),
|
externalCloudProvider: cloudprovider.IsExternal(kubeCfg.CloudProvider),
|
||||||
|
@ -475,7 +465,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
|
||||||
containerManager: kubeDeps.ContainerManager,
|
containerManager: kubeDeps.ContainerManager,
|
||||||
nodeIP: net.ParseIP(nodeIP),
|
nodeIP: net.ParseIP(nodeIP),
|
||||||
clock: clock.RealClock{},
|
clock: clock.RealClock{},
|
||||||
outOfDiskTransitionFrequency: kubeCfg.OutOfDiskTransitionFrequency.Duration,
|
|
||||||
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
|
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
|
||||||
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
|
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
|
||||||
makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains,
|
makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains,
|
||||||
|
@ -924,9 +913,6 @@ type Kubelet struct {
|
||||||
// Manager for image garbage collection.
|
// Manager for image garbage collection.
|
||||||
imageManager images.ImageGCManager
|
imageManager images.ImageGCManager
|
||||||
|
|
||||||
// Diskspace manager.
|
|
||||||
diskSpaceManager diskSpaceManager
|
|
||||||
|
|
||||||
// Secret manager.
|
// Secret manager.
|
||||||
secretManager secret.Manager
|
secretManager secret.Manager
|
||||||
|
|
||||||
|
@ -1044,12 +1030,6 @@ type Kubelet struct {
|
||||||
// easy to test the code.
|
// easy to test the code.
|
||||||
clock clock.Clock
|
clock clock.Clock
|
||||||
|
|
||||||
// outOfDiskTransitionFrequency specifies the amount of time the kubelet has to be actually
|
|
||||||
// not out of disk before it can transition the node condition status from out-of-disk to
|
|
||||||
// not-out-of-disk. This prevents a pod that causes out-of-disk condition from repeatedly
|
|
||||||
// getting rescheduled onto the node.
|
|
||||||
outOfDiskTransitionFrequency time.Duration
|
|
||||||
|
|
||||||
// handlers called during the tryUpdateNodeStatus cycle
|
// handlers called during the tryUpdateNodeStatus cycle
|
||||||
setNodeStatusFuncs []func(*v1.Node) error
|
setNodeStatusFuncs []func(*v1.Node) error
|
||||||
|
|
||||||
|
@ -1650,27 +1630,6 @@ func (kl *Kubelet) deletePod(pod *v1.Pod) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// isOutOfDisk detects if pods can't fit due to lack of disk space.
|
|
||||||
func (kl *Kubelet) isOutOfDisk() bool {
|
|
||||||
// Check disk space once globally and reject or accept all new pods.
|
|
||||||
withinBounds, err := kl.diskSpaceManager.IsRuntimeDiskSpaceAvailable()
|
|
||||||
// Assume enough space in case of errors.
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Failed to check if disk space is available for the runtime: %v", err)
|
|
||||||
} else if !withinBounds {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
withinBounds, err = kl.diskSpaceManager.IsRootDiskSpaceAvailable()
|
|
||||||
// Assume enough space in case of errors.
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("Failed to check if disk space is available on the root partition: %v", err)
|
|
||||||
} else if !withinBounds {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// rejectPod records an event about the pod with the given reason and message,
|
// rejectPod records an event about the pod with the given reason and message,
|
||||||
// and updates the pod to the failed phase in the status manage.
|
// and updates the pod to the failed phase in the status manage.
|
||||||
func (kl *Kubelet) rejectPod(pod *v1.Pod, reason, message string) {
|
func (kl *Kubelet) rejectPod(pod *v1.Pod, reason, message string) {
|
||||||
|
@ -1697,12 +1656,6 @@ func (kl *Kubelet) canAdmitPod(pods []*v1.Pod, pod *v1.Pod) (bool, string, strin
|
||||||
return false, result.Reason, result.Message
|
return false, result.Reason, result.Message
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
|
|
||||||
// add the disk space predicate to predicates.GeneralPredicates.
|
|
||||||
if kl.isOutOfDisk() {
|
|
||||||
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), "predicate fails due to OutOfDisk")
|
|
||||||
return false, "OutOfDisk", "cannot be started due to lack of disk space."
|
|
||||||
}
|
|
||||||
|
|
||||||
return true, "", ""
|
return true, "", ""
|
||||||
}
|
}
|
||||||
|
|
|
@ -852,70 +852,6 @@ func (kl *Kubelet) setNodeDiskPressureCondition(node *v1.Node) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set OODCondition for the node.
|
|
||||||
func (kl *Kubelet) setNodeOODCondition(node *v1.Node) {
|
|
||||||
currentTime := metav1.NewTime(kl.clock.Now())
|
|
||||||
var nodeOODCondition *v1.NodeCondition
|
|
||||||
|
|
||||||
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
|
|
||||||
for i := range node.Status.Conditions {
|
|
||||||
if node.Status.Conditions[i].Type == v1.NodeOutOfDisk {
|
|
||||||
nodeOODCondition = &node.Status.Conditions[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newOODCondition := false
|
|
||||||
// If the NodeOutOfDisk condition doesn't exist, create one.
|
|
||||||
if nodeOODCondition == nil {
|
|
||||||
nodeOODCondition = &v1.NodeCondition{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionUnknown,
|
|
||||||
}
|
|
||||||
// nodeOODCondition cannot be appended to node.Status.Conditions here because it gets
|
|
||||||
// copied to the slice. So if we append nodeOODCondition to the slice here none of the
|
|
||||||
// updates we make to nodeOODCondition below are reflected in the slice.
|
|
||||||
newOODCondition = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update the heartbeat time irrespective of all the conditions.
|
|
||||||
nodeOODCondition.LastHeartbeatTime = currentTime
|
|
||||||
|
|
||||||
// Note: The conditions below take care of the case when a new NodeOutOfDisk condition is
|
|
||||||
// created and as well as the case when the condition already exists. When a new condition
|
|
||||||
// is created its status is set to v1.ConditionUnknown which matches either
|
|
||||||
// nodeOODCondition.Status != v1.ConditionTrue or
|
|
||||||
// nodeOODCondition.Status != v1.ConditionFalse in the conditions below depending on whether
|
|
||||||
// the kubelet is out of disk or not.
|
|
||||||
if kl.isOutOfDisk() {
|
|
||||||
if nodeOODCondition.Status != v1.ConditionTrue {
|
|
||||||
nodeOODCondition.Status = v1.ConditionTrue
|
|
||||||
nodeOODCondition.Reason = "KubeletOutOfDisk"
|
|
||||||
nodeOODCondition.Message = "out of disk space"
|
|
||||||
nodeOODCondition.LastTransitionTime = currentTime
|
|
||||||
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeOutOfDisk")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if nodeOODCondition.Status != v1.ConditionFalse {
|
|
||||||
// Update the out of disk condition when the condition status is unknown even if we
|
|
||||||
// are within the outOfDiskTransitionFrequency duration. We do this to set the
|
|
||||||
// condition status correctly at kubelet startup.
|
|
||||||
if nodeOODCondition.Status == v1.ConditionUnknown || kl.clock.Since(nodeOODCondition.LastTransitionTime.Time) >= kl.outOfDiskTransitionFrequency {
|
|
||||||
nodeOODCondition.Status = v1.ConditionFalse
|
|
||||||
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
|
|
||||||
nodeOODCondition.Message = "kubelet has sufficient disk space available"
|
|
||||||
nodeOODCondition.LastTransitionTime = currentTime
|
|
||||||
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientDisk")
|
|
||||||
} else {
|
|
||||||
glog.Infof("Node condition status for OutOfDisk is false, but last transition time is less than %s", kl.outOfDiskTransitionFrequency)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if newOODCondition {
|
|
||||||
node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
|
// Maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
|
||||||
// TODO: why is this a package var?
|
// TODO: why is this a package var?
|
||||||
var oldNodeUnschedulable bool
|
var oldNodeUnschedulable bool
|
||||||
|
@ -966,7 +902,6 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
|
||||||
return []func(*v1.Node) error{
|
return []func(*v1.Node) error{
|
||||||
kl.setNodeAddress,
|
kl.setNodeAddress,
|
||||||
withoutError(kl.setNodeStatusInfo),
|
withoutError(kl.setNodeStatusInfo),
|
||||||
withoutError(kl.setNodeOODCondition),
|
|
||||||
withoutError(kl.setNodeMemoryPressureCondition),
|
withoutError(kl.setNodeMemoryPressureCondition),
|
||||||
withoutError(kl.setNodeDiskPressureCondition),
|
withoutError(kl.setNodeDiskPressureCondition),
|
||||||
withoutError(kl.setNodeReadyCondition),
|
withoutError(kl.setNodeReadyCondition),
|
||||||
|
|
|
@ -163,22 +163,11 @@ func TestUpdateNewNodeStatus(t *testing.T) {
|
||||||
}
|
}
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
||||||
|
|
||||||
// Make kubelet report that it has sufficient disk space.
|
|
||||||
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
|
|
||||||
|
|
||||||
expectedNode := &v1.Node{
|
expectedNode := &v1.Node{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
||||||
Spec: v1.NodeSpec{},
|
Spec: v1.NodeSpec{},
|
||||||
Status: v1.NodeStatus{
|
Status: v1.NodeStatus{
|
||||||
Conditions: []v1.NodeCondition{
|
Conditions: []v1.NodeCondition{
|
||||||
{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionFalse,
|
|
||||||
Reason: "KubeletHasSufficientDisk",
|
|
||||||
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
|
|
||||||
LastHeartbeatTime: metav1.Time{},
|
|
||||||
LastTransitionTime: metav1.Time{},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Type: v1.NodeMemoryPressure,
|
Type: v1.NodeMemoryPressure,
|
||||||
Status: v1.ConditionFalse,
|
Status: v1.ConditionFalse,
|
||||||
|
@ -256,80 +245,6 @@ func TestUpdateNewNodeStatus(t *testing.T) {
|
||||||
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
|
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUpdateNewNodeOutOfDiskStatusWithTransitionFrequency(t *testing.T) {
|
|
||||||
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
|
||||||
defer testKubelet.Cleanup()
|
|
||||||
kubelet := testKubelet.kubelet
|
|
||||||
kubelet.containerManager = &localCM{
|
|
||||||
ContainerManager: cm.NewStubContainerManager(),
|
|
||||||
allocatable: v1.ResourceList{
|
|
||||||
v1.ResourceCPU: *resource.NewMilliQuantity(200, resource.DecimalSI),
|
|
||||||
v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
|
|
||||||
},
|
|
||||||
capacity: v1.ResourceList{
|
|
||||||
v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI),
|
|
||||||
v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
kubeClient := testKubelet.fakeKubeClient
|
|
||||||
existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}
|
|
||||||
kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
|
|
||||||
machineInfo := &cadvisorapi.MachineInfo{
|
|
||||||
MachineID: "123",
|
|
||||||
SystemUUID: "abc",
|
|
||||||
BootID: "1b3",
|
|
||||||
NumCores: 2,
|
|
||||||
MemoryCapacity: 1024,
|
|
||||||
}
|
|
||||||
mockCadvisor := testKubelet.fakeCadvisor
|
|
||||||
mockCadvisor.On("Start").Return(nil)
|
|
||||||
mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
|
|
||||||
versionInfo := &cadvisorapi.VersionInfo{
|
|
||||||
KernelVersion: "3.16.0-0.bpo.4-amd64",
|
|
||||||
ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
|
|
||||||
}
|
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
|
||||||
|
|
||||||
// Make Kubelet report that it has sufficient disk space.
|
|
||||||
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100)
|
|
||||||
require.NoError(t, err, "update the disk space manager")
|
|
||||||
|
|
||||||
kubelet.outOfDiskTransitionFrequency = 10 * time.Second
|
|
||||||
|
|
||||||
expectedNodeOutOfDiskCondition := v1.NodeCondition{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionFalse,
|
|
||||||
Reason: "KubeletHasSufficientDisk",
|
|
||||||
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
|
|
||||||
LastHeartbeatTime: metav1.Time{},
|
|
||||||
LastTransitionTime: metav1.Time{},
|
|
||||||
}
|
|
||||||
|
|
||||||
kubelet.updateRuntimeUp()
|
|
||||||
assert.NoError(t, kubelet.updateNodeStatus())
|
|
||||||
|
|
||||||
actions := kubeClient.Actions()
|
|
||||||
require.Len(t, actions, 2)
|
|
||||||
require.True(t, actions[1].Matches("patch", "nodes"))
|
|
||||||
require.Equal(t, "status", actions[1].GetSubresource())
|
|
||||||
|
|
||||||
updatedNode, err := applyNodeStatusPatch(&existingNode, actions[1].(core.PatchActionImpl).GetPatch())
|
|
||||||
assert.NoError(t, err, "apply the node status patch")
|
|
||||||
|
|
||||||
var oodCondition v1.NodeCondition
|
|
||||||
for i, cond := range updatedNode.Status.Conditions {
|
|
||||||
assert.False(t, cond.LastHeartbeatTime.IsZero(), "LastHeartbeatTime for %v condition is zero", cond.Type)
|
|
||||||
assert.False(t, cond.LastTransitionTime.IsZero(), "LastTransitionTime for %v condition is zero", cond.Type)
|
|
||||||
updatedNode.Status.Conditions[i].LastHeartbeatTime = metav1.Time{}
|
|
||||||
updatedNode.Status.Conditions[i].LastTransitionTime = metav1.Time{}
|
|
||||||
if cond.Type == v1.NodeOutOfDisk {
|
|
||||||
oodCondition = updatedNode.Status.Conditions[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert.EqualValues(t, expectedNodeOutOfDiskCondition, oodCondition)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateExistingNodeStatus(t *testing.T) {
|
func TestUpdateExistingNodeStatus(t *testing.T) {
|
||||||
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
||||||
defer testKubelet.Cleanup()
|
defer testKubelet.Cleanup()
|
||||||
|
@ -352,14 +267,6 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
|
||||||
Spec: v1.NodeSpec{},
|
Spec: v1.NodeSpec{},
|
||||||
Status: v1.NodeStatus{
|
Status: v1.NodeStatus{
|
||||||
Conditions: []v1.NodeCondition{
|
Conditions: []v1.NodeCondition{
|
||||||
{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionTrue,
|
|
||||||
Reason: "KubeletOutOfDisk",
|
|
||||||
Message: "out of disk space",
|
|
||||||
LastHeartbeatTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
|
|
||||||
LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Type: v1.NodeMemoryPressure,
|
Type: v1.NodeMemoryPressure,
|
||||||
Status: v1.ConditionFalse,
|
Status: v1.ConditionFalse,
|
||||||
|
@ -414,23 +321,11 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
|
||||||
}
|
}
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
||||||
|
|
||||||
// Make kubelet report that it is out of disk space.
|
|
||||||
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 50, 50, 100, 100)
|
|
||||||
require.NoError(t, err, "update the disk space manager")
|
|
||||||
|
|
||||||
expectedNode := &v1.Node{
|
expectedNode := &v1.Node{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
||||||
Spec: v1.NodeSpec{},
|
Spec: v1.NodeSpec{},
|
||||||
Status: v1.NodeStatus{
|
Status: v1.NodeStatus{
|
||||||
Conditions: []v1.NodeCondition{
|
Conditions: []v1.NodeCondition{
|
||||||
{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionTrue,
|
|
||||||
Reason: "KubeletOutOfDisk",
|
|
||||||
Message: "out of disk space",
|
|
||||||
LastHeartbeatTime: metav1.Time{}, // placeholder
|
|
||||||
LastTransitionTime: metav1.Time{}, // placeholder
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Type: v1.NodeMemoryPressure,
|
Type: v1.NodeMemoryPressure,
|
||||||
Status: v1.ConditionFalse,
|
Status: v1.ConditionFalse,
|
||||||
|
@ -524,167 +419,6 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
|
||||||
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
|
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUpdateExistingNodeOutOfDiskStatusWithTransitionFrequency(t *testing.T) {
|
|
||||||
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
|
||||||
defer testKubelet.Cleanup()
|
|
||||||
kubelet := testKubelet.kubelet
|
|
||||||
kubelet.containerManager = &localCM{
|
|
||||||
ContainerManager: cm.NewStubContainerManager(),
|
|
||||||
allocatable: v1.ResourceList{
|
|
||||||
v1.ResourceCPU: *resource.NewMilliQuantity(200, resource.DecimalSI),
|
|
||||||
v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
|
|
||||||
},
|
|
||||||
capacity: v1.ResourceList{
|
|
||||||
v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI),
|
|
||||||
v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
clock := testKubelet.fakeClock
|
|
||||||
// Do not set nano second, because apiserver function doesn't support nano second. (Only support
|
|
||||||
// RFC3339).
|
|
||||||
clock.SetTime(time.Unix(123456, 0))
|
|
||||||
kubeClient := testKubelet.fakeKubeClient
|
|
||||||
existingNode := v1.Node{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
|
||||||
Spec: v1.NodeSpec{},
|
|
||||||
Status: v1.NodeStatus{
|
|
||||||
Conditions: []v1.NodeCondition{
|
|
||||||
{
|
|
||||||
Type: v1.NodeReady,
|
|
||||||
Status: v1.ConditionTrue,
|
|
||||||
Reason: "KubeletReady",
|
|
||||||
Message: fmt.Sprintf("kubelet is posting ready status"),
|
|
||||||
LastHeartbeatTime: metav1.NewTime(clock.Now()),
|
|
||||||
LastTransitionTime: metav1.NewTime(clock.Now()),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionTrue,
|
|
||||||
Reason: "KubeletOutOfDisk",
|
|
||||||
Message: "out of disk space",
|
|
||||||
LastHeartbeatTime: metav1.NewTime(clock.Now()),
|
|
||||||
LastTransitionTime: metav1.NewTime(clock.Now()),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
|
|
||||||
mockCadvisor := testKubelet.fakeCadvisor
|
|
||||||
machineInfo := &cadvisorapi.MachineInfo{
|
|
||||||
MachineID: "123",
|
|
||||||
SystemUUID: "abc",
|
|
||||||
BootID: "1b3",
|
|
||||||
NumCores: 2,
|
|
||||||
MemoryCapacity: 1024,
|
|
||||||
}
|
|
||||||
fsInfo := cadvisorapiv2.FsInfo{
|
|
||||||
Device: "123",
|
|
||||||
}
|
|
||||||
mockCadvisor.On("Start").Return(nil)
|
|
||||||
mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(fsInfo, nil)
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(fsInfo, nil)
|
|
||||||
versionInfo := &cadvisorapi.VersionInfo{
|
|
||||||
KernelVersion: "3.16.0-0.bpo.4-amd64",
|
|
||||||
ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
|
|
||||||
DockerVersion: "1.5.0",
|
|
||||||
}
|
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
|
||||||
|
|
||||||
kubelet.outOfDiskTransitionFrequency = 5 * time.Second
|
|
||||||
|
|
||||||
ood := v1.NodeCondition{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionTrue,
|
|
||||||
Reason: "KubeletOutOfDisk",
|
|
||||||
Message: "out of disk space",
|
|
||||||
LastHeartbeatTime: metav1.NewTime(clock.Now()), // placeholder
|
|
||||||
LastTransitionTime: metav1.NewTime(clock.Now()), // placeholder
|
|
||||||
}
|
|
||||||
noOod := v1.NodeCondition{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionFalse,
|
|
||||||
Reason: "KubeletHasSufficientDisk",
|
|
||||||
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
|
|
||||||
LastHeartbeatTime: metav1.NewTime(clock.Now()), // placeholder
|
|
||||||
LastTransitionTime: metav1.NewTime(clock.Now()), // placeholder
|
|
||||||
}
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
rootFsAvail uint64
|
|
||||||
dockerFsAvail uint64
|
|
||||||
expected v1.NodeCondition
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
// NodeOutOfDisk==false
|
|
||||||
rootFsAvail: 200,
|
|
||||||
dockerFsAvail: 200,
|
|
||||||
expected: ood,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// NodeOutOfDisk==true
|
|
||||||
rootFsAvail: 50,
|
|
||||||
dockerFsAvail: 200,
|
|
||||||
expected: ood,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// NodeOutOfDisk==false
|
|
||||||
rootFsAvail: 200,
|
|
||||||
dockerFsAvail: 200,
|
|
||||||
expected: ood,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// NodeOutOfDisk==true
|
|
||||||
rootFsAvail: 200,
|
|
||||||
dockerFsAvail: 50,
|
|
||||||
expected: ood,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// NodeOutOfDisk==false
|
|
||||||
rootFsAvail: 200,
|
|
||||||
dockerFsAvail: 200,
|
|
||||||
expected: noOod,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
kubelet.updateRuntimeUp()
|
|
||||||
for tcIdx, tc := range testCases {
|
|
||||||
// Step by a second
|
|
||||||
clock.Step(1 * time.Second)
|
|
||||||
|
|
||||||
// Setup expected times.
|
|
||||||
tc.expected.LastHeartbeatTime = metav1.NewTime(clock.Now())
|
|
||||||
// In the last case, there should be a status transition for NodeOutOfDisk
|
|
||||||
if tcIdx == len(testCases)-1 {
|
|
||||||
tc.expected.LastTransitionTime = metav1.NewTime(clock.Now())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make kubelet report that it has sufficient disk space
|
|
||||||
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, tc.rootFsAvail, tc.dockerFsAvail, 100, 100)
|
|
||||||
require.NoError(t, err, "can't update disk space manager")
|
|
||||||
assert.NoError(t, kubelet.updateNodeStatus())
|
|
||||||
|
|
||||||
actions := kubeClient.Actions()
|
|
||||||
assert.Len(t, actions, 2, "test [%d]", tcIdx)
|
|
||||||
|
|
||||||
assert.IsType(t, core.PatchActionImpl{}, actions[1])
|
|
||||||
patchAction := actions[1].(core.PatchActionImpl)
|
|
||||||
|
|
||||||
updatedNode, err := applyNodeStatusPatch(&existingNode, patchAction.GetPatch())
|
|
||||||
require.NoError(t, err, "can't apply node status patch")
|
|
||||||
kubeClient.ClearActions()
|
|
||||||
|
|
||||||
var oodCondition v1.NodeCondition
|
|
||||||
for i, cond := range updatedNode.Status.Conditions {
|
|
||||||
if cond.Type == v1.NodeOutOfDisk {
|
|
||||||
oodCondition = updatedNode.Status.Conditions[i]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert.EqualValues(t, tc.expected, oodCondition)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
|
func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
|
||||||
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
||||||
defer testKubelet.Cleanup()
|
defer testKubelet.Cleanup()
|
||||||
|
@ -721,22 +455,11 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
|
||||||
}
|
}
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
||||||
|
|
||||||
// Make kubelet report that it has sufficient disk space.
|
|
||||||
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
|
|
||||||
|
|
||||||
expectedNode := &v1.Node{
|
expectedNode := &v1.Node{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
||||||
Spec: v1.NodeSpec{},
|
Spec: v1.NodeSpec{},
|
||||||
Status: v1.NodeStatus{
|
Status: v1.NodeStatus{
|
||||||
Conditions: []v1.NodeCondition{
|
Conditions: []v1.NodeCondition{
|
||||||
{
|
|
||||||
Type: v1.NodeOutOfDisk,
|
|
||||||
Status: v1.ConditionFalse,
|
|
||||||
Reason: "KubeletHasSufficientDisk",
|
|
||||||
Message: "kubelet has sufficient disk space available",
|
|
||||||
LastHeartbeatTime: metav1.Time{},
|
|
||||||
LastTransitionTime: metav1.Time{},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
Type: v1.NodeMemoryPressure,
|
Type: v1.NodeMemoryPressure,
|
||||||
Status: v1.ConditionFalse,
|
Status: v1.ConditionFalse,
|
||||||
|
@ -943,13 +666,13 @@ func TestRegisterWithApiServer(t *testing.T) {
|
||||||
}
|
}
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
|
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
|
||||||
Usage: 400 * mb,
|
Usage: 400,
|
||||||
Capacity: 1000 * mb,
|
Capacity: 1000,
|
||||||
Available: 600 * mb,
|
Available: 600,
|
||||||
}, nil)
|
}, nil)
|
||||||
mockCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
|
mockCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
|
||||||
Usage: 9 * mb,
|
Usage: 9,
|
||||||
Capacity: 10 * mb,
|
Capacity: 10,
|
||||||
}, nil)
|
}, nil)
|
||||||
|
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
|
@ -1178,9 +901,6 @@ func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) {
|
||||||
}
|
}
|
||||||
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
|
||||||
|
|
||||||
// Make kubelet report that it has sufficient disk space.
|
|
||||||
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
|
|
||||||
|
|
||||||
expectedNode := &v1.Node{
|
expectedNode := &v1.Node{
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
|
||||||
Spec: v1.NodeSpec{},
|
Spec: v1.NodeSpec{},
|
||||||
|
|
|
@ -213,11 +213,6 @@ func newTestKubeletWithImageList(
|
||||||
kubelet.configMapManager = configMapManager
|
kubelet.configMapManager = configMapManager
|
||||||
kubelet.podManager = kubepod.NewBasicPodManager(fakeMirrorClient, kubelet.secretManager, kubelet.configMapManager)
|
kubelet.podManager = kubepod.NewBasicPodManager(fakeMirrorClient, kubelet.secretManager, kubelet.configMapManager)
|
||||||
kubelet.statusManager = status.NewManager(fakeKubeClient, kubelet.podManager, &statustest.FakePodDeletionSafetyProvider{})
|
kubelet.statusManager = status.NewManager(fakeKubeClient, kubelet.podManager, &statustest.FakePodDeletionSafetyProvider{})
|
||||||
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, DiskSpacePolicy{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("can't initialize disk space manager: %v", err)
|
|
||||||
}
|
|
||||||
kubelet.diskSpaceManager = diskSpaceManager
|
|
||||||
|
|
||||||
kubelet.containerRuntime = fakeRuntime
|
kubelet.containerRuntime = fakeRuntime
|
||||||
kubelet.runtimeCache = containertest.NewFakeRuntimeCache(kubelet.containerRuntime)
|
kubelet.runtimeCache = containertest.NewFakeRuntimeCache(kubelet.containerRuntime)
|
||||||
|
@ -737,25 +732,6 @@ func TestValidateContainerLogStatus(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateDiskSpacePolicy creates a new DiskSpaceManager with a new policy. This new manager along
|
|
||||||
// with the mock FsInfo values added to Cadvisor should make the kubelet report that it has
|
|
||||||
// sufficient disk space or it is out of disk, depending on the capacity, availability and
|
|
||||||
// threshold values.
|
|
||||||
func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error {
|
|
||||||
dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb}
|
|
||||||
rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb}
|
|
||||||
mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil)
|
|
||||||
mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)
|
|
||||||
|
|
||||||
dsp := DiskSpacePolicy{DockerFreeDiskMB: rootThreshold, RootFreeDiskMB: dockerThreshold}
|
|
||||||
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, dsp)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
kubelet.diskSpaceManager = diskSpaceManager
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCreateMirrorPod(t *testing.T) {
|
func TestCreateMirrorPod(t *testing.T) {
|
||||||
for _, updateType := range []kubetypes.SyncPodType{kubetypes.SyncPodCreate, kubetypes.SyncPodUpdate} {
|
for _, updateType := range []kubetypes.SyncPodType{kubetypes.SyncPodCreate, kubetypes.SyncPodUpdate} {
|
||||||
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
|
||||||
|
|
|
@ -55,19 +55,18 @@ func TestRunOnce(t *testing.T) {
|
||||||
cadvisor := &cadvisortest.Mock{}
|
cadvisor := &cadvisortest.Mock{}
|
||||||
cadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil)
|
cadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil)
|
||||||
cadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
|
cadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
|
||||||
Usage: 400 * mb,
|
Usage: 400,
|
||||||
Capacity: 1000 * mb,
|
Capacity: 1000,
|
||||||
Available: 600 * mb,
|
Available: 600,
|
||||||
}, nil)
|
}, nil)
|
||||||
cadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
|
cadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
|
||||||
Usage: 9 * mb,
|
Usage: 9,
|
||||||
Capacity: 10 * mb,
|
Capacity: 10,
|
||||||
}, nil)
|
}, nil)
|
||||||
fakeSecretManager := secret.NewFakeManager()
|
fakeSecretManager := secret.NewFakeManager()
|
||||||
fakeConfigMapManager := configmap.NewFakeManager()
|
fakeConfigMapManager := configmap.NewFakeManager()
|
||||||
podManager := kubepod.NewBasicPodManager(
|
podManager := kubepod.NewBasicPodManager(
|
||||||
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
|
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
|
||||||
diskSpaceManager, _ := newDiskSpaceManager(cadvisor, DiskSpacePolicy{})
|
|
||||||
fakeRuntime := &containertest.FakeRuntime{}
|
fakeRuntime := &containertest.FakeRuntime{}
|
||||||
basePath, err := utiltesting.MkTmpdir("kubelet")
|
basePath, err := utiltesting.MkTmpdir("kubelet")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -82,7 +81,6 @@ func TestRunOnce(t *testing.T) {
|
||||||
statusManager: status.NewManager(nil, podManager, &statustest.FakePodDeletionSafetyProvider{}),
|
statusManager: status.NewManager(nil, podManager, &statustest.FakePodDeletionSafetyProvider{}),
|
||||||
podManager: podManager,
|
podManager: podManager,
|
||||||
os: &containertest.FakeOS{},
|
os: &containertest.FakeOS{},
|
||||||
diskSpaceManager: diskSpaceManager,
|
|
||||||
containerRuntime: fakeRuntime,
|
containerRuntime: fakeRuntime,
|
||||||
reasonCache: NewReasonCache(),
|
reasonCache: NewReasonCache(),
|
||||||
clock: clock.RealClock{},
|
clock: clock.RealClock{},
|
||||||
|
|
|
@ -134,14 +134,12 @@ func GetHollowKubeletConfig(
|
||||||
c.MinimumGCAge.Duration = 1 * time.Minute
|
c.MinimumGCAge.Duration = 1 * time.Minute
|
||||||
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
|
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
|
||||||
c.SyncFrequency.Duration = 10 * time.Second
|
c.SyncFrequency.Duration = 10 * time.Second
|
||||||
c.OutOfDiskTransitionFrequency.Duration = 5 * time.Minute
|
|
||||||
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
|
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
|
||||||
c.MaxPods = int32(maxPods)
|
c.MaxPods = int32(maxPods)
|
||||||
c.PodsPerCore = int32(podsPerCore)
|
c.PodsPerCore = int32(podsPerCore)
|
||||||
c.ClusterDNS = []string{}
|
c.ClusterDNS = []string{}
|
||||||
c.ImageGCHighThresholdPercent = 90
|
c.ImageGCHighThresholdPercent = 90
|
||||||
c.ImageGCLowThresholdPercent = 80
|
c.ImageGCLowThresholdPercent = 80
|
||||||
c.LowDiskSpaceThresholdMB = 256
|
|
||||||
c.VolumeStatsAggPeriod.Duration = time.Minute
|
c.VolumeStatsAggPeriod.Duration = time.Minute
|
||||||
c.CgroupRoot = ""
|
c.CgroupRoot = ""
|
||||||
c.ContainerRuntime = kubetypes.DockerContainerRuntime
|
c.ContainerRuntime = kubetypes.DockerContainerRuntime
|
||||||
|
|
|
@ -1588,7 +1588,7 @@ func TestPrintPod(t *testing.T) {
|
||||||
ObjectMeta: metav1.ObjectMeta{Name: "test5"},
|
ObjectMeta: metav1.ObjectMeta{Name: "test5"},
|
||||||
Spec: api.PodSpec{Containers: make([]api.Container, 2)},
|
Spec: api.PodSpec{Containers: make([]api.Container, 2)},
|
||||||
Status: api.PodStatus{
|
Status: api.PodStatus{
|
||||||
Reason: "OutOfDisk",
|
Reason: "podReason",
|
||||||
Phase: "podPhase",
|
Phase: "podPhase",
|
||||||
ContainerStatuses: []api.ContainerStatus{
|
ContainerStatuses: []api.ContainerStatus{
|
||||||
{Ready: true, RestartCount: 3, State: api.ContainerState{Running: &api.ContainerStateRunning{}}},
|
{Ready: true, RestartCount: 3, State: api.ContainerState{Running: &api.ContainerStateRunning{}}},
|
||||||
|
@ -1596,7 +1596,7 @@ func TestPrintPod(t *testing.T) {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "OutOfDisk", 6, "<unknown>"}}},
|
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", 6, "<unknown>"}}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -2624,7 +2624,7 @@ type PodStatus struct {
|
||||||
// +optional
|
// +optional
|
||||||
Message string `json:"message,omitempty" protobuf:"bytes,3,opt,name=message"`
|
Message string `json:"message,omitempty" protobuf:"bytes,3,opt,name=message"`
|
||||||
// A brief CamelCase message indicating details about why the pod is in this state.
|
// A brief CamelCase message indicating details about why the pod is in this state.
|
||||||
// e.g. 'OutOfDisk'
|
// e.g. 'Evicted'
|
||||||
// +optional
|
// +optional
|
||||||
Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`
|
Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`
|
||||||
|
|
||||||
|
|
|
@ -72,7 +72,6 @@ go_library(
|
||||||
"networking.go",
|
"networking.go",
|
||||||
"networking_perf.go",
|
"networking_perf.go",
|
||||||
"no-snat.go",
|
"no-snat.go",
|
||||||
"nodeoutofdisk.go",
|
|
||||||
"pod_gc.go",
|
"pod_gc.go",
|
||||||
"podpreset.go",
|
"podpreset.go",
|
||||||
"pods.go",
|
"pods.go",
|
||||||
|
@ -117,7 +116,6 @@ go_library(
|
||||||
"//test/images/net/nat:go_default_library",
|
"//test/images/net/nat:go_default_library",
|
||||||
"//test/utils:go_default_library",
|
"//test/utils:go_default_library",
|
||||||
"//vendor/github.com/golang/glog:go_default_library",
|
"//vendor/github.com/golang/glog:go_default_library",
|
||||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
|
||||||
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
"//vendor/github.com/onsi/ginkgo:go_default_library",
|
||||||
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
|
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
|
||||||
"//vendor/github.com/onsi/ginkgo/reporters:go_default_library",
|
"//vendor/github.com/onsi/ginkgo/reporters:go_default_library",
|
||||||
|
|
|
@ -1,269 +0,0 @@
|
||||||
/*
|
|
||||||
Copyright 2015 The Kubernetes Authors.
|
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
you may not use this file except in compliance with the License.
|
|
||||||
You may obtain a copy of the License at
|
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
See the License for the specific language governing permissions and
|
|
||||||
limitations under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package e2e
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
|
||||||
"k8s.io/api/core/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/api/resource"
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/fields"
|
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
|
||||||
clientset "k8s.io/client-go/kubernetes"
|
|
||||||
"k8s.io/kubernetes/test/e2e/framework"
|
|
||||||
|
|
||||||
. "github.com/onsi/ginkgo"
|
|
||||||
. "github.com/onsi/gomega"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
mb = 1024 * 1024
|
|
||||||
gb = 1024 * mb
|
|
||||||
|
|
||||||
// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
|
|
||||||
// is the default that is set today. This test might break if the default value changes. This value
|
|
||||||
// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
|
|
||||||
// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
|
|
||||||
// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
|
|
||||||
// flag value affects all the e2e tests. So we are hard-coding this value for now.
|
|
||||||
lowDiskSpaceThreshold uint64 = 256 * mb
|
|
||||||
|
|
||||||
nodeOODTimeOut = 5 * time.Minute
|
|
||||||
|
|
||||||
numNodeOODPods = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
// Plan:
|
|
||||||
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
|
|
||||||
// on that node. Arbitrarily choose that node to be node with index 0. This makes this a disruptive test.
|
|
||||||
// 2. Get the CPU capacity on unfilled node.
|
|
||||||
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
|
|
||||||
// to schedule 3 pods, so divide CPU capacity by 2.
|
|
||||||
// 4. Request the divided CPU for each pod.
|
|
||||||
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
|
|
||||||
// pod stays pending and does not schedule onto the nodes whose disks are full nor the node
|
|
||||||
// with the other two pods, since there is not enough free CPU capacity there.
|
|
||||||
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
|
|
||||||
// choose that node to be node with index 1.
|
|
||||||
// 7. Observe that the pod in pending status schedules on that node.
|
|
||||||
//
|
|
||||||
// Flaky issue #20015. We have no clear path for how to test this functionality in a non-flaky way.
|
|
||||||
var _ = framework.KubeDescribe("NodeOutOfDisk [Serial] [Flaky] [Disruptive]", func() {
|
|
||||||
var c clientset.Interface
|
|
||||||
var unfilledNodeName, recoveredNodeName string
|
|
||||||
f := framework.NewDefaultFramework("node-outofdisk")
|
|
||||||
|
|
||||||
BeforeEach(func() {
|
|
||||||
c = f.ClientSet
|
|
||||||
|
|
||||||
framework.Skipf("test is broken. #40249")
|
|
||||||
|
|
||||||
nodelist := framework.GetReadySchedulableNodesOrDie(c)
|
|
||||||
|
|
||||||
// Skip this test on small clusters. No need to fail since it is not a use
|
|
||||||
// case that any cluster of small size needs to support.
|
|
||||||
framework.SkipUnlessNodeCountIsAtLeast(2)
|
|
||||||
|
|
||||||
unfilledNodeName = nodelist.Items[0].Name
|
|
||||||
for _, node := range nodelist.Items[1:] {
|
|
||||||
fillDiskSpace(c, &node)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
AfterEach(func() {
|
|
||||||
|
|
||||||
nodelist := framework.GetReadySchedulableNodesOrDie(c)
|
|
||||||
Expect(len(nodelist.Items)).ToNot(BeZero())
|
|
||||||
for _, node := range nodelist.Items {
|
|
||||||
if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
recoverDiskSpace(c, &node)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
It("runs out of disk space", func() {
|
|
||||||
unfilledNode, err := c.Core().Nodes().Get(unfilledNodeName, metav1.GetOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
|
|
||||||
milliCpu, err := availCpu(c, unfilledNode)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
|
|
||||||
// node. We compute this value by dividing the available CPU capacity on the node by
|
|
||||||
// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
|
|
||||||
// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
|
|
||||||
// subtracting 1% from the value, we directly use 0.99 as the multiplier.
|
|
||||||
podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)
|
|
||||||
|
|
||||||
ns := f.Namespace.Name
|
|
||||||
podClient := c.Core().Pods(ns)
|
|
||||||
|
|
||||||
By("Creating pods and waiting for all but one pods to be scheduled")
|
|
||||||
|
|
||||||
for i := 0; i < numNodeOODPods-1; i++ {
|
|
||||||
name := fmt.Sprintf("pod-node-outofdisk-%d", i)
|
|
||||||
createOutOfDiskPod(c, ns, name, podCPU)
|
|
||||||
|
|
||||||
framework.ExpectNoError(f.WaitForPodRunning(name))
|
|
||||||
pod, err := podClient.Get(name, metav1.GetOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
|
|
||||||
}
|
|
||||||
|
|
||||||
pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
|
|
||||||
createOutOfDiskPod(c, ns, pendingPodName, podCPU)
|
|
||||||
|
|
||||||
By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
|
|
||||||
wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
|
|
||||||
selector := fields.Set{
|
|
||||||
"involvedObject.kind": "Pod",
|
|
||||||
"involvedObject.name": pendingPodName,
|
|
||||||
"involvedObject.namespace": ns,
|
|
||||||
"source": v1.DefaultSchedulerName,
|
|
||||||
"reason": "FailedScheduling",
|
|
||||||
}.AsSelector().String()
|
|
||||||
options := metav1.ListOptions{FieldSelector: selector}
|
|
||||||
schedEvents, err := c.Core().Events(ns).List(options)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
|
|
||||||
if len(schedEvents.Items) > 0 {
|
|
||||||
return true, nil
|
|
||||||
} else {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
nodelist := framework.GetReadySchedulableNodesOrDie(c)
|
|
||||||
Expect(len(nodelist.Items)).To(BeNumerically(">", 1))
|
|
||||||
|
|
||||||
nodeToRecover := nodelist.Items[1]
|
|
||||||
Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))
|
|
||||||
|
|
||||||
recoverDiskSpace(c, &nodeToRecover)
|
|
||||||
recoveredNodeName = nodeToRecover.Name
|
|
||||||
|
|
||||||
By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
|
|
||||||
framework.ExpectNoError(f.WaitForPodRunning(pendingPodName))
|
|
||||||
pendingPod, err := podClient.Get(pendingPodName, metav1.GetOptions{})
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
|
|
||||||
func createOutOfDiskPod(c clientset.Interface, ns, name string, milliCPU int64) {
|
|
||||||
podClient := c.Core().Pods(ns)
|
|
||||||
|
|
||||||
pod := &v1.Pod{
|
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
|
||||||
Name: name,
|
|
||||||
},
|
|
||||||
Spec: v1.PodSpec{
|
|
||||||
Containers: []v1.Container{
|
|
||||||
{
|
|
||||||
Name: "pause",
|
|
||||||
Image: framework.GetPauseImageName(c),
|
|
||||||
Resources: v1.ResourceRequirements{
|
|
||||||
Requests: v1.ResourceList{
|
|
||||||
// Request enough CPU to fit only two pods on a given node.
|
|
||||||
v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := podClient.Create(pod)
|
|
||||||
framework.ExpectNoError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
|
|
||||||
// all the pods from the total available CPU capacity on the node.
|
|
||||||
func availCpu(c clientset.Interface, node *v1.Node) (int64, error) {
|
|
||||||
podClient := c.Core().Pods(metav1.NamespaceAll)
|
|
||||||
|
|
||||||
selector := fields.Set{"spec.nodeName": node.Name}.AsSelector().String()
|
|
||||||
options := metav1.ListOptions{FieldSelector: selector}
|
|
||||||
pods, err := podClient.List(options)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
|
|
||||||
}
|
|
||||||
avail := node.Status.Capacity.Cpu().MilliValue()
|
|
||||||
for _, pod := range pods.Items {
|
|
||||||
for _, cont := range pod.Spec.Containers {
|
|
||||||
avail -= cont.Resources.Requests.Cpu().MilliValue()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return avail, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// availSize returns the available disk space on a given node by querying node stats which
|
|
||||||
// is in turn obtained internally from cadvisor.
|
|
||||||
func availSize(c clientset.Interface, node *v1.Node) (uint64, error) {
|
|
||||||
statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
|
|
||||||
framework.Logf("Querying stats for node %s using url %s", node.Name, statsResource)
|
|
||||||
res, err := c.Core().RESTClient().Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
|
|
||||||
}
|
|
||||||
ci := cadvisorapi.ContainerInfo{}
|
|
||||||
err = json.Unmarshal(res, &ci)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
|
|
||||||
}
|
|
||||||
return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
|
|
||||||
// space on the node is filled in such a way that the available space after filling the disk is just
|
|
||||||
// below the lowDiskSpaceThreshold mark.
|
|
||||||
func fillDiskSpace(c clientset.Interface, node *v1.Node) {
|
|
||||||
avail, err := availSize(c, node)
|
|
||||||
framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)
|
|
||||||
|
|
||||||
fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))
|
|
||||||
|
|
||||||
framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
|
|
||||||
By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))
|
|
||||||
|
|
||||||
cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
|
|
||||||
framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))
|
|
||||||
|
|
||||||
ood := framework.WaitForNodeToBe(c, node.Name, v1.NodeOutOfDisk, true, nodeOODTimeOut)
|
|
||||||
Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)
|
|
||||||
|
|
||||||
avail, err = availSize(c, node)
|
|
||||||
framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
|
|
||||||
Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
|
|
||||||
}
|
|
||||||
|
|
||||||
// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
|
|
||||||
func recoverDiskSpace(c clientset.Interface, node *v1.Node) {
|
|
||||||
By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
|
|
||||||
cmd := "rm -f test.img"
|
|
||||||
framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))
|
|
||||||
|
|
||||||
ood := framework.WaitForNodeToBe(c, node.Name, v1.NodeOutOfDisk, false, nodeOODTimeOut)
|
|
||||||
Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
|
|
||||||
}
|
|
|
@ -336,7 +336,6 @@ NoExecuteTaintManager doesn't evict pod with tolerations from tainted nodes,free
|
||||||
NoExecuteTaintManager eventually evict pod with finite tolerations from tainted nodes,freehan,0,scheduling
|
NoExecuteTaintManager eventually evict pod with finite tolerations from tainted nodes,freehan,0,scheduling
|
||||||
NoExecuteTaintManager evicts pods from tainted nodes,freehan,0,scheduling
|
NoExecuteTaintManager evicts pods from tainted nodes,freehan,0,scheduling
|
||||||
NoExecuteTaintManager removing taint cancels eviction,freehan,0,scheduling
|
NoExecuteTaintManager removing taint cancels eviction,freehan,0,scheduling
|
||||||
NodeOutOfDisk runs out of disk space,vishh,0,node
|
|
||||||
NodeProblemDetector KernelMonitor should generate node condition and events for corresponding errors,Random-Liu,0,node
|
NodeProblemDetector KernelMonitor should generate node condition and events for corresponding errors,Random-Liu,0,node
|
||||||
Nodes Resize should be able to add nodes,piosz,1,cluster-lifecycle
|
Nodes Resize should be able to add nodes,piosz,1,cluster-lifecycle
|
||||||
Nodes Resize should be able to delete nodes,zmerlynn,1,cluster-lifecycle
|
Nodes Resize should be able to delete nodes,zmerlynn,1,cluster-lifecycle
|
||||||
|
|
|
|
@ -83,7 +83,6 @@
|
||||||
"Logging",
|
"Logging",
|
||||||
"MemoryEviction",
|
"MemoryEviction",
|
||||||
"MirrorPod",
|
"MirrorPod",
|
||||||
"NodeOutOfDisk",
|
|
||||||
"NodeProblemDetector",
|
"NodeProblemDetector",
|
||||||
"Opaque",
|
"Opaque",
|
||||||
"Pod garbage",
|
"Pod garbage",
|
||||||
|
|
Loading…
Reference in New Issue