github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

remove deprecated flags LowDiskSpaceThresholdMB and OutOfDiskTransitionFrequency

pull/6/head
David Ashpole 2017-07-20 13:23:13 -07:00
parent c2dd291ecb
commit 7a23f8b018
25 changed files with 18 additions and 1171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cluster/gce/config-test.sh
View File

@ -158,7 +158,7 @@ TEST_CLUSTER_RESYNC_PERIOD="${TEST_CLUSTER_RESYNC_PERIOD:---min-resync-period=3m
# ContentType used by all components to communicate with apiserver.
TEST_CLUSTER_API_CONTENT_TYPE="${TEST_CLUSTER_API_CONTENT_TYPE:-}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false --outofdisk-transition-frequency=0 ${TEST_CLUSTER_API_CONTENT_TYPE}"
KUBELET_TEST_ARGS="${KUBELET_TEST_ARGS:-} --max-pods=110 --serialize-image-pulls=false ${TEST_CLUSTER_API_CONTENT_TYPE}"
if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]] || [[ "${NODE_OS_DISTRIBUTION}" == "ubuntu" ]]; then
NODE_KUBELET_TEST_ARGS=" --experimental-kernel-memcg-notification=true"
fi

4
cmd/kubelet/app/options/options.go
View File

@ -226,8 +226,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
fs.DurationVar(&c.ImageMinimumGCAge.Duration, "minimum-image-ttl-duration", c.ImageMinimumGCAge.Duration, "Minimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'.")
fs.Int32Var(&c.ImageGCHighThresholdPercent, "image-gc-high-threshold", c.ImageGCHighThresholdPercent, "The percent of disk usage after which image garbage collection is always run.")
fs.Int32Var(&c.ImageGCLowThresholdPercent, "image-gc-low-threshold", c.ImageGCLowThresholdPercent, "The percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to.")
fs.Int32Var(&c.LowDiskSpaceThresholdMB, "low-diskspace-threshold-mb", c.LowDiskSpaceThresholdMB, "The absolute free disk space, in MB, to maintain. When disk space falls below this threshold, new pods would be rejected.")
fs.MarkDeprecated("low-diskspace-threshold-mb", "Use --eviction-hard instead. Will be removed in a future version.")
fs.DurationVar(&c.VolumeStatsAggPeriod.Duration, "volume-stats-agg-period", c.VolumeStatsAggPeriod.Duration, "Specifies interval for kubelet to calculate and cache the volume disk usage for all pods and volumes. To disable volume calculations, set to 0.")
fs.StringVar(&c.VolumePluginDir, "volume-plugin-dir", c.VolumePluginDir, "<Warning: Alpha feature> The full path of the directory in which to search for additional third party volume plugins")
fs.StringVar(&c.CloudProvider, "cloud-provider", c.CloudProvider, "The provider for cloud services. By default, kubelet will attempt to auto-detect the cloud provider. Specify empty string for running with no cloud provider.")
@ -269,8 +267,6 @@ func (c *kubeletConfiguration) addFlags(fs *pflag.FlagSet) {
fs.Int32Var(&c.KubeAPIQPS, "kube-api-qps", c.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
fs.Int32Var(&c.KubeAPIBurst, "kube-api-burst", c.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
fs.BoolVar(&c.SerializeImagePulls, "serialize-image-pulls", c.SerializeImagePulls, "Pull images one at a time. We recommend *not* changing the default value on nodes that run docker daemon with version < 1.9 or an Aufs storage backend. Issue #10959 has more details.")
fs.DurationVar(&c.OutOfDiskTransitionFrequency.Duration, "outofdisk-transition-frequency", c.OutOfDiskTransitionFrequency.Duration, "Duration for which the kubelet has to wait before transitioning out of out-of-disk node condition status.")
fs.MarkDeprecated("outofdisk-transition-frequency", "Use --eviction-pressure-transition-period instead. Will be removed in a future version.")
fs.BoolVar(&c.EnableCustomMetrics, "enable-custom-metrics", c.EnableCustomMetrics, "Support for gathering custom metrics.")
fs.StringVar(&c.RuntimeCgroups, "runtime-cgroups", c.RuntimeCgroups, "Optional absolute name of cgroups to create and run the runtime in.")

2
docs/api-reference/v1/definitions.html
View File

@ -4998,7 +4998,7 @@ The resulting set of endpoints can be viewed as:<br>
</tr>
<tr>
<td class="tableblock halign-left valign-top"><p class="tableblock">reason</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>OutOfDisk</em></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">A brief CamelCase message indicating details about why the pod is in this state. e.g. <em>Evicted</em></p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">false</p></td>
<td class="tableblock halign-left valign-top"><p class="tableblock">string</p></td>
<td class="tableblock halign-left valign-top"></td>

2
hack/verify-flags/known-flags.txt
View File

@ -449,7 +449,6 @@ log-flush-frequency
log-lines-total
logexporter-gcs-path
long-running-request-regexp
low-diskspace-threshold-mb
make-iptables-util-chains
make-symlinks
manifest-url
@ -521,7 +520,6 @@ oidc-issuer-url
oidc-username-claim
only-idl
oom-score-adj
outofdisk-transition-frequency
output-base
output-directory
output-file-base

2
pkg/api/types.go
View File

@ -2334,7 +2334,7 @@ type PodStatus struct {
// A human readable message indicating details about why the pod is in this state.
// +optional
Message string
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'OutOfDisk'
// A brief CamelCase message indicating details about why the pod is in this state. e.g. 'Evicted'
// +optional
Reason string

8
pkg/apis/componentconfig/types.go
View File

@ -309,10 +309,6 @@ type KubeletConfiguration struct {
// image garbage collection is never run. Lowest disk usage to garbage
// collect to.
ImageGCLowThresholdPercent int32
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
// maintain. When disk space falls below this threshold, new pods would
// be rejected.
LowDiskSpaceThresholdMB int32
// How frequently to calculate and cache volume disk usage for all pods
VolumeStatsAggPeriod metav1.Duration
// volumePluginDir is the full path of the directory in which to search
@ -412,10 +408,6 @@ type KubeletConfiguration struct {
// run docker daemon with version < 1.9 or an Aufs storage backend.
// Issue #10959 has more details.
SerializeImagePulls bool
// outOfDiskTransitionFrequency is duration for which the kubelet has to
// wait before transitioning out of out-of-disk node condition status.
// +optional
OutOfDiskTransitionFrequency metav1.Duration
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.

3
pkg/apis/componentconfig/v1alpha1/defaults.go
View File

@ -363,9 +363,6 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
if obj.KubeAPIBurst == 0 {
obj.KubeAPIBurst = 10
}
if obj.OutOfDiskTransitionFrequency == zeroDuration {
obj.OutOfDiskTransitionFrequency = metav1.Duration{Duration: 5 * time.Minute}
}
if string(obj.HairpinMode) == "" {
obj.HairpinMode = PromiscuousBridge
}

7
pkg/apis/componentconfig/v1alpha1/types.go
View File

@ -386,10 +386,6 @@ type KubeletConfiguration struct {
// image garbage collection is never run. Lowest disk usage to garbage
// collect to. The percent is calculated as this field value out of 100.
ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent"`
// lowDiskSpaceThresholdMB is the absolute free disk space, in MB, to
// maintain. When disk space falls below this threshold, new pods would
// be rejected.
LowDiskSpaceThresholdMB int32 `json:"lowDiskSpaceThresholdMB"`
// How frequently to calculate and cache volume disk usage for all pods
VolumeStatsAggPeriod metav1.Duration `json:"volumeStatsAggPeriod"`
// volumePluginDir is the full path of the directory in which to search
@ -483,9 +479,6 @@ type KubeletConfiguration struct {
// run docker daemon with version < 1.9 or an Aufs storage backend.
// Issue #10959 has more details.
SerializeImagePulls *bool `json:"serializeImagePulls"`
// outOfDiskTransitionFrequency is duration for which the kubelet has to
// wait before transitioning out of out-of-disk node condition status.
OutOfDiskTransitionFrequency metav1.Duration `json:"outOfDiskTransitionFrequency"`
// nodeLabels to add when registering the node in the cluster.
NodeLabels map[string]string `json:"nodeLabels"`
// nonMasqueradeCIDR configures masquerading: traffic to IPs outside this range will use IP masquerade.

2
pkg/kubelet/BUILD
View File

@ -12,7 +12,6 @@ go_library(
name = "go_default_library",
srcs = [
"active_deadline.go",
"disk_manager.go",
"doc.go",
"kubelet.go",
"kubelet_cadvisor.go",
@ -148,7 +147,6 @@ go_test(
name = "go_default_test",
srcs = [
"active_deadline_test.go",
"disk_manager_test.go",
"kubelet_cadvisor_test.go",
"kubelet_getters_test.go",
"kubelet_network_test.go",

138
pkg/kubelet/disk_manager.go
View File

@ -1,138 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"sync"
"time"
"github.com/golang/glog"
cadvisorapi "github.com/google/cadvisor/info/v2"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
)
// Manages policy for diskspace management for disks holding docker images and root fs.
// mb is used to easily convert an int to an mb
const mb = 1024 * 1024
// Implementation is thread-safe.
type diskSpaceManager interface {
// Checks the available disk space
IsRootDiskSpaceAvailable() (bool, error)
IsRuntimeDiskSpaceAvailable() (bool, error)
}
// DiskSpacePolicy defines the free disk for Docker and Root.
type DiskSpacePolicy struct {
// free disk space threshold for filesystem holding docker images.
DockerFreeDiskMB int
// free disk space threshold for root filesystem. Host volumes are created on root fs.
RootFreeDiskMB int
}
type fsInfo struct {
Usage int64
Capacity int64
Available int64
Timestamp time.Time
}
type realDiskSpaceManager struct {
cadvisor cadvisor.Interface
cachedInfo map[string]fsInfo // cache of filesystem info.
lock sync.Mutex // protecting cachedInfo.
policy DiskSpacePolicy // thresholds. Set at creation time.
}
func (dm *realDiskSpaceManager) getFsInfo(fsType string, f func() (cadvisorapi.FsInfo, error)) (fsInfo, error) {
dm.lock.Lock()
defer dm.lock.Unlock()
fsi := fsInfo{}
if info, ok := dm.cachedInfo[fsType]; ok {
timeLimit := time.Now().Add(-2 * time.Second)
if info.Timestamp.After(timeLimit) {
fsi = info
}
}
if fsi.Timestamp.IsZero() {
fs, err := f()
if err != nil {
return fsInfo{}, err
}
fsi.Timestamp = time.Now()
fsi.Usage = int64(fs.Usage)
fsi.Capacity = int64(fs.Capacity)
fsi.Available = int64(fs.Available)
dm.cachedInfo[fsType] = fsi
}
return fsi, nil
}
func (dm *realDiskSpaceManager) IsRuntimeDiskSpaceAvailable() (bool, error) {
return dm.isSpaceAvailable("runtime", dm.policy.DockerFreeDiskMB, dm.cadvisor.ImagesFsInfo)
}
func (dm *realDiskSpaceManager) IsRootDiskSpaceAvailable() (bool, error) {
return dm.isSpaceAvailable("root", dm.policy.RootFreeDiskMB, dm.cadvisor.RootFsInfo)
}
func (dm *realDiskSpaceManager) isSpaceAvailable(fsType string, threshold int, f func() (cadvisorapi.FsInfo, error)) (bool, error) {
fsInfo, err := dm.getFsInfo(fsType, f)
if err != nil {
return true, fmt.Errorf("failed to get fs info for %q: %v", fsType, err)
}
if fsInfo.Capacity == 0 {
return true, fmt.Errorf("could not determine capacity for %q fs. Info: %+v", fsType, fsInfo)
}
if fsInfo.Available < 0 {
return true, fmt.Errorf("wrong available space for %q: %+v", fsType, fsInfo)
}
if fsInfo.Available < int64(threshold)*mb {
glog.Infof("Running out of space on disk for %q: available %d MB, threshold %d MB", fsType, fsInfo.Available/mb, threshold)
return false, nil
}
return true, nil
}
func validatePolicy(policy DiskSpacePolicy) error {
if policy.DockerFreeDiskMB < 0 {
return fmt.Errorf("free disk space should be non-negative; invalid value %d for docker disk space threshold", policy.DockerFreeDiskMB)
}
if policy.RootFreeDiskMB < 0 {
return fmt.Errorf("free disk space should be non-negative; invalid value %d for root disk space threshold", policy.RootFreeDiskMB)
}
return nil
}
func newDiskSpaceManager(cadvisorInterface cadvisor.Interface, policy DiskSpacePolicy) (diskSpaceManager, error) {
// validate policy
err := validatePolicy(policy)
if err != nil {
return nil, err
}
dm := &realDiskSpaceManager{
cadvisor: cadvisorInterface,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
return dm, nil
}

295
pkg/kubelet/disk_manager_test.go
View File

@ -1,295 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kubelet
import (
"fmt"
"testing"
cadvisorapi "github.com/google/cadvisor/info/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
)
func testPolicy() DiskSpacePolicy {
return DiskSpacePolicy{
DockerFreeDiskMB: 250,
RootFreeDiskMB: 250,
}
}
func setUp(t *testing.T) (*assert.Assertions, DiskSpacePolicy, *cadvisortest.Mock) {
assert := assert.New(t)
policy := testPolicy()
c := new(cadvisortest.Mock)
return assert, policy, c
}
func TestValidPolicy(t *testing.T) {
assert, policy, c := setUp(t)
_, err := newDiskSpaceManager(c, policy)
assert.NoError(err)
policy = testPolicy()
policy.DockerFreeDiskMB = -1
_, err = newDiskSpaceManager(c, policy)
assert.Error(err)
policy = testPolicy()
policy.RootFreeDiskMB = -1
_, err = newDiskSpaceManager(c, policy)
assert.Error(err)
}
func TestSpaceAvailable(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
}, nil)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
}, nil)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestIsRuntimeDiskSpaceAvailableWithSpace verifies IsRuntimeDiskSpaceAvailable results when
// space is available.
func TestIsRuntimeDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)
// 500MB available
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9500 * mb,
Capacity: 10000 * mb,
Available: 500 * mb,
}, nil)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
}
// TestIsRuntimeDiskSpaceAvailableWithoutSpace verifies IsRuntimeDiskSpaceAvailable results when
// space is not available.
func TestIsRuntimeDiskSpaceAvailableWithoutSpace(t *testing.T) {
// 1MB available
assert, policy, mockCadvisor := setUp(t)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 999 * mb,
Capacity: 1000 * mb,
Available: 1 * mb,
}, nil)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
require.NoError(t, err)
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestIsRootDiskSpaceAvailableWithSpace verifies IsRootDiskSpaceAvailable results when
// space is available.
func TestIsRootDiskSpaceAvailableWithSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
// 999MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 1 * mb,
Capacity: 1000 * mb,
Available: 999 * mb,
}, nil)
ok, err := dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
}
// TestIsRootDiskSpaceAvailableWithoutSpace verifies IsRootDiskSpaceAvailable results when
// space is not available.
func TestIsRootDiskSpaceAvailableWithoutSpace(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
// 9MB available
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 990 * mb,
Capacity: 1000 * mb,
Available: 9 * mb,
}, nil)
ok, err := dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.False(ok)
}
// TestCache verifies that caching works properly with DiskSpaceAvailable calls
func TestCache(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 300 * mb,
}, nil).Once()
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 500 * mb,
Capacity: 1000 * mb,
Available: 500 * mb,
}, nil).Once()
// Initial calls which should be recorded in mockCadvisor
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
// Get the current count of calls to mockCadvisor
cadvisorCallCount := len(mockCadvisor.Calls)
// Checking for space again shouldn't need to mock as cache would serve it.
ok, err = dm.IsRuntimeDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.NoError(err)
assert.True(ok)
// Ensure no more calls to the mockCadvisor occurred
assert.Equal(cadvisorCallCount, len(mockCadvisor.Calls))
}
// TestFsInfoError verifies errors are returned by DiskSpaceAvailable calls
// when FsInfo calls return an error
func TestFsInfoError(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
policy.RootFreeDiskMB = 10
dm, err := newDiskSpaceManager(mockCadvisor, policy)
assert.NoError(err)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("can't find fs"))
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{}, fmt.Errorf("EBUSY"))
ok, err := dm.IsRuntimeDiskSpaceAvailable()
assert.Error(err)
assert.True(ok)
ok, err = dm.IsRootDiskSpaceAvailable()
assert.Error(err)
assert.True(ok)
}
// Test_getFSInfo verifies multiple possible cases for getFsInfo.
func Test_getFsInfo(t *testing.T) {
assert, policy, mockCadvisor := setUp(t)
// Sunny day case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 10 * mb,
Capacity: 100 * mb,
Available: 90 * mb,
}, nil).Once()
dm := &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err := dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.NoError(err)
// Threshold case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 100 * mb,
Available: 9 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.False(available)
assert.NoError(err)
// Frozen case
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Available: 500 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.NoError(err)
// Capacity error case
mockCadvisor = new(cadvisortest.Mock)
mockCadvisor.On("RootFsInfo").Return(cadvisorapi.FsInfo{
Usage: 9 * mb,
Capacity: 0,
Available: 500 * mb,
}, nil).Once()
dm = &realDiskSpaceManager{
cadvisor: mockCadvisor,
policy: policy,
cachedInfo: map[string]fsInfo{},
}
available, err = dm.isSpaceAvailable("root", 10, dm.cadvisor.RootFsInfo)
assert.True(available)
assert.Error(err)
assert.Contains(fmt.Sprintf("%s", err), "could not determine capacity")
// Available error case skipped as v2.FSInfo uses uint64 and this
// can not be less than 0
}

1
pkg/kubelet/events/event.go
View File

@ -53,7 +53,6 @@ const (
NodeSelectorMismatching = "NodeSelectorMismatching"
InsufficientFreeCPU = "InsufficientFreeCPU"
InsufficientFreeMemory = "InsufficientFreeMemory"
OutOfDisk = "OutOfDisk"
HostNetworkNotSupported = "HostNetworkNotSupported"
UndefinedShaper = "NilShaper"
NodeRebooted = "Rebooted"

4
pkg/kubelet/eviction/helpers_test.go
View File

@ -1363,12 +1363,12 @@ func TestHasNodeConditions(t *testing.T) {
result bool
}{
"has-condition": {
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk, v1.NodeMemoryPressure},
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure, v1.NodeMemoryPressure},
item: v1.NodeMemoryPressure,
result: true,
},
"does-not-have-condition": {
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeOutOfDisk},
inputs: []v1.NodeConditionType{v1.NodeReady, v1.NodeDiskPressure},
item: v1.NodeMemoryPressure,
result: false,
},

47
pkg/kubelet/kubelet.go
View File

@ -368,11 +368,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
LowThresholdPercent: int(kubeCfg.ImageGCLowThresholdPercent),
}
diskSpacePolicy := DiskSpacePolicy{
DockerFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
RootFreeDiskMB: int(kubeCfg.LowDiskSpaceThresholdMB),
}
enforceNodeAllocatable := kubeCfg.EnforceNodeAllocatable
if kubeCfg.ExperimentalNodeAllocatableIgnoreEvictionThreshold {
// Do not provide kubeCfg.EnforceNodeAllocatable to eviction threshold parsing if we are not enforcing Evictions
@ -416,10 +411,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
Namespace: "",
}
diskSpaceManager, err := newDiskSpaceManager(kubeDeps.CAdvisorInterface, diskSpacePolicy)
if err != nil {
return nil, fmt.Errorf("failed to initialize disk manager: %v", err)
}
containerRefManager := kubecontainer.NewRefManager()
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
@ -453,7 +444,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
streamingConnectionIdleTimeout: kubeCfg.StreamingConnectionIdleTimeout.Duration,
recorder: kubeDeps.Recorder,
cadvisor: kubeDeps.CAdvisorInterface,
diskSpaceManager: diskSpaceManager,
cloud: kubeDeps.Cloud,
autoDetectCloudProvider: (componentconfigv1alpha1.AutoDetectCloudProvider == kubeCfg.CloudProvider),
externalCloudProvider: cloudprovider.IsExternal(kubeCfg.CloudProvider),
@ -475,7 +465,6 @@ func NewMainKubelet(kubeCfg *componentconfig.KubeletConfiguration, kubeDeps *Dep
containerManager: kubeDeps.ContainerManager,
nodeIP: net.ParseIP(nodeIP),
clock: clock.RealClock{},
outOfDiskTransitionFrequency: kubeCfg.OutOfDiskTransitionFrequency.Duration,
enableControllerAttachDetach: kubeCfg.EnableControllerAttachDetach,
iptClient: utilipt.New(utilexec.New(), utildbus.New(), utilipt.ProtocolIpv4),
makeIPTablesUtilChains: kubeCfg.MakeIPTablesUtilChains,
@ -924,9 +913,6 @@ type Kubelet struct {
// Manager for image garbage collection.
imageManager images.ImageGCManager
// Diskspace manager.
diskSpaceManager diskSpaceManager
// Secret manager.
secretManager secret.Manager
@ -1044,12 +1030,6 @@ type Kubelet struct {
// easy to test the code.
clock clock.Clock
// outOfDiskTransitionFrequency specifies the amount of time the kubelet has to be actually
// not out of disk before it can transition the node condition status from out-of-disk to
// not-out-of-disk. This prevents a pod that causes out-of-disk condition from repeatedly
// getting rescheduled onto the node.
outOfDiskTransitionFrequency time.Duration
// handlers called during the tryUpdateNodeStatus cycle
setNodeStatusFuncs []func(*v1.Node) error
@ -1650,27 +1630,6 @@ func (kl *Kubelet) deletePod(pod *v1.Pod) error {
return nil
}
// isOutOfDisk detects if pods can't fit due to lack of disk space.
func (kl *Kubelet) isOutOfDisk() bool {
// Check disk space once globally and reject or accept all new pods.
withinBounds, err := kl.diskSpaceManager.IsRuntimeDiskSpaceAvailable()
// Assume enough space in case of errors.
if err != nil {
glog.Errorf("Failed to check if disk space is available for the runtime: %v", err)
} else if !withinBounds {
return true
}
withinBounds, err = kl.diskSpaceManager.IsRootDiskSpaceAvailable()
// Assume enough space in case of errors.
if err != nil {
glog.Errorf("Failed to check if disk space is available on the root partition: %v", err)
} else if !withinBounds {
return true
}
return false
}
// rejectPod records an event about the pod with the given reason and message,
// and updates the pod to the failed phase in the status manage.
func (kl *Kubelet) rejectPod(pod *v1.Pod, reason, message string) {
@ -1697,12 +1656,6 @@ func (kl *Kubelet) canAdmitPod(pods []*v1.Pod, pod *v1.Pod) (bool, string, strin
return false, result.Reason, result.Message
}
}
// TODO: When disk space scheduling is implemented (#11976), remove the out-of-disk check here and
// add the disk space predicate to predicates.GeneralPredicates.
if kl.isOutOfDisk() {
glog.Warningf("Failed to admit pod %v - %s", format.Pod(pod), "predicate fails due to OutOfDisk")
return false, "OutOfDisk", "cannot be started due to lack of disk space."
}
return true, "", ""
}

65
pkg/kubelet/kubelet_node_status.go
View File

@ -852,70 +852,6 @@ func (kl *Kubelet) setNodeDiskPressureCondition(node *v1.Node) {
}
}
// Set OODCondition for the node.
func (kl *Kubelet) setNodeOODCondition(node *v1.Node) {
currentTime := metav1.NewTime(kl.clock.Now())
var nodeOODCondition *v1.NodeCondition
// Check if NodeOutOfDisk condition already exists and if it does, just pick it up for update.
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Type == v1.NodeOutOfDisk {
nodeOODCondition = &node.Status.Conditions[i]
}
}
newOODCondition := false
// If the NodeOutOfDisk condition doesn't exist, create one.
if nodeOODCondition == nil {
nodeOODCondition = &v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionUnknown,
}
// nodeOODCondition cannot be appended to node.Status.Conditions here because it gets
// copied to the slice. So if we append nodeOODCondition to the slice here none of the
// updates we make to nodeOODCondition below are reflected in the slice.
newOODCondition = true
}
// Update the heartbeat time irrespective of all the conditions.
nodeOODCondition.LastHeartbeatTime = currentTime
// Note: The conditions below take care of the case when a new NodeOutOfDisk condition is
// created and as well as the case when the condition already exists. When a new condition
// is created its status is set to v1.ConditionUnknown which matches either
// nodeOODCondition.Status != v1.ConditionTrue or
// nodeOODCondition.Status != v1.ConditionFalse in the conditions below depending on whether
// the kubelet is out of disk or not.
if kl.isOutOfDisk() {
if nodeOODCondition.Status != v1.ConditionTrue {
nodeOODCondition.Status = v1.ConditionTrue
nodeOODCondition.Reason = "KubeletOutOfDisk"
nodeOODCondition.Message = "out of disk space"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeOutOfDisk")
}
} else {
if nodeOODCondition.Status != v1.ConditionFalse {
// Update the out of disk condition when the condition status is unknown even if we
// are within the outOfDiskTransitionFrequency duration. We do this to set the
// condition status correctly at kubelet startup.
if nodeOODCondition.Status == v1.ConditionUnknown || kl.clock.Since(nodeOODCondition.LastTransitionTime.Time) >= kl.outOfDiskTransitionFrequency {
nodeOODCondition.Status = v1.ConditionFalse
nodeOODCondition.Reason = "KubeletHasSufficientDisk"
nodeOODCondition.Message = "kubelet has sufficient disk space available"
nodeOODCondition.LastTransitionTime = currentTime
kl.recordNodeStatusEvent(v1.EventTypeNormal, "NodeHasSufficientDisk")
} else {
glog.Infof("Node condition status for OutOfDisk is false, but last transition time is less than %s", kl.outOfDiskTransitionFrequency)
}
}
}
if newOODCondition {
node.Status.Conditions = append(node.Status.Conditions, *nodeOODCondition)
}
}
// Maintains Node.Spec.Unschedulable value from previous run of tryUpdateNodeStatus()
// TODO: why is this a package var?
var oldNodeUnschedulable bool
@ -966,7 +902,6 @@ func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error {
return []func(*v1.Node) error{
kl.setNodeAddress,
withoutError(kl.setNodeStatusInfo),
withoutError(kl.setNodeOODCondition),
withoutError(kl.setNodeMemoryPressureCondition),
withoutError(kl.setNodeDiskPressureCondition),
withoutError(kl.setNodeReadyCondition),

290
pkg/kubelet/kubelet_node_status_test.go
View File

@ -163,22 +163,11 @@ func TestUpdateNewNodeStatus(t *testing.T) {
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
// Make kubelet report that it has sufficient disk space.
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
expectedNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionFalse,
Reason: "KubeletHasSufficientDisk",
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
LastHeartbeatTime: metav1.Time{},
LastTransitionTime: metav1.Time{},
},
{
Type: v1.NodeMemoryPressure,
Status: v1.ConditionFalse,
@ -256,80 +245,6 @@ func TestUpdateNewNodeStatus(t *testing.T) {
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
}
func TestUpdateNewNodeOutOfDiskStatusWithTransitionFrequency(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()
kubelet := testKubelet.kubelet
kubelet.containerManager = &localCM{
ContainerManager: cm.NewStubContainerManager(),
allocatable: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(200, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
},
capacity: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
},
}
kubeClient := testKubelet.fakeKubeClient
existingNode := v1.Node{ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname}}
kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
machineInfo := &cadvisorapi.MachineInfo{
MachineID: "123",
SystemUUID: "abc",
BootID: "1b3",
NumCores: 2,
MemoryCapacity: 1024,
}
mockCadvisor := testKubelet.fakeCadvisor
mockCadvisor.On("Start").Return(nil)
mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
versionInfo := &cadvisorapi.VersionInfo{
KernelVersion: "3.16.0-0.bpo.4-amd64",
ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
// Make Kubelet report that it has sufficient disk space.
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100)
require.NoError(t, err, "update the disk space manager")
kubelet.outOfDiskTransitionFrequency = 10 * time.Second
expectedNodeOutOfDiskCondition := v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionFalse,
Reason: "KubeletHasSufficientDisk",
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
LastHeartbeatTime: metav1.Time{},
LastTransitionTime: metav1.Time{},
}
kubelet.updateRuntimeUp()
assert.NoError(t, kubelet.updateNodeStatus())
actions := kubeClient.Actions()
require.Len(t, actions, 2)
require.True(t, actions[1].Matches("patch", "nodes"))
require.Equal(t, "status", actions[1].GetSubresource())
updatedNode, err := applyNodeStatusPatch(&existingNode, actions[1].(core.PatchActionImpl).GetPatch())
assert.NoError(t, err, "apply the node status patch")
var oodCondition v1.NodeCondition
for i, cond := range updatedNode.Status.Conditions {
assert.False(t, cond.LastHeartbeatTime.IsZero(), "LastHeartbeatTime for %v condition is zero", cond.Type)
assert.False(t, cond.LastTransitionTime.IsZero(), "LastTransitionTime for %v condition is zero", cond.Type)
updatedNode.Status.Conditions[i].LastHeartbeatTime = metav1.Time{}
updatedNode.Status.Conditions[i].LastTransitionTime = metav1.Time{}
if cond.Type == v1.NodeOutOfDisk {
oodCondition = updatedNode.Status.Conditions[i]
}
}
assert.EqualValues(t, expectedNodeOutOfDiskCondition, oodCondition)
}
func TestUpdateExistingNodeStatus(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()
@ -352,14 +267,6 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionTrue,
Reason: "KubeletOutOfDisk",
Message: "out of disk space",
LastHeartbeatTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
LastTransitionTime: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
},
{
Type: v1.NodeMemoryPressure,
Status: v1.ConditionFalse,
@ -414,23 +321,11 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
// Make kubelet report that it is out of disk space.
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 50, 50, 100, 100)
require.NoError(t, err, "update the disk space manager")
expectedNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionTrue,
Reason: "KubeletOutOfDisk",
Message: "out of disk space",
LastHeartbeatTime: metav1.Time{}, // placeholder
LastTransitionTime: metav1.Time{}, // placeholder
},
{
Type: v1.NodeMemoryPressure,
Status: v1.ConditionFalse,
@ -524,167 +419,6 @@ func TestUpdateExistingNodeStatus(t *testing.T) {
assert.True(t, apiequality.Semantic.DeepEqual(expectedNode, updatedNode), "%s", diff.ObjectDiff(expectedNode, updatedNode))
}
func TestUpdateExistingNodeOutOfDiskStatusWithTransitionFrequency(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()
kubelet := testKubelet.kubelet
kubelet.containerManager = &localCM{
ContainerManager: cm.NewStubContainerManager(),
allocatable: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(200, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(100E6, resource.BinarySI),
},
capacity: v1.ResourceList{
v1.ResourceCPU: *resource.NewMilliQuantity(2000, resource.DecimalSI),
v1.ResourceMemory: *resource.NewQuantity(20E9, resource.BinarySI),
},
}
clock := testKubelet.fakeClock
// Do not set nano second, because apiserver function doesn't support nano second. (Only support
// RFC3339).
clock.SetTime(time.Unix(123456, 0))
kubeClient := testKubelet.fakeKubeClient
existingNode := v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: v1.NodeReady,
Status: v1.ConditionTrue,
Reason: "KubeletReady",
Message: fmt.Sprintf("kubelet is posting ready status"),
LastHeartbeatTime: metav1.NewTime(clock.Now()),
LastTransitionTime: metav1.NewTime(clock.Now()),
},
{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionTrue,
Reason: "KubeletOutOfDisk",
Message: "out of disk space",
LastHeartbeatTime: metav1.NewTime(clock.Now()),
LastTransitionTime: metav1.NewTime(clock.Now()),
},
},
},
}
kubeClient.ReactionChain = fake.NewSimpleClientset(&v1.NodeList{Items: []v1.Node{existingNode}}).ReactionChain
mockCadvisor := testKubelet.fakeCadvisor
machineInfo := &cadvisorapi.MachineInfo{
MachineID: "123",
SystemUUID: "abc",
BootID: "1b3",
NumCores: 2,
MemoryCapacity: 1024,
}
fsInfo := cadvisorapiv2.FsInfo{
Device: "123",
}
mockCadvisor.On("Start").Return(nil)
mockCadvisor.On("MachineInfo").Return(machineInfo, nil)
mockCadvisor.On("ImagesFsInfo").Return(fsInfo, nil)
mockCadvisor.On("RootFsInfo").Return(fsInfo, nil)
versionInfo := &cadvisorapi.VersionInfo{
KernelVersion: "3.16.0-0.bpo.4-amd64",
ContainerOsVersion: "Debian GNU/Linux 7 (wheezy)",
DockerVersion: "1.5.0",
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
kubelet.outOfDiskTransitionFrequency = 5 * time.Second
ood := v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionTrue,
Reason: "KubeletOutOfDisk",
Message: "out of disk space",
LastHeartbeatTime: metav1.NewTime(clock.Now()), // placeholder
LastTransitionTime: metav1.NewTime(clock.Now()), // placeholder
}
noOod := v1.NodeCondition{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionFalse,
Reason: "KubeletHasSufficientDisk",
Message: fmt.Sprintf("kubelet has sufficient disk space available"),
LastHeartbeatTime: metav1.NewTime(clock.Now()), // placeholder
LastTransitionTime: metav1.NewTime(clock.Now()), // placeholder
}
testCases := []struct {
rootFsAvail uint64
dockerFsAvail uint64
expected v1.NodeCondition
}{
{
// NodeOutOfDisk==false
rootFsAvail: 200,
dockerFsAvail: 200,
expected: ood,
},
{
// NodeOutOfDisk==true
rootFsAvail: 50,
dockerFsAvail: 200,
expected: ood,
},
{
// NodeOutOfDisk==false
rootFsAvail: 200,
dockerFsAvail: 200,
expected: ood,
},
{
// NodeOutOfDisk==true
rootFsAvail: 200,
dockerFsAvail: 50,
expected: ood,
},
{
// NodeOutOfDisk==false
rootFsAvail: 200,
dockerFsAvail: 200,
expected: noOod,
},
}
kubelet.updateRuntimeUp()
for tcIdx, tc := range testCases {
// Step by a second
clock.Step(1 * time.Second)
// Setup expected times.
tc.expected.LastHeartbeatTime = metav1.NewTime(clock.Now())
// In the last case, there should be a status transition for NodeOutOfDisk
if tcIdx == len(testCases)-1 {
tc.expected.LastTransitionTime = metav1.NewTime(clock.Now())
}
// Make kubelet report that it has sufficient disk space
err := updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, tc.rootFsAvail, tc.dockerFsAvail, 100, 100)
require.NoError(t, err, "can't update disk space manager")
assert.NoError(t, kubelet.updateNodeStatus())
actions := kubeClient.Actions()
assert.Len(t, actions, 2, "test [%d]", tcIdx)
assert.IsType(t, core.PatchActionImpl{}, actions[1])
patchAction := actions[1].(core.PatchActionImpl)
updatedNode, err := applyNodeStatusPatch(&existingNode, patchAction.GetPatch())
require.NoError(t, err, "can't apply node status patch")
kubeClient.ClearActions()
var oodCondition v1.NodeCondition
for i, cond := range updatedNode.Status.Conditions {
if cond.Type == v1.NodeOutOfDisk {
oodCondition = updatedNode.Status.Conditions[i]
}
}
assert.EqualValues(t, tc.expected, oodCondition)
}
}
func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
defer testKubelet.Cleanup()
@ -721,22 +455,11 @@ func TestUpdateNodeStatusWithRuntimeStateError(t *testing.T) {
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
// Make kubelet report that it has sufficient disk space.
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
expectedNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
Spec: v1.NodeSpec{},
Status: v1.NodeStatus{
Conditions: []v1.NodeCondition{
{
Type: v1.NodeOutOfDisk,
Status: v1.ConditionFalse,
Reason: "KubeletHasSufficientDisk",
Message: "kubelet has sufficient disk space available",
LastHeartbeatTime: metav1.Time{},
LastTransitionTime: metav1.Time{},
},
{
Type: v1.NodeMemoryPressure,
Status: v1.ConditionFalse,
@ -943,13 +666,13 @@ func TestRegisterWithApiServer(t *testing.T) {
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
mockCadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
Usage: 400,
Capacity: 1000,
Available: 600,
}, nil)
mockCadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Usage: 9,
Capacity: 10,
}, nil)
done := make(chan struct{})
@ -1178,9 +901,6 @@ func TestUpdateNewNodeStatusTooLargeReservation(t *testing.T) {
}
mockCadvisor.On("VersionInfo").Return(versionInfo, nil)
// Make kubelet report that it has sufficient disk space.
require.NoError(t, updateDiskSpacePolicy(kubelet, mockCadvisor, 500, 500, 200, 200, 100, 100))
expectedNode := &v1.Node{
ObjectMeta: metav1.ObjectMeta{Name: testKubeletHostname},
Spec: v1.NodeSpec{},

24
pkg/kubelet/kubelet_test.go
View File

@ -213,11 +213,6 @@ func newTestKubeletWithImageList(
kubelet.configMapManager = configMapManager
kubelet.podManager = kubepod.NewBasicPodManager(fakeMirrorClient, kubelet.secretManager, kubelet.configMapManager)
kubelet.statusManager = status.NewManager(fakeKubeClient, kubelet.podManager, &statustest.FakePodDeletionSafetyProvider{})
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, DiskSpacePolicy{})
if err != nil {
t.Fatalf("can't initialize disk space manager: %v", err)
}
kubelet.diskSpaceManager = diskSpaceManager
kubelet.containerRuntime = fakeRuntime
kubelet.runtimeCache = containertest.NewFakeRuntimeCache(kubelet.containerRuntime)
@ -737,25 +732,6 @@ func TestValidateContainerLogStatus(t *testing.T) {
}
}
// updateDiskSpacePolicy creates a new DiskSpaceManager with a new policy. This new manager along
// with the mock FsInfo values added to Cadvisor should make the kubelet report that it has
// sufficient disk space or it is out of disk, depending on the capacity, availability and
// threshold values.
func updateDiskSpacePolicy(kubelet *Kubelet, mockCadvisor *cadvisortest.Mock, rootCap, dockerCap, rootAvail, dockerAvail uint64, rootThreshold, dockerThreshold int) error {
dockerimagesFsInfo := cadvisorapiv2.FsInfo{Capacity: rootCap * mb, Available: rootAvail * mb}
rootFsInfo := cadvisorapiv2.FsInfo{Capacity: dockerCap * mb, Available: dockerAvail * mb}
mockCadvisor.On("ImagesFsInfo").Return(dockerimagesFsInfo, nil)
mockCadvisor.On("RootFsInfo").Return(rootFsInfo, nil)
dsp := DiskSpacePolicy{DockerFreeDiskMB: rootThreshold, RootFreeDiskMB: dockerThreshold}
diskSpaceManager, err := newDiskSpaceManager(mockCadvisor, dsp)
if err != nil {
return err
}
kubelet.diskSpaceManager = diskSpaceManager
return nil
}
func TestCreateMirrorPod(t *testing.T) {
for _, updateType := range []kubetypes.SyncPodType{kubetypes.SyncPodCreate, kubetypes.SyncPodUpdate} {
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)

12
pkg/kubelet/runonce_test.go
View File

@ -55,19 +55,18 @@ func TestRunOnce(t *testing.T) {
cadvisor := &cadvisortest.Mock{}
cadvisor.On("MachineInfo").Return(&cadvisorapi.MachineInfo{}, nil)
cadvisor.On("ImagesFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 400 * mb,
Capacity: 1000 * mb,
Available: 600 * mb,
Usage: 400,
Capacity: 1000,
Available: 600,
}, nil)
cadvisor.On("RootFsInfo").Return(cadvisorapiv2.FsInfo{
Usage: 9 * mb,
Capacity: 10 * mb,
Usage: 9,
Capacity: 10,
}, nil)
fakeSecretManager := secret.NewFakeManager()
fakeConfigMapManager := configmap.NewFakeManager()
podManager := kubepod.NewBasicPodManager(
podtest.NewFakeMirrorClient(), fakeSecretManager, fakeConfigMapManager)
diskSpaceManager, _ := newDiskSpaceManager(cadvisor, DiskSpacePolicy{})
fakeRuntime := &containertest.FakeRuntime{}
basePath, err := utiltesting.MkTmpdir("kubelet")
if err != nil {
@ -82,7 +81,6 @@ func TestRunOnce(t *testing.T) {
statusManager: status.NewManager(nil, podManager, &statustest.FakePodDeletionSafetyProvider{}),
podManager: podManager,
os: &containertest.FakeOS{},
diskSpaceManager: diskSpaceManager,
containerRuntime: fakeRuntime,
reasonCache: NewReasonCache(),
clock: clock.RealClock{},

2
pkg/kubemark/hollow_kubelet.go
View File

@ -134,14 +134,12 @@ func GetHollowKubeletConfig(
c.MinimumGCAge.Duration = 1 * time.Minute
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
c.SyncFrequency.Duration = 10 * time.Second
c.OutOfDiskTransitionFrequency.Duration = 5 * time.Minute
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
c.MaxPods = int32(maxPods)
c.PodsPerCore = int32(podsPerCore)
c.ClusterDNS = []string{}
c.ImageGCHighThresholdPercent = 90
c.ImageGCLowThresholdPercent = 80
c.LowDiskSpaceThresholdMB = 256
c.VolumeStatsAggPeriod.Duration = time.Minute
c.CgroupRoot = ""
c.ContainerRuntime = kubetypes.DockerContainerRuntime

4
pkg/printers/internalversion/printers_test.go
View File

@ -1588,7 +1588,7 @@ func TestPrintPod(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{Name: "test5"},
Spec: api.PodSpec{Containers: make([]api.Container, 2)},
Status: api.PodStatus{
Reason: "OutOfDisk",
Reason: "podReason",
Phase: "podPhase",
ContainerStatuses: []api.ContainerStatus{
{Ready: true, RestartCount: 3, State: api.ContainerState{Running: &api.ContainerStateRunning{}}},
@ -1596,7 +1596,7 @@ func TestPrintPod(t *testing.T) {
},
},
},
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "OutOfDisk", 6, "<unknown>"}}},
[]metav1alpha1.TableRow{{Cells: []interface{}{"test5", "1/2", "podReason", 6, "<unknown>"}}},
},
}

2
staging/src/k8s.io/api/core/v1/types.go
View File

@ -2624,7 +2624,7 @@ type PodStatus struct {
// +optional
Message string `json:"message,omitempty" protobuf:"bytes,3,opt,name=message"`
// A brief CamelCase message indicating details about why the pod is in this state.
// e.g. 'OutOfDisk'
// e.g. 'Evicted'
// +optional
Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`

2
test/e2e/BUILD
View File

@ -72,7 +72,6 @@ go_library(
"networking.go",
"networking_perf.go",
"no-snat.go",
"nodeoutofdisk.go",
"pod_gc.go",
"podpreset.go",
"pods.go",
@ -117,7 +116,6 @@ go_library(
"//test/images/net/nat:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
"//vendor/github.com/onsi/ginkgo/reporters:go_default_library",

269
test/e2e/nodeoutofdisk.go
View File

@ -1,269 +0,0 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
import (
"encoding/json"
"fmt"
"time"
cadvisorapi "github.com/google/cadvisor/info/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
const (
mb = 1024 * 1024
gb = 1024 * mb
// TODO(madhusudancs): find a way to query kubelet's disk space manager to obtain this value. 256MB
// is the default that is set today. This test might break if the default value changes. This value
// can be configured by setting the "low-diskspace-threshold-mb" flag while starting a kubelet.
// However, kubelets are started as part of the cluster start up, once, before any e2e test is run,
// and remain unchanged until all the tests are run and the cluster is brought down. Changing the
// flag value affects all the e2e tests. So we are hard-coding this value for now.
lowDiskSpaceThreshold uint64 = 256 * mb
nodeOODTimeOut = 5 * time.Minute
numNodeOODPods = 3
)
// Plan:
// 1. Fill disk space on all nodes except one. One node is left out so that we can schedule pods
// on that node. Arbitrarily choose that node to be node with index 0. This makes this a disruptive test.
// 2. Get the CPU capacity on unfilled node.
// 3. Divide the available CPU into one less than the number of pods we want to schedule. We want
// to schedule 3 pods, so divide CPU capacity by 2.
// 4. Request the divided CPU for each pod.
// 5. Observe that 2 of the pods schedule onto the node whose disk is not full, and the remaining
// pod stays pending and does not schedule onto the nodes whose disks are full nor the node
// with the other two pods, since there is not enough free CPU capacity there.
// 6. Recover disk space from one of the nodes whose disk space was previously filled. Arbritrarily
// choose that node to be node with index 1.
// 7. Observe that the pod in pending status schedules on that node.
//
// Flaky issue #20015. We have no clear path for how to test this functionality in a non-flaky way.
var _ = framework.KubeDescribe("NodeOutOfDisk [Serial] [Flaky] [Disruptive]", func() {
var c clientset.Interface
var unfilledNodeName, recoveredNodeName string
f := framework.NewDefaultFramework("node-outofdisk")
BeforeEach(func() {
c = f.ClientSet
framework.Skipf("test is broken. #40249")
nodelist := framework.GetReadySchedulableNodesOrDie(c)
// Skip this test on small clusters. No need to fail since it is not a use
// case that any cluster of small size needs to support.
framework.SkipUnlessNodeCountIsAtLeast(2)
unfilledNodeName = nodelist.Items[0].Name
for _, node := range nodelist.Items[1:] {
fillDiskSpace(c, &node)
}
})
AfterEach(func() {
nodelist := framework.GetReadySchedulableNodesOrDie(c)
Expect(len(nodelist.Items)).ToNot(BeZero())
for _, node := range nodelist.Items {
if unfilledNodeName == node.Name || recoveredNodeName == node.Name {
continue
}
recoverDiskSpace(c, &node)
}
})
It("runs out of disk space", func() {
unfilledNode, err := c.Core().Nodes().Get(unfilledNodeName, metav1.GetOptions{})
framework.ExpectNoError(err)
By(fmt.Sprintf("Calculating CPU availability on node %s", unfilledNode.Name))
milliCpu, err := availCpu(c, unfilledNode)
framework.ExpectNoError(err)
// Per pod CPU should be just enough to fit only (numNodeOODPods - 1) pods on the given
// node. We compute this value by dividing the available CPU capacity on the node by
// (numNodeOODPods - 1) and subtracting ϵ from it. We arbitrarily choose ϵ to be 1%
// of the available CPU per pod, i.e. 0.01 * milliCpu/(numNodeOODPods-1). Instead of
// subtracting 1% from the value, we directly use 0.99 as the multiplier.
podCPU := int64(float64(milliCpu/(numNodeOODPods-1)) * 0.99)
ns := f.Namespace.Name
podClient := c.Core().Pods(ns)
By("Creating pods and waiting for all but one pods to be scheduled")
for i := 0; i < numNodeOODPods-1; i++ {
name := fmt.Sprintf("pod-node-outofdisk-%d", i)
createOutOfDiskPod(c, ns, name, podCPU)
framework.ExpectNoError(f.WaitForPodRunning(name))
pod, err := podClient.Get(name, metav1.GetOptions{})
framework.ExpectNoError(err)
Expect(pod.Spec.NodeName).To(Equal(unfilledNodeName))
}
pendingPodName := fmt.Sprintf("pod-node-outofdisk-%d", numNodeOODPods-1)
createOutOfDiskPod(c, ns, pendingPodName, podCPU)
By(fmt.Sprintf("Finding a failed scheduler event for pod %s", pendingPodName))
wait.Poll(2*time.Second, 5*time.Minute, func() (bool, error) {
selector := fields.Set{
"involvedObject.kind": "Pod",
"involvedObject.name": pendingPodName,
"involvedObject.namespace": ns,
"source": v1.DefaultSchedulerName,
"reason": "FailedScheduling",
}.AsSelector().String()
options := metav1.ListOptions{FieldSelector: selector}
schedEvents, err := c.Core().Events(ns).List(options)
framework.ExpectNoError(err)
if len(schedEvents.Items) > 0 {
return true, nil
} else {
return false, nil
}
})
nodelist := framework.GetReadySchedulableNodesOrDie(c)
Expect(len(nodelist.Items)).To(BeNumerically(">", 1))
nodeToRecover := nodelist.Items[1]
Expect(nodeToRecover.Name).ToNot(Equal(unfilledNodeName))
recoverDiskSpace(c, &nodeToRecover)
recoveredNodeName = nodeToRecover.Name
By(fmt.Sprintf("Verifying that pod %s schedules on node %s", pendingPodName, recoveredNodeName))
framework.ExpectNoError(f.WaitForPodRunning(pendingPodName))
pendingPod, err := podClient.Get(pendingPodName, metav1.GetOptions{})
framework.ExpectNoError(err)
Expect(pendingPod.Spec.NodeName).To(Equal(recoveredNodeName))
})
})
// createOutOfDiskPod creates a pod in the given namespace with the requested amount of CPU.
func createOutOfDiskPod(c clientset.Interface, ns, name string, milliCPU int64) {
podClient := c.Core().Pods(ns)
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "pause",
Image: framework.GetPauseImageName(c),
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
// Request enough CPU to fit only two pods on a given node.
v1.ResourceCPU: *resource.NewMilliQuantity(milliCPU, resource.DecimalSI),
},
},
},
},
},
}
_, err := podClient.Create(pod)
framework.ExpectNoError(err)
}
// availCpu calculates the available CPU on a given node by subtracting the CPU requested by
// all the pods from the total available CPU capacity on the node.
func availCpu(c clientset.Interface, node *v1.Node) (int64, error) {
podClient := c.Core().Pods(metav1.NamespaceAll)
selector := fields.Set{"spec.nodeName": node.Name}.AsSelector().String()
options := metav1.ListOptions{FieldSelector: selector}
pods, err := podClient.List(options)
if err != nil {
return 0, fmt.Errorf("failed to retrieve all the pods on node %s: %v", node.Name, err)
}
avail := node.Status.Capacity.Cpu().MilliValue()
for _, pod := range pods.Items {
for _, cont := range pod.Spec.Containers {
avail -= cont.Resources.Requests.Cpu().MilliValue()
}
}
return avail, nil
}
// availSize returns the available disk space on a given node by querying node stats which
// is in turn obtained internally from cadvisor.
func availSize(c clientset.Interface, node *v1.Node) (uint64, error) {
statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name)
framework.Logf("Querying stats for node %s using url %s", node.Name, statsResource)
res, err := c.Core().RESTClient().Get().AbsPath(statsResource).Timeout(time.Minute).Do().Raw()
if err != nil {
return 0, fmt.Errorf("error querying cAdvisor API: %v", err)
}
ci := cadvisorapi.ContainerInfo{}
err = json.Unmarshal(res, &ci)
if err != nil {
return 0, fmt.Errorf("couldn't unmarshal container info: %v", err)
}
return ci.Stats[len(ci.Stats)-1].Filesystem[0].Available, nil
}
// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk
// space on the node is filled in such a way that the available space after filling the disk is just
// below the lowDiskSpaceThreshold mark.
func fillDiskSpace(c clientset.Interface, node *v1.Node) {
avail, err := availSize(c, node)
framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err)
fillSize := (avail - lowDiskSpaceThreshold + (100 * mb))
framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize))
cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize)
framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))
ood := framework.WaitForNodeToBe(c, node.Name, v1.NodeOutOfDisk, true, nodeOODTimeOut)
Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut)
avail, err = availSize(c, node)
framework.Logf("Node %s: disk space available %d bytes", node.Name, avail)
Expect(avail < lowDiskSpaceThreshold).To(BeTrue())
}
// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node.
func recoverDiskSpace(c clientset.Interface, node *v1.Node) {
By(fmt.Sprintf("Recovering disk space on node %s", node.Name))
cmd := "rm -f test.img"
framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node))
ood := framework.WaitForNodeToBe(c, node.Name, v1.NodeOutOfDisk, false, nodeOODTimeOut)
Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut)
}

1
test/test_owners.csv
View File

@ -336,7 +336,6 @@ NoExecuteTaintManager doesn't evict pod with tolerations from tainted nodes,free
NoExecuteTaintManager eventually evict pod with finite tolerations from tainted nodes,freehan,0,scheduling
NoExecuteTaintManager evicts pods from tainted nodes,freehan,0,scheduling
NoExecuteTaintManager removing taint cancels eviction,freehan,0,scheduling
NodeOutOfDisk runs out of disk space,vishh,0,node
NodeProblemDetector KernelMonitor should generate node condition and events for corresponding errors,Random-Liu,0,node
Nodes Resize should be able to add nodes,piosz,1,cluster-lifecycle
Nodes Resize should be able to delete nodes,zmerlynn,1,cluster-lifecycle

1 name owner auto-assigned sig
336 NoExecuteTaintManager eventually evict pod with finite tolerations from tainted nodes freehan 0 scheduling
337 NoExecuteTaintManager evicts pods from tainted nodes freehan 0 scheduling
338 NoExecuteTaintManager removing taint cancels eviction freehan 0 scheduling
NodeOutOfDisk runs out of disk space vishh 0 node
339 NodeProblemDetector KernelMonitor should generate node condition and events for corresponding errors Random-Liu 0 node
340 Nodes Resize should be able to add nodes piosz 1 cluster-lifecycle
341 Nodes Resize should be able to delete nodes zmerlynn 1 cluster-lifecycle

1
test/test_owners.json
View File

@ -83,7 +83,6 @@
"Logging",
"MemoryEviction",
"MirrorPod",
"NodeOutOfDisk",
"NodeProblemDetector",
"Opaque",
"Pod garbage",