mirror of https://github.com/k3s-io/k3s
Merge pull request #35136 from dashpole/cadvisor_root_path_configuration
Automatic merge from submit-queue Cadvisor root path configuration This solves #33444 and reverts PR #33520. This takes the root directory and passes it to cadvisor, which sets up the "nodefs" based on the provided path, rather than using "/" as before. This PR is pending on changes in cadvisor, and will not pass tests until those changes are merged, and cadvisor godeps are updated. ```release-note kubelet summary rootfs now refers to the filesystem that contains the Kubelet RootDirectory (var/lib/kubelet) instead of cadvisor's rootfs ( / ), since they may be different filesystems. ```pull/6/head
Kubernetes Submit Queue
GitHub
commit
793b000f90
|
|
@ -1118,203 +1118,203 @@
|
|||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/api",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/cache/memory",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/client/v2",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/collector",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/common",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/docker",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/libcontainer",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/raw",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/rkt",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/container/systemd",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/devicemapper",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/events",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/fs",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/healthz",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/http",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/http/mux",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/info/v1",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/info/v2",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/machine",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/manager",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/manager/watcher",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/manager/watcher/raw",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/manager/watcher/rkt",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/metrics",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/pages",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/pages/static",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/storage",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/summary",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/cloudinfo",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/cpuload",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/cpuload/netlink",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/docker",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/oomparser",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/sysfs",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/sysinfo",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/utils/tail",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/validate",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/cadvisor/version",
|
||||
"Comment": "v0.24.0-alpha1-39-ga9b1ab1",
|
||||
"Rev": "a9b1ab1dd8d811d419487a631c3361df582346d2"
|
||||
"Comment": "v0.24.0-alpha1-56-ga726d13",
|
||||
"Rev": "a726d13de8cb32860e73d72a78dc8e0124267709"
|
||||
},
|
||||
{
|
||||
"ImportPath": "github.com/google/certificate-transparency/go",
|
||||
|
|
|
|||
|
|
@ -163,12 +163,7 @@ HAIRPIN_MODE="${HAIRPIN_MODE:-promiscuous-bridge}" # promiscuous-bridge, hairpin
|
|||
E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
|
||||
|
||||
# Evict pods whenever compute resource availability on the nodes gets below a threshold.
|
||||
# TODO: Get rid of the conditionals once https://github.com/kubernetes/kubernetes/issues/33444 is resolved.
|
||||
if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
|
||||
else
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,imagefs.available<10%,imagefs.inodesFree<5%}"
|
||||
fi
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
|
||||
|
||||
# Optional: custom scheduling algorithm
|
||||
SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}"
|
||||
|
|
|
|||
|
|
@ -198,12 +198,7 @@ E2E_STORAGE_TEST_ENVIRONMENT=${KUBE_E2E_STORAGE_TEST_ENVIRONMENT:-false}
|
|||
PREPULL_E2E_IMAGES="${PREPULL_E2E_IMAGES:-true}"
|
||||
|
||||
# Evict pods whenever compute resource availability on the nodes gets below a threshold.
|
||||
# TODO: Get rid of the conditionals once https://github.com/kubernetes/kubernetes/issues/33444 is resolved.
|
||||
if [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
|
||||
else
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,imagefs.available<10%,imagefs.inodesFree<5%}"
|
||||
fi
|
||||
EVICTION_HARD="${EVICTION_HARD:-memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%}"
|
||||
|
||||
# Optional: custom scheduling algorithm
|
||||
SCHEDULING_ALGORITHM_PROVIDER="${SCHEDULING_ALGORITHM_PROVIDER:-}"
|
||||
|
|
|
|||
|
|
@ -413,7 +413,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
|
|||
}
|
||||
|
||||
if kubeDeps.CAdvisorInterface == nil {
|
||||
kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime)
|
||||
kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ import (
|
|||
|
||||
type cadvisorClient struct {
|
||||
runtime string
|
||||
rootPath string
|
||||
manager.Manager
|
||||
}
|
||||
|
||||
|
|
@ -93,7 +94,7 @@ func containerLabels(c *cadvisorapi.ContainerInfo) map[string]string {
|
|||
}
|
||||
|
||||
// New creates a cAdvisor and exports its API on the specified port if port > 0.
|
||||
func New(port uint, runtime string) (Interface, error) {
|
||||
func New(port uint, runtime string, rootPath string) (Interface, error) {
|
||||
sysFs, err := sysfs.NewRealSysFs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -107,6 +108,7 @@ func New(port uint, runtime string) (Interface, error) {
|
|||
|
||||
cadvisorClient := &cadvisorClient{
|
||||
runtime: runtime,
|
||||
rootPath: rootPath,
|
||||
Manager: m,
|
||||
}
|
||||
|
||||
|
|
@ -202,7 +204,7 @@ func (cc *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
|
|||
}
|
||||
|
||||
func (cc *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
|
||||
return cc.getFsInfo(cadvisorfs.LabelSystemRoot)
|
||||
return cc.GetDirFsInfo(cc.rootPath)
|
||||
}
|
||||
|
||||
func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error) {
|
||||
|
|
|
|||
|
|
@ -212,7 +212,7 @@ func RegisterNodeFlags() {
|
|||
flag.BoolVar(&TestContext.DisableKubenet, "disable-kubenet", false, "If true, start kubelet without kubenet. (default false)")
|
||||
// TODO: uncomment this when the flag is re-enabled in kubelet
|
||||
//flag.BoolVar(&TestContext.CgroupsPerQOS, "cgroups-per-qos", false, "Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created.")
|
||||
flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,imagefs.available<10%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
|
||||
flag.StringVar(&TestContext.EvictionHard, "eviction-hard", "memory.available<250Mi,nodefs.available<10%,nodefs.inodesFree<5%", "The hard eviction thresholds. If set, pods get evicted when the specified resources drop below the thresholds.")
|
||||
flag.StringVar(&TestContext.ManifestPath, "manifest-path", "", "The path to the static pod manifest file.")
|
||||
flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
|
||||
flag.StringVar(&TestContext.RuntimeIntegrationType, "runtime-integration-type", "", "Choose the integration path for the container runtime, mainly used for CRI validation.")
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ func containerRuntime() error {
|
|||
}
|
||||
|
||||
// Setup cadvisor to check the container environment
|
||||
c, err := cadvisor.New(0 /*don't start the http server*/, "docker")
|
||||
c, err := cadvisor.New(0 /*don't start the http server*/, "docker", "/var/lib/kubelet")
|
||||
if err != nil {
|
||||
return printError("Container Runtime Check: %s Could not start cadvisor %v", failed, err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
package common
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
|
@ -26,15 +27,20 @@ import (
|
|||
|
||||
type FsHandler interface {
|
||||
Start()
|
||||
Usage() (baseUsageBytes uint64, totalUsageBytes uint64)
|
||||
Usage() FsUsage
|
||||
Stop()
|
||||
}
|
||||
|
||||
type FsUsage struct {
|
||||
BaseUsageBytes uint64
|
||||
TotalUsageBytes uint64
|
||||
InodeUsage uint64
|
||||
}
|
||||
|
||||
type realFsHandler struct {
|
||||
sync.RWMutex
|
||||
lastUpdate time.Time
|
||||
usageBytes uint64
|
||||
baseUsageBytes uint64
|
||||
usage FsUsage
|
||||
period time.Duration
|
||||
minPeriod time.Duration
|
||||
rootfs string
|
||||
|
|
@ -45,18 +51,19 @@ type realFsHandler struct {
|
|||
}
|
||||
|
||||
const (
|
||||
longDu = time.Second
|
||||
duTimeout = time.Minute
|
||||
maxDuBackoffFactor = 20
|
||||
longOp = time.Second
|
||||
timeout = 2 * time.Minute
|
||||
maxBackoffFactor = 20
|
||||
)
|
||||
|
||||
const DefaultPeriod = time.Minute
|
||||
|
||||
var _ FsHandler = &realFsHandler{}
|
||||
|
||||
func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
|
||||
return &realFsHandler{
|
||||
lastUpdate: time.Time{},
|
||||
usageBytes: 0,
|
||||
baseUsageBytes: 0,
|
||||
usage: FsUsage{},
|
||||
period: period,
|
||||
minPeriod: period,
|
||||
rootfs: rootfs,
|
||||
|
|
@ -68,29 +75,37 @@ func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInf
|
|||
|
||||
func (fh *realFsHandler) update() error {
|
||||
var (
|
||||
baseUsage, extraDirUsage uint64
|
||||
err error
|
||||
baseUsage, extraDirUsage, inodeUsage uint64
|
||||
rootDiskErr, rootInodeErr, extraDiskErr error
|
||||
)
|
||||
// TODO(vishh): Add support for external mounts.
|
||||
if fh.rootfs != "" {
|
||||
baseUsage, err = fh.fsInfo.GetDirUsage(fh.rootfs, duTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
baseUsage, rootDiskErr = fh.fsInfo.GetDirDiskUsage(fh.rootfs, timeout)
|
||||
inodeUsage, rootInodeErr = fh.fsInfo.GetDirInodeUsage(fh.rootfs, timeout)
|
||||
}
|
||||
|
||||
if fh.extraDir != "" {
|
||||
extraDirUsage, err = fh.fsInfo.GetDirUsage(fh.extraDir, duTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extraDirUsage, extraDiskErr = fh.fsInfo.GetDirDiskUsage(fh.extraDir, timeout)
|
||||
}
|
||||
|
||||
// Wait to handle errors until after all operartions are run.
|
||||
// An error in one will not cause an early return, skipping others
|
||||
fh.Lock()
|
||||
defer fh.Unlock()
|
||||
fh.lastUpdate = time.Now()
|
||||
fh.usageBytes = baseUsage + extraDirUsage
|
||||
fh.baseUsageBytes = baseUsage
|
||||
if rootDiskErr == nil && fh.rootfs != "" {
|
||||
fh.usage.InodeUsage = inodeUsage
|
||||
}
|
||||
if rootInodeErr == nil && fh.rootfs != "" {
|
||||
fh.usage.TotalUsageBytes = baseUsage + extraDirUsage
|
||||
}
|
||||
if extraDiskErr == nil && fh.extraDir != "" {
|
||||
fh.usage.BaseUsageBytes = baseUsage
|
||||
}
|
||||
// Combine errors into a single error to return
|
||||
if rootDiskErr != nil || rootInodeErr != nil || extraDiskErr != nil {
|
||||
return fmt.Errorf("rootDiskErr: %v, rootInodeErr: %v, extraDiskErr: %v", rootDiskErr, rootInodeErr, extraDiskErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -105,15 +120,15 @@ func (fh *realFsHandler) trackUsage() {
|
|||
if err := fh.update(); err != nil {
|
||||
glog.Errorf("failed to collect filesystem stats - %v", err)
|
||||
fh.period = fh.period * 2
|
||||
if fh.period > maxDuBackoffFactor*fh.minPeriod {
|
||||
fh.period = maxDuBackoffFactor * fh.minPeriod
|
||||
if fh.period > maxBackoffFactor*fh.minPeriod {
|
||||
fh.period = maxBackoffFactor * fh.minPeriod
|
||||
}
|
||||
} else {
|
||||
fh.period = fh.minPeriod
|
||||
}
|
||||
duration := time.Since(start)
|
||||
if duration > longDu {
|
||||
glog.V(2).Infof("`du` on following dirs took %v: %v", duration, []string{fh.rootfs, fh.extraDir})
|
||||
if duration > longOp {
|
||||
glog.V(2).Infof("du and find on following dirs took %v: %v", duration, []string{fh.rootfs, fh.extraDir})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -127,8 +142,8 @@ func (fh *realFsHandler) Stop() {
|
|||
close(fh.stopChan)
|
||||
}
|
||||
|
||||
func (fh *realFsHandler) Usage() (baseUsageBytes, totalUsageBytes uint64) {
|
||||
func (fh *realFsHandler) Usage() FsUsage {
|
||||
fh.RLock()
|
||||
defer fh.RUnlock()
|
||||
return fh.baseUsageBytes, fh.usageBytes
|
||||
return fh.usage
|
||||
}
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ func newDockerContainerHandler(
|
|||
|
||||
if !ignoreMetrics.Has(container.DiskUsageMetrics) {
|
||||
handler.fsHandler = &dockerFsHandler{
|
||||
fsHandler: common.NewFsHandler(time.Minute, rootfsStorageDir, otherStorageDir, fsInfo),
|
||||
fsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo),
|
||||
thinPoolWatcher: thinPoolWatcher,
|
||||
deviceID: handler.deviceID,
|
||||
}
|
||||
|
|
@ -283,8 +283,8 @@ func (h *dockerFsHandler) Stop() {
|
|||
h.fsHandler.Stop()
|
||||
}
|
||||
|
||||
func (h *dockerFsHandler) Usage() (uint64, uint64) {
|
||||
baseUsage, usage := h.fsHandler.Usage()
|
||||
func (h *dockerFsHandler) Usage() common.FsUsage {
|
||||
usage := h.fsHandler.Usage()
|
||||
|
||||
// When devicemapper is the storage driver, the base usage of the container comes from the thin pool.
|
||||
// We still need the result of the fsHandler for any extra storage associated with the container.
|
||||
|
|
@ -299,12 +299,12 @@ func (h *dockerFsHandler) Usage() (uint64, uint64) {
|
|||
// had at least 1 refresh and we still can't find the device.
|
||||
glog.V(5).Infof("unable to get fs usage from thin pool for device %s: %v", h.deviceID, err)
|
||||
} else {
|
||||
baseUsage = thinPoolUsage
|
||||
usage += thinPoolUsage
|
||||
usage.BaseUsageBytes = thinPoolUsage
|
||||
usage.TotalUsageBytes += thinPoolUsage
|
||||
}
|
||||
}
|
||||
|
||||
return baseUsage, usage
|
||||
return usage
|
||||
}
|
||||
|
||||
func (self *dockerContainerHandler) Start() {
|
||||
|
|
@ -387,7 +387,10 @@ func (self *dockerContainerHandler) getFsStats(stats *info.ContainerStats) error
|
|||
}
|
||||
|
||||
fsStat := info.FsStats{Device: device, Type: fsType, Limit: limit}
|
||||
fsStat.BaseUsage, fsStat.Usage = self.fsHandler.Usage()
|
||||
usage := self.fsHandler.Usage()
|
||||
fsStat.BaseUsage = usage.BaseUsageBytes
|
||||
fsStat.Usage = usage.TotalUsageBytes
|
||||
fsStat.Inodes = usage.InodeUsage
|
||||
|
||||
stats.Filesystem = append(stats.Filesystem, fsStat)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ package rkt
|
|||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
rktapi "github.com/coreos/rkt/api/v1alpha"
|
||||
"github.com/google/cadvisor/container"
|
||||
|
|
@ -150,7 +149,7 @@ func newRktContainerHandler(name string, rktClient rktapi.PublicAPIClient, rktPa
|
|||
}
|
||||
|
||||
if !ignoreMetrics.Has(container.DiskUsageMetrics) {
|
||||
handler.fsHandler = common.NewFsHandler(time.Minute, rootfsStorageDir, "", fsInfo)
|
||||
handler.fsHandler = common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, "", fsInfo)
|
||||
}
|
||||
|
||||
return handler, nil
|
||||
|
|
@ -228,7 +227,10 @@ func (handler *rktContainerHandler) getFsStats(stats *info.ContainerStats) error
|
|||
|
||||
fsStat := info.FsStats{Device: deviceInfo.Device, Limit: limit}
|
||||
|
||||
fsStat.BaseUsage, fsStat.Usage = handler.fsHandler.Usage()
|
||||
usage := handler.fsHandler.Usage()
|
||||
fsStat.BaseUsage = usage.BaseUsageBytes
|
||||
fsStat.Usage = usage.TotalUsageBytes
|
||||
fsStat.Inodes = usage.InodeUsage
|
||||
|
||||
stats.Filesystem = append(stats.Filesystem, fsStat)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package fs
|
|||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
|
|
@ -44,24 +45,24 @@ const (
|
|||
LabelRktImages = "rkt-images"
|
||||
)
|
||||
|
||||
// The maximum number of `du` tasks that can be running at once.
|
||||
const maxConsecutiveDus = 20
|
||||
// The maximum number of `du` and `find` tasks that can be running at once.
|
||||
const maxConcurrentOps = 20
|
||||
|
||||
// A pool for restricting the number of consecutive `du` tasks running.
|
||||
var duPool = make(chan struct{}, maxConsecutiveDus)
|
||||
// A pool for restricting the number of consecutive `du` and `find` tasks running.
|
||||
var pool = make(chan struct{}, maxConcurrentOps)
|
||||
|
||||
func init() {
|
||||
for i := 0; i < maxConsecutiveDus; i++ {
|
||||
releaseDuToken()
|
||||
for i := 0; i < maxConcurrentOps; i++ {
|
||||
releaseToken()
|
||||
}
|
||||
}
|
||||
|
||||
func claimDuToken() {
|
||||
<-duPool
|
||||
func claimToken() {
|
||||
<-pool
|
||||
}
|
||||
|
||||
func releaseDuToken() {
|
||||
duPool <- struct{}{}
|
||||
func releaseToken() {
|
||||
pool <- struct{}{}
|
||||
}
|
||||
|
||||
type partition struct {
|
||||
|
|
@ -428,12 +429,12 @@ func (self *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
|
|||
return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
|
||||
}
|
||||
|
||||
func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64, error) {
|
||||
func (self *RealFsInfo) GetDirDiskUsage(dir string, timeout time.Duration) (uint64, error) {
|
||||
if dir == "" {
|
||||
return 0, fmt.Errorf("invalid directory")
|
||||
}
|
||||
claimDuToken()
|
||||
defer releaseDuToken()
|
||||
claimToken()
|
||||
defer releaseToken()
|
||||
cmd := exec.Command("nice", "-n", "19", "du", "-s", dir)
|
||||
stdoutp, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
|
|
@ -447,21 +448,21 @@ func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64,
|
|||
if err := cmd.Start(); err != nil {
|
||||
return 0, fmt.Errorf("failed to exec du - %v", err)
|
||||
}
|
||||
stdoutb, souterr := ioutil.ReadAll(stdoutp)
|
||||
stderrb, _ := ioutil.ReadAll(stderrp)
|
||||
timer := time.AfterFunc(timeout, func() {
|
||||
glog.Infof("killing cmd %v due to timeout(%s)", cmd.Args, timeout.String())
|
||||
cmd.Process.Kill()
|
||||
})
|
||||
stdoutb, souterr := ioutil.ReadAll(stdoutp)
|
||||
if souterr != nil {
|
||||
glog.Errorf("failed to read from stdout for cmd %v - %v", cmd.Args, souterr)
|
||||
}
|
||||
stderrb, _ := ioutil.ReadAll(stderrp)
|
||||
err = cmd.Wait()
|
||||
timer.Stop()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("du command failed on %s with output stdout: %s, stderr: %s - %v", dir, string(stdoutb), string(stderrb), err)
|
||||
}
|
||||
stdout := string(stdoutb)
|
||||
if souterr != nil {
|
||||
glog.Errorf("failed to read from stdout for cmd %v - %v", cmd.Args, souterr)
|
||||
}
|
||||
usageInKb, err := strconv.ParseUint(strings.Fields(stdout)[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse 'du' output %s - %s", stdout, err)
|
||||
|
|
@ -469,6 +470,48 @@ func (self *RealFsInfo) GetDirUsage(dir string, timeout time.Duration) (uint64,
|
|||
return usageInKb * 1024, nil
|
||||
}
|
||||
|
||||
func (self *RealFsInfo) GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error) {
|
||||
if dir == "" {
|
||||
return 0, fmt.Errorf("invalid directory")
|
||||
}
|
||||
var stdout, stdwcerr, stdfinderr bytes.Buffer
|
||||
var err error
|
||||
claimToken()
|
||||
defer releaseToken()
|
||||
findCmd := exec.Command("find", dir, "-xdev", "-printf", ".")
|
||||
wcCmd := exec.Command("wc", "-c")
|
||||
if wcCmd.Stdin, err = findCmd.StdoutPipe(); err != nil {
|
||||
return 0, fmt.Errorf("failed to setup stdout for cmd %v - %v", findCmd.Args, err)
|
||||
}
|
||||
wcCmd.Stdout, wcCmd.Stderr, findCmd.Stderr = &stdout, &stdwcerr, &stdfinderr
|
||||
if err = findCmd.Start(); err != nil {
|
||||
return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stdfinderr.String())
|
||||
}
|
||||
|
||||
if err = wcCmd.Start(); err != nil {
|
||||
return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr %v", wcCmd.Args, err, stdwcerr.String())
|
||||
}
|
||||
timer := time.AfterFunc(timeout, func() {
|
||||
glog.Infof("killing cmd %v, and cmd %v due to timeout(%s)", findCmd.Args, wcCmd.Args, timeout.String())
|
||||
wcCmd.Process.Kill()
|
||||
findCmd.Process.Kill()
|
||||
})
|
||||
err = findCmd.Wait()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stdfinderr.String(), err)
|
||||
}
|
||||
err = wcCmd.Wait()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", wcCmd.Args, stdwcerr.String(), err)
|
||||
}
|
||||
timer.Stop()
|
||||
inodeUsage, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot parse cmds: %v, %v output %s - %s", findCmd.Args, wcCmd.Args, stdout.String(), err)
|
||||
}
|
||||
return inodeUsage, nil
|
||||
}
|
||||
|
||||
func getVfsStats(path string) (total uint64, free uint64, avail uint64, inodes uint64, inodesFree uint64, err error) {
|
||||
var s syscall.Statfs_t
|
||||
if err = syscall.Statfs(path, &s); err != nil {
|
||||
|
|
|
|||
|
|
@ -67,7 +67,10 @@ type FsInfo interface {
|
|||
GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error)
|
||||
|
||||
// Returns number of bytes occupied by 'dir'.
|
||||
GetDirUsage(dir string, timeout time.Duration) (uint64, error)
|
||||
GetDirDiskUsage(dir string, timeout time.Duration) (uint64, error)
|
||||
|
||||
// Returns number of inodes used by 'dir'.
|
||||
GetDirInodeUsage(dir string, timeout time.Duration) (uint64, error)
|
||||
|
||||
// Returns the block device info of the filesystem on which 'dir' resides.
|
||||
GetDirFsDevice(dir string) (*DeviceInfo, error)
|
||||
|
|
|
|||
|
|
@ -301,4 +301,8 @@ type FilesystemStats struct {
|
|||
TotalUsageBytes *uint64 `json:"totalUsageBytes,omitempty"`
|
||||
// Number of bytes consumed by a container through its root filesystem.
|
||||
BaseUsageBytes *uint64 `json:"baseUsageBytes,omitempty"`
|
||||
// Number of inodes used within the container's root filesystem.
|
||||
// This only accounts for inodes that are shared across containers,
|
||||
// and does not include inodes used in mounted directories.
|
||||
InodeUsage *uint64 `json:"containter_inode_usage,omitempty"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ func ContainerStatsFromV1(spec *v1.ContainerSpec, stats []*v1.ContainerStats) []
|
|||
stat.Filesystem = &FilesystemStats{
|
||||
TotalUsageBytes: &val.Filesystem[0].Usage,
|
||||
BaseUsageBytes: &val.Filesystem[0].BaseUsage,
|
||||
InodeUsage: &val.Filesystem[0].Inodes,
|
||||
}
|
||||
} else if len(val.Filesystem) > 1 {
|
||||
// Cannot handle multiple devices per container.
|
||||
|
|
|
|||
|
|
@ -101,6 +101,9 @@ type Manager interface {
|
|||
// Get version information about different components we depend on.
|
||||
GetVersionInfo() (*info.VersionInfo, error)
|
||||
|
||||
// Get filesystem information for the filesystem that contains the given directory
|
||||
GetDirFsInfo(dir string) (v2.FsInfo, error)
|
||||
|
||||
// Get filesystem information for a given label.
|
||||
// Returns information for all global filesystems if label is empty.
|
||||
GetFsInfo(label string) ([]v2.FsInfo, error)
|
||||
|
|
@ -657,6 +660,27 @@ func (self *manager) getRequestedContainers(containerName string, options v2.Req
|
|||
return containersMap, nil
|
||||
}
|
||||
|
||||
func (self *manager) GetDirFsInfo(dir string) (v2.FsInfo, error) {
|
||||
dirDevice, err := self.fsInfo.GetDirFsDevice(dir)
|
||||
if err != nil {
|
||||
return v2.FsInfo{}, fmt.Errorf("error trying to get filesystem Device for dir %v: err: %v", dir, err)
|
||||
}
|
||||
dirMountpoint, err := self.fsInfo.GetMountpointForDevice(dirDevice.Device)
|
||||
if err != nil {
|
||||
return v2.FsInfo{}, fmt.Errorf("error trying to get MountPoint for Root Device: %v, err: %v", dirDevice, err)
|
||||
}
|
||||
infos, err := self.GetFsInfo("")
|
||||
if err != nil {
|
||||
return v2.FsInfo{}, err
|
||||
}
|
||||
for _, info := range infos {
|
||||
if info.Mountpoint == dirMountpoint {
|
||||
return info, nil
|
||||
}
|
||||
}
|
||||
return v2.FsInfo{}, fmt.Errorf("did not find fs info for dir: %v", dir)
|
||||
}
|
||||
|
||||
func (self *manager) GetFsInfo(label string) ([]v2.FsInfo, error) {
|
||||
var empty time.Time
|
||||
// Get latest data from filesystems hanging off root container.
|
||||
|
|
|
|||
Loading…
Reference in New Issue