mirror of https://github.com/k3s-io/k3s
Change Oom to OOM
parent
ae81f0b55f
commit
12299087da
|
@ -72,7 +72,7 @@ type ProxyServer struct {
|
||||||
EndpointsConfig *proxyconfig.EndpointsConfig
|
EndpointsConfig *proxyconfig.EndpointsConfig
|
||||||
EndpointsHandler proxyconfig.EndpointsConfigHandler
|
EndpointsHandler proxyconfig.EndpointsConfigHandler
|
||||||
IptInterface utiliptables.Interface
|
IptInterface utiliptables.Interface
|
||||||
OomAdjuster *oom.OomAdjuster
|
OOMAdjuster *oom.OOMAdjuster
|
||||||
Proxier proxy.ProxyProvider
|
Proxier proxy.ProxyProvider
|
||||||
Recorder record.EventRecorder
|
Recorder record.EventRecorder
|
||||||
ServiceConfig *proxyconfig.ServiceConfig
|
ServiceConfig *proxyconfig.ServiceConfig
|
||||||
|
@ -114,7 +114,7 @@ func NewProxyConfig() *ProxyServerConfig {
|
||||||
BindAddress: net.ParseIP("0.0.0.0"),
|
BindAddress: net.ParseIP("0.0.0.0"),
|
||||||
HealthzPort: 10249,
|
HealthzPort: 10249,
|
||||||
HealthzBindAddress: net.ParseIP("127.0.0.1"),
|
HealthzBindAddress: net.ParseIP("127.0.0.1"),
|
||||||
OOMScoreAdj: qos.KubeProxyOomScoreAdj,
|
OOMScoreAdj: qos.KubeProxyOOMScoreAdj,
|
||||||
ResourceContainer: "/kube-proxy",
|
ResourceContainer: "/kube-proxy",
|
||||||
SyncPeriod: 5 * time.Second,
|
SyncPeriod: 5 * time.Second,
|
||||||
}
|
}
|
||||||
|
@ -126,7 +126,7 @@ func NewProxyServer(
|
||||||
endpointsConfig *proxyconfig.EndpointsConfig,
|
endpointsConfig *proxyconfig.EndpointsConfig,
|
||||||
endpointsHandler proxyconfig.EndpointsConfigHandler,
|
endpointsHandler proxyconfig.EndpointsConfigHandler,
|
||||||
iptInterface utiliptables.Interface,
|
iptInterface utiliptables.Interface,
|
||||||
oomAdjuster *oom.OomAdjuster,
|
oomAdjuster *oom.OOMAdjuster,
|
||||||
proxier proxy.ProxyProvider,
|
proxier proxy.ProxyProvider,
|
||||||
recorder record.EventRecorder,
|
recorder record.EventRecorder,
|
||||||
serviceConfig *proxyconfig.ServiceConfig,
|
serviceConfig *proxyconfig.ServiceConfig,
|
||||||
|
@ -137,7 +137,7 @@ func NewProxyServer(
|
||||||
EndpointsConfig: endpointsConfig,
|
EndpointsConfig: endpointsConfig,
|
||||||
EndpointsHandler: endpointsHandler,
|
EndpointsHandler: endpointsHandler,
|
||||||
IptInterface: iptInterface,
|
IptInterface: iptInterface,
|
||||||
OomAdjuster: oomAdjuster,
|
OOMAdjuster: oomAdjuster,
|
||||||
Proxier: proxier,
|
Proxier: proxier,
|
||||||
Recorder: recorder,
|
Recorder: recorder,
|
||||||
ServiceConfig: serviceConfig,
|
ServiceConfig: serviceConfig,
|
||||||
|
@ -162,10 +162,10 @@ func NewProxyServerDefault(config *ProxyServerConfig) (*ProxyServer, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(vmarmol): Use container config for this.
|
// TODO(vmarmol): Use container config for this.
|
||||||
var oomAdjuster *oom.OomAdjuster
|
var oomAdjuster *oom.OOMAdjuster
|
||||||
if config.OOMScoreAdj != 0 {
|
if config.OOMScoreAdj != 0 {
|
||||||
oomAdjuster := oom.NewOomAdjuster()
|
oomAdjuster := oom.NewOOMAdjuster()
|
||||||
if err := oomAdjuster.ApplyOomScoreAdj(0, config.OOMScoreAdj); err != nil {
|
if err := oomAdjuster.ApplyOOMScoreAdj(0, config.OOMScoreAdj); err != nil {
|
||||||
glog.V(2).Info(err)
|
glog.V(2).Info(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -189,7 +189,7 @@ func NewKubeletServer() *KubeletServer {
|
||||||
NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/",
|
NetworkPluginDir: "/usr/libexec/kubernetes/kubelet-plugins/net/exec/",
|
||||||
NetworkPluginName: "",
|
NetworkPluginName: "",
|
||||||
NodeStatusUpdateFrequency: 10 * time.Second,
|
NodeStatusUpdateFrequency: 10 * time.Second,
|
||||||
OOMScoreAdj: qos.KubeletOomScoreAdj,
|
OOMScoreAdj: qos.KubeletOOMScoreAdj,
|
||||||
PodInfraContainerImage: dockertools.PodInfraContainerImage,
|
PodInfraContainerImage: dockertools.PodInfraContainerImage,
|
||||||
Port: ports.KubeletPort,
|
Port: ports.KubeletPort,
|
||||||
ReadOnlyPort: ports.KubeletReadOnlyPort,
|
ReadOnlyPort: ports.KubeletReadOnlyPort,
|
||||||
|
@ -449,8 +449,8 @@ func (s *KubeletServer) Run(kcfg *KubeletConfig) error {
|
||||||
glog.V(2).Infof("Using root directory: %v", s.RootDirectory)
|
glog.V(2).Infof("Using root directory: %v", s.RootDirectory)
|
||||||
|
|
||||||
// TODO(vmarmol): Do this through container config.
|
// TODO(vmarmol): Do this through container config.
|
||||||
oomAdjuster := oom.NewOomAdjuster()
|
oomAdjuster := oom.NewOOMAdjuster()
|
||||||
if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
|
if err := oomAdjuster.ApplyOOMScoreAdj(0, s.OOMScoreAdj); err != nil {
|
||||||
glog.Warning(err)
|
glog.Warning(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -105,8 +105,8 @@ func (s *KubeletExecutorServer) syncExternalShutdownWatcher() (io.Closer, error)
|
||||||
func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
|
func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
|
||||||
rand.Seed(time.Now().UTC().UnixNano())
|
rand.Seed(time.Now().UTC().UnixNano())
|
||||||
|
|
||||||
oomAdjuster := oom.NewOomAdjuster()
|
oomAdjuster := oom.NewOOMAdjuster()
|
||||||
if err := oomAdjuster.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
|
if err := oomAdjuster.ApplyOOMScoreAdj(0, s.OOMScoreAdj); err != nil {
|
||||||
log.Info(err)
|
log.Info(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -149,7 +149,7 @@ Container OOM score configuration
|
||||||
- Hack, because these critical tasks might die if they conflict with guaranteed containers. in the future, we should place all user-pods into a separate cgroup, and set a limit on the memory they can consume.
|
- Hack, because these critical tasks might die if they conflict with guaranteed containers. in the future, we should place all user-pods into a separate cgroup, and set a limit on the memory they can consume.
|
||||||
|
|
||||||
Setting OOM_SCORE_ADJ for a container
|
Setting OOM_SCORE_ADJ for a container
|
||||||
- Refactor existing ApplyOomScoreAdj to util/oom.go
|
- Refactor existing ApplyOOMScoreAdj to util/oom.go
|
||||||
- To set OOM_SCORE_ADJ of a container, we loop through all processes in the container, and set OOM_SCORE_ADJ
|
- To set OOM_SCORE_ADJ of a container, we loop through all processes in the container, and set OOM_SCORE_ADJ
|
||||||
- We keep looping until the list of processes in the container stabilizes. This is sufficient because child processes inherit OOM_SCORE_ADJ.
|
- We keep looping until the list of processes in the container stabilizes. This is sufficient because child processes inherit OOM_SCORE_ADJ.
|
||||||
|
|
||||||
|
|
|
@ -71,9 +71,9 @@ hack/lib/logging.sh: local source_file=${BASH_SOURCE[$stack_skip]}
|
||||||
hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}"
|
hack/local-up-cluster.sh: runtime_config="--runtime-config=${RUNTIME_CONFIG}"
|
||||||
hack/local-up-cluster.sh: runtime_config=""
|
hack/local-up-cluster.sh: runtime_config=""
|
||||||
pkg/cloudprovider/providers/vagrant/vagrant_test.go: testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "node_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`)
|
pkg/cloudprovider/providers/vagrant/vagrant_test.go: testSaltMinionsResponse = []byte(`{ "return": [{"kubernetes-minion-1": {"kernel": "Linux", "domain": "", "zmqversion": "3.2.4", "kernelrelease": "3.11.10-301.fc20.x86_64", "pythonpath": ["/usr/bin", "/usr/lib64/python27.zip", "/usr/lib64/python2.7", "/usr/lib64/python2.7/plat-linux2", "/usr/lib64/python2.7/lib-tk", "/usr/lib64/python2.7/lib-old", "/usr/lib64/python2.7/lib-dynload", "/usr/lib64/python2.7/site-packages", "/usr/lib/python2.7/site-packages"], "etcd_servers": "10.245.1.2", "ip_interfaces": {"lo": ["127.0.0.1"], "docker0": ["172.17.42.1"], "enp0s8": ["10.245.2.2"], "p2p1": ["10.0.2.15"]}, "shell": "/bin/sh", "mem_total": 491, "saltversioninfo": [2014, 1, 7], "osmajorrelease": ["20"], "node_ip": "10.245.2.2", "id": "kubernetes-minion-1", "osrelease": "20", "ps": "ps -efH", "server_id": 1005530826, "num_cpus": 1, "hwaddr_interfaces": {"lo": "00:00:00:00:00:00", "docker0": "56:84:7a:fe:97:99", "enp0s8": "08:00:27:17:c5:0f", "p2p1": "08:00:27:96:96:e1"}, "virtual": "VirtualBox", "osfullname": "Fedora", "master": "kubernetes-master", "ipv4": ["10.0.2.15", "10.245.2.2", "127.0.0.1", "172.17.42.1"], "ipv6": ["::1", "fe80::a00:27ff:fe17:c50f", "fe80::a00:27ff:fe96:96e1"], "cpu_flags": ["fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce", "cx8", "apic", "sep", "mtrr", "pge", "mca", "cmov", "pat", "pse36", "clflush", "mmx", "fxsr", "sse", "sse2", "syscall", "nx", "rdtscp", "lm", "constant_tsc", "rep_good", "nopl", "pni", "monitor", "ssse3", "lahf_lm"], "localhost": "kubernetes-minion-1", "lsb_distrib_id": "Fedora", "fqdn_ip4": ["127.0.0.1"], "fqdn_ip6": [], "nodename": "kubernetes-minion-1", "saltversion": "2014.1.7", "saltpath": "/usr/lib/python2.7/site-packages/salt", "pythonversion": [2, 7, 5, "final", 0], "host": "kubernetes-minion-1", "os_family": "RedHat", "oscodename": "Heisenbug", "defaultencoding": "UTF-8", "osfinger": "Fedora-20", "roles": ["kubernetes-pool"], "num_gpus": 1, "cpu_model": "Intel(R) Core(TM) i7-4600U CPU @ 2.10GHz", "fqdn": "kubernetes-minion-1", "osarch": "x86_64", "cpuarch": "x86_64", "gpus": [{"model": "VirtualBox Graphics Adapter", "vendor": "unknown"}], "path": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin", "os": "Fedora", "defaultlanguage": "en_US"}}]}`)
|
||||||
pkg/kubelet/qos/memory_policy_test.go: t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOomScoreAdj, test.highOomScoreAdj, oomScoreAdj)
|
pkg/kubelet/qos/memory_policy_test.go: t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOOMScoreAdj, test.highOOMScoreAdj, oomScoreAdj)
|
||||||
pkg/kubelet/qos/memory_policy_test.go: highOomScoreAdj int // The min oom_score_adj score the container should be assigned.
|
pkg/kubelet/qos/memory_policy_test.go: highOOMScoreAdj int // The min oom_score_adj score the container should be assigned.
|
||||||
pkg/kubelet/qos/memory_policy_test.go: lowOomScoreAdj int // The max oom_score_adj score the container should be assigned.
|
pkg/kubelet/qos/memory_policy_test.go: lowOOMScoreAdj int // The max oom_score_adj score the container should be assigned.
|
||||||
pkg/util/oom/oom_linux.go: err = fmt.Errorf("failed to set oom_score_adj to %d: %v", oomScoreAdj, writeErr)
|
pkg/util/oom/oom_linux.go: err = fmt.Errorf("failed to set oom_score_adj to %d: %v", oomScoreAdj, writeErr)
|
||||||
pkg/util/oom/oom_linux.go: return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
|
pkg/util/oom/oom_linux.go: return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
|
||||||
pkg/util/oom/oom_linux.go: oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
|
pkg/util/oom/oom_linux.go: oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
|
||||||
|
|
|
@ -283,8 +283,8 @@ func ensureDockerInContainer(cadvisor cadvisor.Interface, oomScoreAdj int, manag
|
||||||
}
|
}
|
||||||
|
|
||||||
// Also apply oom-score-adj to processes
|
// Also apply oom-score-adj to processes
|
||||||
oomAdjuster := oom.NewOomAdjuster()
|
oomAdjuster := oom.NewOOMAdjuster()
|
||||||
if err := oomAdjuster.ApplyOomScoreAdj(pid, oomScoreAdj); err != nil {
|
if err := oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err != nil {
|
||||||
errs = append(errs, fmt.Errorf("failed to apply oom score %d to PID %d", oomScoreAdj, pid))
|
errs = append(errs, fmt.Errorf("failed to apply oom score %d to PID %d", oomScoreAdj, pid))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -42,11 +42,11 @@ func NewFakeDockerManager(
|
||||||
generator kubecontainer.RunContainerOptionsGenerator,
|
generator kubecontainer.RunContainerOptionsGenerator,
|
||||||
httpClient kubeletTypes.HttpGetter) *DockerManager {
|
httpClient kubeletTypes.HttpGetter) *DockerManager {
|
||||||
|
|
||||||
fakeOomAdjuster := oom.NewFakeOomAdjuster()
|
fakeOOMAdjuster := oom.NewFakeOOMAdjuster()
|
||||||
fakeProcFs := procfs.NewFakeProcFs()
|
fakeProcFs := procfs.NewFakeProcFs()
|
||||||
dm := NewDockerManager(client, recorder, readinessManager, containerRefManager, machineInfo, podInfraContainerImage, qps,
|
dm := NewDockerManager(client, recorder, readinessManager, containerRefManager, machineInfo, podInfraContainerImage, qps,
|
||||||
burst, containerLogsDir, osInterface, networkPlugin, generator, httpClient, &NativeExecHandler{},
|
burst, containerLogsDir, osInterface, networkPlugin, generator, httpClient, &NativeExecHandler{},
|
||||||
fakeOomAdjuster, fakeProcFs, false)
|
fakeOOMAdjuster, fakeProcFs, false)
|
||||||
dm.dockerPuller = &FakeDockerPuller{}
|
dm.dockerPuller = &FakeDockerPuller{}
|
||||||
dm.prober = prober.New(nil, readinessManager, containerRefManager, recorder)
|
dm.prober = prober.New(nil, readinessManager, containerRefManager, recorder)
|
||||||
return dm
|
return dm
|
||||||
|
|
|
@ -131,7 +131,7 @@ type DockerManager struct {
|
||||||
execHandler ExecHandler
|
execHandler ExecHandler
|
||||||
|
|
||||||
// Used to set OOM scores of processes.
|
// Used to set OOM scores of processes.
|
||||||
oomAdjuster *oom.OomAdjuster
|
oomAdjuster *oom.OOMAdjuster
|
||||||
|
|
||||||
// Get information from /proc mount.
|
// Get information from /proc mount.
|
||||||
procFs procfs.ProcFsInterface
|
procFs procfs.ProcFsInterface
|
||||||
|
@ -155,7 +155,7 @@ func NewDockerManager(
|
||||||
generator kubecontainer.RunContainerOptionsGenerator,
|
generator kubecontainer.RunContainerOptionsGenerator,
|
||||||
httpClient kubeletTypes.HttpGetter,
|
httpClient kubeletTypes.HttpGetter,
|
||||||
execHandler ExecHandler,
|
execHandler ExecHandler,
|
||||||
oomAdjuster *oom.OomAdjuster,
|
oomAdjuster *oom.OOMAdjuster,
|
||||||
procFs procfs.ProcFsInterface,
|
procFs procfs.ProcFsInterface,
|
||||||
cpuCFSQuota bool) *DockerManager {
|
cpuCFSQuota bool) *DockerManager {
|
||||||
// Work out the location of the Docker runtime, defaulting to /var/lib/docker
|
// Work out the location of the Docker runtime, defaulting to /var/lib/docker
|
||||||
|
@ -1470,15 +1470,15 @@ func (dm *DockerManager) runContainerInPod(pod *api.Pod, container *api.Containe
|
||||||
// whole pod will die.
|
// whole pod will die.
|
||||||
var oomScoreAdj int
|
var oomScoreAdj int
|
||||||
if container.Name == PodInfraContainerName {
|
if container.Name == PodInfraContainerName {
|
||||||
oomScoreAdj = qos.PodInfraOomAdj
|
oomScoreAdj = qos.PodInfraOOMAdj
|
||||||
} else {
|
} else {
|
||||||
oomScoreAdj = qos.GetContainerOomScoreAdjust(container, dm.machineInfo.MemoryCapacity)
|
oomScoreAdj = qos.GetContainerOOMScoreAdjust(container, dm.machineInfo.MemoryCapacity)
|
||||||
}
|
}
|
||||||
cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid)
|
cgroupName, err := dm.procFs.GetFullContainerName(containerInfo.State.Pid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if err = dm.oomAdjuster.ApplyOomScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil {
|
if err = dm.oomAdjuster.ApplyOOMScoreAdjContainer(cgroupName, oomScoreAdj, 5); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -311,7 +311,7 @@ func NewMainKubelet(
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
oomAdjuster := oom.NewOomAdjuster()
|
oomAdjuster := oom.NewOOMAdjuster()
|
||||||
procFs := procfs.NewProcFs()
|
procFs := procfs.NewProcFs()
|
||||||
|
|
||||||
// Initialize the runtime.
|
// Initialize the runtime.
|
||||||
|
|
|
@ -21,9 +21,9 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
PodInfraOomAdj int = -999
|
PodInfraOOMAdj int = -999
|
||||||
KubeletOomScoreAdj int = -999
|
KubeletOOMScoreAdj int = -999
|
||||||
KubeProxyOomScoreAdj int = -999
|
KubeProxyOOMScoreAdj int = -999
|
||||||
)
|
)
|
||||||
|
|
||||||
// isMemoryBestEffort returns true if the container's memory requirements are best-effort.
|
// isMemoryBestEffort returns true if the container's memory requirements are best-effort.
|
||||||
|
@ -42,12 +42,12 @@ func isMemoryGuaranteed(container *api.Container) bool {
|
||||||
return (*memoryRequest).Cmp(*memoryLimit) == 0 && memoryRequest.Value() != 0
|
return (*memoryRequest).Cmp(*memoryLimit) == 0 && memoryRequest.Value() != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetContainerOomAdjust returns the amount by which the OOM score of all processes in the
|
// GetContainerOOMAdjust returns the amount by which the OOM score of all processes in the
|
||||||
// container should be adjusted. The OOM score of a process is the percentage of memory it consumes
|
// container should be adjusted. The OOM score of a process is the percentage of memory it consumes
|
||||||
// multiplied by 100 (barring exceptional cases) + a configurable quantity which is between -1000
|
// multiplied by 100 (barring exceptional cases) + a configurable quantity which is between -1000
|
||||||
// and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
|
// and 1000. Containers with higher OOM scores are killed if the system runs out of memory.
|
||||||
// See https://lwn.net/Articles/391222/ for more information.
|
// See https://lwn.net/Articles/391222/ for more information.
|
||||||
func GetContainerOomScoreAdjust(container *api.Container, memoryCapacity int64) int {
|
func GetContainerOOMScoreAdjust(container *api.Container, memoryCapacity int64) int {
|
||||||
if isMemoryGuaranteed(container) {
|
if isMemoryGuaranteed(container) {
|
||||||
// Memory guaranteed containers should be the last to get killed.
|
// Memory guaranteed containers should be the last to get killed.
|
||||||
return -999
|
return -999
|
||||||
|
|
|
@ -128,60 +128,60 @@ func TestIsMemoryGuaranteed(t *testing.T) {
|
||||||
type oomTest struct {
|
type oomTest struct {
|
||||||
container *api.Container
|
container *api.Container
|
||||||
memoryCapacity int64
|
memoryCapacity int64
|
||||||
lowOomScoreAdj int // The max oom_score_adj score the container should be assigned.
|
lowOOMScoreAdj int // The max oom_score_adj score the container should be assigned.
|
||||||
highOomScoreAdj int // The min oom_score_adj score the container should be assigned.
|
highOOMScoreAdj int // The min oom_score_adj score the container should be assigned.
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetContainerOomScoreAdjust(t *testing.T) {
|
func TestGetContainerOOMScoreAdjust(t *testing.T) {
|
||||||
|
|
||||||
oomTests := []oomTest{
|
oomTests := []oomTest{
|
||||||
{
|
{
|
||||||
container: &zeroRequestMemoryBestEffort,
|
container: &zeroRequestMemoryBestEffort,
|
||||||
memoryCapacity: 4000000000,
|
memoryCapacity: 4000000000,
|
||||||
lowOomScoreAdj: 1000,
|
lowOOMScoreAdj: 1000,
|
||||||
highOomScoreAdj: 1000,
|
highOOMScoreAdj: 1000,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &edgeMemoryBestEffort,
|
container: &edgeMemoryBestEffort,
|
||||||
memoryCapacity: 8000000000,
|
memoryCapacity: 8000000000,
|
||||||
lowOomScoreAdj: 1000,
|
lowOOMScoreAdj: 1000,
|
||||||
highOomScoreAdj: 1000,
|
highOOMScoreAdj: 1000,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &noRequestMemoryBestEffort,
|
container: &noRequestMemoryBestEffort,
|
||||||
memoryCapacity: 7230457451,
|
memoryCapacity: 7230457451,
|
||||||
lowOomScoreAdj: 1000,
|
lowOOMScoreAdj: 1000,
|
||||||
highOomScoreAdj: 1000,
|
highOOMScoreAdj: 1000,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &noLimitMemoryBestEffort,
|
container: &noLimitMemoryBestEffort,
|
||||||
memoryCapacity: 4000000000,
|
memoryCapacity: 4000000000,
|
||||||
lowOomScoreAdj: 1000,
|
lowOOMScoreAdj: 1000,
|
||||||
highOomScoreAdj: 1000,
|
highOOMScoreAdj: 1000,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &memoryGuaranteed,
|
container: &memoryGuaranteed,
|
||||||
memoryCapacity: 123456789,
|
memoryCapacity: 123456789,
|
||||||
lowOomScoreAdj: -999,
|
lowOOMScoreAdj: -999,
|
||||||
highOomScoreAdj: -999,
|
highOOMScoreAdj: -999,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &memoryBurstable,
|
container: &memoryBurstable,
|
||||||
memoryCapacity: standardMemoryAmount,
|
memoryCapacity: standardMemoryAmount,
|
||||||
lowOomScoreAdj: 495,
|
lowOOMScoreAdj: 495,
|
||||||
highOomScoreAdj: 505,
|
highOOMScoreAdj: 505,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
container: &memoryBurstableNoLimit,
|
container: &memoryBurstableNoLimit,
|
||||||
memoryCapacity: standardMemoryAmount,
|
memoryCapacity: standardMemoryAmount,
|
||||||
lowOomScoreAdj: 2,
|
lowOOMScoreAdj: 2,
|
||||||
highOomScoreAdj: 2,
|
highOOMScoreAdj: 2,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for _, test := range oomTests {
|
for _, test := range oomTests {
|
||||||
oomScoreAdj := GetContainerOomScoreAdjust(test.container, test.memoryCapacity)
|
oomScoreAdj := GetContainerOOMScoreAdjust(test.container, test.memoryCapacity)
|
||||||
if oomScoreAdj < test.lowOomScoreAdj || oomScoreAdj > test.highOomScoreAdj {
|
if oomScoreAdj < test.lowOOMScoreAdj || oomScoreAdj > test.highOOMScoreAdj {
|
||||||
t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOomScoreAdj, test.highOomScoreAdj, oomScoreAdj)
|
t.Errorf("oom_score_adj should be between %d and %d, but was %d", test.lowOOMScoreAdj, test.highOOMScoreAdj, oomScoreAdj)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,8 +19,8 @@ package oom
|
||||||
// This is a struct instead of an interface to allow injection of process ID listers and
|
// This is a struct instead of an interface to allow injection of process ID listers and
|
||||||
// applying OOM score in tests.
|
// applying OOM score in tests.
|
||||||
// TODO: make this an interface, and inject a mock ioutil struct for testing.
|
// TODO: make this an interface, and inject a mock ioutil struct for testing.
|
||||||
type OomAdjuster struct {
|
type OOMAdjuster struct {
|
||||||
pidLister func(cgroupName string) ([]int, error)
|
pidLister func(cgroupName string) ([]int, error)
|
||||||
ApplyOomScoreAdj func(pid int, oomScoreAdj int) error
|
ApplyOOMScoreAdj func(pid int, oomScoreAdj int) error
|
||||||
ApplyOomScoreAdjContainer func(cgroupName string, oomScoreAdj, maxTries int) error
|
ApplyOOMScoreAdjContainer func(cgroupName string, oomScoreAdj, maxTries int) error
|
||||||
}
|
}
|
||||||
|
|
|
@ -16,19 +16,19 @@ limitations under the License.
|
||||||
|
|
||||||
package oom
|
package oom
|
||||||
|
|
||||||
type FakeOomAdjuster struct{}
|
type FakeOOMAdjuster struct{}
|
||||||
|
|
||||||
func NewFakeOomAdjuster() *OomAdjuster {
|
func NewFakeOOMAdjuster() *OOMAdjuster {
|
||||||
return &OomAdjuster{
|
return &OOMAdjuster{
|
||||||
ApplyOomScoreAdj: fakeApplyOomScoreAdj,
|
ApplyOOMScoreAdj: fakeApplyOOMScoreAdj,
|
||||||
ApplyOomScoreAdjContainer: fakeApplyOomScoreAdjContainer,
|
ApplyOOMScoreAdjContainer: fakeApplyOOMScoreAdjContainer,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeApplyOomScoreAdj(pid int, oomScoreAdj int) error {
|
func fakeApplyOOMScoreAdj(pid int, oomScoreAdj int) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fakeApplyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
func fakeApplyOOMScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -29,12 +29,12 @@ import (
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
)
|
)
|
||||||
|
|
||||||
func NewOomAdjuster() *OomAdjuster {
|
func NewOOMAdjuster() *OOMAdjuster {
|
||||||
oomAdjuster := &OomAdjuster{
|
oomAdjuster := &OOMAdjuster{
|
||||||
pidLister: getPids,
|
pidLister: getPids,
|
||||||
ApplyOomScoreAdj: applyOomScoreAdj,
|
ApplyOOMScoreAdj: applyOOMScoreAdj,
|
||||||
}
|
}
|
||||||
oomAdjuster.ApplyOomScoreAdjContainer = oomAdjuster.applyOomScoreAdjContainer
|
oomAdjuster.ApplyOOMScoreAdjContainer = oomAdjuster.applyOOMScoreAdjContainer
|
||||||
return oomAdjuster
|
return oomAdjuster
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -48,7 +48,7 @@ func getPids(cgroupName string) ([]int, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
|
// Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
|
||||||
func applyOomScoreAdj(pid int, oomScoreAdj int) error {
|
func applyOOMScoreAdj(pid int, oomScoreAdj int) error {
|
||||||
if pid < 0 {
|
if pid < 0 {
|
||||||
return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
|
return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
|
||||||
}
|
}
|
||||||
|
@ -79,7 +79,7 @@ func applyOomScoreAdj(pid int, oomScoreAdj int) error {
|
||||||
|
|
||||||
// Writes 'value' to /proc/<pid>/oom_score_adj for all processes in cgroup cgroupName.
|
// Writes 'value' to /proc/<pid>/oom_score_adj for all processes in cgroup cgroupName.
|
||||||
// Keeps trying to write until the process list of the cgroup stabilizes, or until maxTries tries.
|
// Keeps trying to write until the process list of the cgroup stabilizes, or until maxTries tries.
|
||||||
func (oomAdjuster *OomAdjuster) applyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
||||||
adjustedProcessSet := make(map[int]bool)
|
adjustedProcessSet := make(map[int]bool)
|
||||||
for i := 0; i < maxTries; i++ {
|
for i := 0; i < maxTries; i++ {
|
||||||
continueAdjusting := false
|
continueAdjusting := false
|
||||||
|
@ -93,7 +93,7 @@ func (oomAdjuster *OomAdjuster) applyOomScoreAdjContainer(cgroupName string, oom
|
||||||
for _, pid := range pidList {
|
for _, pid := range pidList {
|
||||||
if !adjustedProcessSet[pid] {
|
if !adjustedProcessSet[pid] {
|
||||||
continueAdjusting = true
|
continueAdjusting = true
|
||||||
if err = oomAdjuster.ApplyOomScoreAdj(pid, oomScoreAdj); err == nil {
|
if err = oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err == nil {
|
||||||
adjustedProcessSet[pid] = true
|
adjustedProcessSet[pid] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,63 +38,63 @@ func sequenceToPidLister(pidListSequence [][]int) func(string) ([]int, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tests that applyOomScoreAdjContainer correctly applies OOM scores to relevant processes, or
|
// Tests that applyOOMScoreAdjContainer correctly applies OOM scores to relevant processes, or
|
||||||
// returns the right error.
|
// returns the right error.
|
||||||
func applyOomScoreAdjContainerTester(pidListSequence [][]int, maxTries int, appliedPids []int, expectedError bool, t *testing.T) {
|
func applyOOMScoreAdjContainerTester(pidListSequence [][]int, maxTries int, appliedPids []int, expectedError bool, t *testing.T) {
|
||||||
pidOoms := make(map[int]bool)
|
pidOOMs := make(map[int]bool)
|
||||||
|
|
||||||
// Mock ApplyOomScoreAdj and pidLister.
|
// Mock ApplyOOMScoreAdj and pidLister.
|
||||||
oomAdjuster := NewOomAdjuster()
|
oomAdjuster := NewOOMAdjuster()
|
||||||
oomAdjuster.ApplyOomScoreAdj = func(pid int, oomScoreAdj int) error {
|
oomAdjuster.ApplyOOMScoreAdj = func(pid int, oomScoreAdj int) error {
|
||||||
pidOoms[pid] = true
|
pidOOMs[pid] = true
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
oomAdjuster.pidLister = sequenceToPidLister(pidListSequence)
|
oomAdjuster.pidLister = sequenceToPidLister(pidListSequence)
|
||||||
err := oomAdjuster.ApplyOomScoreAdjContainer("", 100, maxTries)
|
err := oomAdjuster.ApplyOOMScoreAdjContainer("", 100, maxTries)
|
||||||
|
|
||||||
// Check error value.
|
// Check error value.
|
||||||
if expectedError && err == nil {
|
if expectedError && err == nil {
|
||||||
t.Errorf("Expected error %+v when running ApplyOomScoreAdjContainer but got no error", expectedError)
|
t.Errorf("Expected error %+v when running ApplyOOMScoreAdjContainer but got no error", expectedError)
|
||||||
return
|
return
|
||||||
} else if !expectedError && err != nil {
|
} else if !expectedError && err != nil {
|
||||||
t.Errorf("Expected no error but got error %+v when running ApplyOomScoreAdjContainer", err)
|
t.Errorf("Expected no error but got error %+v when running ApplyOOMScoreAdjContainer", err)
|
||||||
return
|
return
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check that OOM scores were applied to the right processes.
|
// Check that OOM scores were applied to the right processes.
|
||||||
if len(appliedPids) != len(pidOoms) {
|
if len(appliedPids) != len(pidOOMs) {
|
||||||
t.Errorf("Applied OOM scores to incorrect number of processes")
|
t.Errorf("Applied OOM scores to incorrect number of processes")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
for _, pid := range appliedPids {
|
for _, pid := range appliedPids {
|
||||||
if !pidOoms[pid] {
|
if !pidOOMs[pid] {
|
||||||
t.Errorf("Failed to apply OOM scores to process %d", pid)
|
t.Errorf("Failed to apply OOM scores to process %d", pid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOomScoreAdjContainer(t *testing.T) {
|
func TestOOMScoreAdjContainer(t *testing.T) {
|
||||||
pidListSequenceEmpty := [][]int{}
|
pidListSequenceEmpty := [][]int{}
|
||||||
applyOomScoreAdjContainerTester(pidListSequenceEmpty, 3, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequenceEmpty, 3, nil, true, t)
|
||||||
|
|
||||||
pidListSequence1 := [][]int{
|
pidListSequence1 := [][]int{
|
||||||
{1, 2},
|
{1, 2},
|
||||||
}
|
}
|
||||||
applyOomScoreAdjContainerTester(pidListSequence1, 1, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequence1, 1, nil, true, t)
|
||||||
applyOomScoreAdjContainerTester(pidListSequence1, 2, []int{1, 2}, false, t)
|
applyOOMScoreAdjContainerTester(pidListSequence1, 2, []int{1, 2}, false, t)
|
||||||
applyOomScoreAdjContainerTester(pidListSequence1, 3, []int{1, 2}, false, t)
|
applyOOMScoreAdjContainerTester(pidListSequence1, 3, []int{1, 2}, false, t)
|
||||||
|
|
||||||
pidListSequence3 := [][]int{
|
pidListSequence3 := [][]int{
|
||||||
{1, 2},
|
{1, 2},
|
||||||
{1, 2, 4, 5},
|
{1, 2, 4, 5},
|
||||||
{2, 1, 4, 5, 3},
|
{2, 1, 4, 5, 3},
|
||||||
}
|
}
|
||||||
applyOomScoreAdjContainerTester(pidListSequence3, 1, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequence3, 1, nil, true, t)
|
||||||
applyOomScoreAdjContainerTester(pidListSequence3, 2, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequence3, 2, nil, true, t)
|
||||||
applyOomScoreAdjContainerTester(pidListSequence3, 3, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequence3, 3, nil, true, t)
|
||||||
applyOomScoreAdjContainerTester(pidListSequence3, 4, []int{1, 2, 3, 4, 5}, false, t)
|
applyOOMScoreAdjContainerTester(pidListSequence3, 4, []int{1, 2, 3, 4, 5}, false, t)
|
||||||
|
|
||||||
pidListSequenceLag := [][]int{
|
pidListSequenceLag := [][]int{
|
||||||
{},
|
{},
|
||||||
|
@ -104,7 +104,7 @@ func TestOomScoreAdjContainer(t *testing.T) {
|
||||||
{1, 2, 4, 5},
|
{1, 2, 4, 5},
|
||||||
}
|
}
|
||||||
for i := 1; i < 5; i++ {
|
for i := 1; i < 5; i++ {
|
||||||
applyOomScoreAdjContainerTester(pidListSequenceLag, i, nil, true, t)
|
applyOOMScoreAdjContainerTester(pidListSequenceLag, i, nil, true, t)
|
||||||
}
|
}
|
||||||
applyOomScoreAdjContainerTester(pidListSequenceLag, 6, []int{1, 2, 4, 5}, false, t)
|
applyOOMScoreAdjContainerTester(pidListSequenceLag, 6, []int{1, 2, 4, 5}, false, t)
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,17 +24,17 @@ import (
|
||||||
|
|
||||||
var unsupportedErr = errors.New("setting OOM scores is unsupported in this build")
|
var unsupportedErr = errors.New("setting OOM scores is unsupported in this build")
|
||||||
|
|
||||||
func NewOomAdjuster() *OomAdjuster {
|
func NewOOMAdjuster() *OOMAdjuster {
|
||||||
return &OomAdjuster{
|
return &OOMAdjuster{
|
||||||
ApplyOomScoreAdj: unsupportedApplyOomScoreAdj,
|
ApplyOOMScoreAdj: unsupportedApplyOOMScoreAdj,
|
||||||
ApplyOomScoreAdjContainer: unsupportedApplyOomScoreAdjContainer,
|
ApplyOOMScoreAdjContainer: unsupportedApplyOOMScoreAdjContainer,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func unsupportedApplyOomScoreAdj(pid int, oomScoreAdj int) error {
|
func unsupportedApplyOOMScoreAdj(pid int, oomScoreAdj int) error {
|
||||||
return unsupportedErr
|
return unsupportedErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func unsupportedApplyOomScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
func unsupportedApplyOOMScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
|
||||||
return unsupportedErr
|
return unsupportedErr
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue