mirror of https://github.com/k3s-io/k3s
Merge pull request #74942 from WanLinghao/event_replace
Migrate oom watcher not relying on cAdviosr's API any morek3s-v1.15.3
Kubernetes Prow Robot
GitHub
commit
8ec6167f61
|
|
@ -186,6 +186,7 @@ pkg/kubelet/dockershim/network/testing
|
|||
pkg/kubelet/events
|
||||
pkg/kubelet/lifecycle
|
||||
pkg/kubelet/metrics
|
||||
pkg/kubelet/oom
|
||||
pkg/kubelet/pod
|
||||
pkg/kubelet/pod/testing
|
||||
pkg/kubelet/preemption
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ go_library(
|
|||
"kubelet_pods.go",
|
||||
"kubelet_resources.go",
|
||||
"kubelet_volumes.go",
|
||||
"oom_watcher.go",
|
||||
"pod_container_deletor.go",
|
||||
"pod_workers.go",
|
||||
"reason_cache.go",
|
||||
|
|
@ -70,6 +69,7 @@ go_library(
|
|||
"//pkg/kubelet/network/dns:go_default_library",
|
||||
"//pkg/kubelet/nodelease:go_default_library",
|
||||
"//pkg/kubelet/nodestatus:go_default_library",
|
||||
"//pkg/kubelet/oom:go_default_library",
|
||||
"//pkg/kubelet/pleg:go_default_library",
|
||||
"//pkg/kubelet/pod:go_default_library",
|
||||
"//pkg/kubelet/preemption:go_default_library",
|
||||
|
|
@ -145,7 +145,6 @@ go_library(
|
|||
"//staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2:go_default_library",
|
||||
"//third_party/forked/golang/expansion:go_default_library",
|
||||
"//vendor/github.com/golang/groupcache/lru:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/events:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/info/v1:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
"//vendor/k8s.io/utils/exec:go_default_library",
|
||||
|
|
@ -168,7 +167,6 @@ go_test(
|
|||
"kubelet_test.go",
|
||||
"kubelet_volumes_linux_test.go",
|
||||
"kubelet_volumes_test.go",
|
||||
"oom_watcher_test.go",
|
||||
"pod_container_deletor_test.go",
|
||||
"pod_workers_test.go",
|
||||
"reason_cache_test.go",
|
||||
|
|
@ -297,6 +295,7 @@ filegroup(
|
|||
"//pkg/kubelet/network:all-srcs",
|
||||
"//pkg/kubelet/nodelease:all-srcs",
|
||||
"//pkg/kubelet/nodestatus:all-srcs",
|
||||
"//pkg/kubelet/oom:all-srcs",
|
||||
"//pkg/kubelet/pleg:all-srcs",
|
||||
"//pkg/kubelet/pod:all-srcs",
|
||||
"//pkg/kubelet/preemption:all-srcs",
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ import (
|
|||
"k8s.io/kubernetes/pkg/kubelet/metrics/collectors"
|
||||
"k8s.io/kubernetes/pkg/kubelet/network/dns"
|
||||
"k8s.io/kubernetes/pkg/kubelet/nodelease"
|
||||
oomwatcher "k8s.io/kubernetes/pkg/kubelet/oom"
|
||||
"k8s.io/kubernetes/pkg/kubelet/pleg"
|
||||
kubepod "k8s.io/kubernetes/pkg/kubelet/pod"
|
||||
"k8s.io/kubernetes/pkg/kubelet/preemption"
|
||||
|
|
@ -464,7 +465,7 @@ func NewMainKubelet(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
|
|||
|
||||
containerRefManager := kubecontainer.NewRefManager()
|
||||
|
||||
oomWatcher := NewOOMWatcher(kubeDeps.CAdvisorInterface, kubeDeps.Recorder)
|
||||
oomWatcher := oomwatcher.NewOOMWatcher(kubeDeps.Recorder)
|
||||
|
||||
clusterDNS := make([]net.IP, 0, len(kubeCfg.ClusterDNS))
|
||||
for _, ipEntry := range kubeCfg.ClusterDNS {
|
||||
|
|
@ -1078,7 +1079,7 @@ type Kubelet struct {
|
|||
os kubecontainer.OSInterface
|
||||
|
||||
// Watcher of out of memory events.
|
||||
oomWatcher OOMWatcher
|
||||
oomWatcher oomwatcher.OOMWatcher
|
||||
|
||||
// Monitor resource usage
|
||||
resourceAnalyzer serverstats.ResourceAnalyzer
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"oom_watcher_linux.go",
|
||||
"oom_watcher_unsupported.go",
|
||||
"types.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/kubelet/oom",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
] + select({
|
||||
"@io_bazel_rules_go//go/platform:android": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:darwin": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:dragonfly": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:freebsd": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:linux": [
|
||||
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
"//vendor/github.com/google/cadvisor/utils/oomparser:go_default_library",
|
||||
"//vendor/k8s.io/klog:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:nacl": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:netbsd": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:openbsd": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:plan9": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:solaris": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows": [
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["oom_watcher_linux_test.go"],
|
||||
embed = [":go_default_library"],
|
||||
deps = select({
|
||||
"@io_bazel_rules_go//go/platform:linux": [
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
"//staging/src/k8s.io/client-go/tools/record:go_default_library",
|
||||
"//vendor/github.com/stretchr/testify/assert:go_default_library",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [":package-srcs"],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
// +build linux
|
||||
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors.
|
||||
|
||||
|
|
@ -14,61 +16,52 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
package oom
|
||||
|
||||
import (
|
||||
"github.com/google/cadvisor/events"
|
||||
cadvisorapi "github.com/google/cadvisor/info/v1"
|
||||
"k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/runtime"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
|
||||
|
||||
"github.com/google/cadvisor/utils/oomparser"
|
||||
)
|
||||
|
||||
// OOMWatcher defines the interface of OOM watchers.
|
||||
type OOMWatcher interface {
|
||||
Start(ref *v1.ObjectReference) error
|
||||
}
|
||||
|
||||
type realOOMWatcher struct {
|
||||
cadvisor cadvisor.Interface
|
||||
recorder record.EventRecorder
|
||||
}
|
||||
|
||||
var _ OOMWatcher = &realOOMWatcher{}
|
||||
|
||||
// NewOOMWatcher creates and initializes a OOMWatcher based on parameters.
|
||||
func NewOOMWatcher(cadvisor cadvisor.Interface, recorder record.EventRecorder) OOMWatcher {
|
||||
func NewOOMWatcher(recorder record.EventRecorder) OOMWatcher {
|
||||
return &realOOMWatcher{
|
||||
cadvisor: cadvisor,
|
||||
recorder: recorder,
|
||||
}
|
||||
}
|
||||
|
||||
const systemOOMEvent = "SystemOOM"
|
||||
|
||||
// Watches cadvisor for system oom's and records an event for every system oom encountered.
|
||||
// Watches for system oom's and records an event for every system oom encountered.
|
||||
func (ow *realOOMWatcher) Start(ref *v1.ObjectReference) error {
|
||||
request := events.Request{
|
||||
EventType: map[cadvisorapi.EventType]bool{
|
||||
cadvisorapi.EventOom: true,
|
||||
},
|
||||
ContainerName: "/",
|
||||
IncludeSubcontainers: false,
|
||||
}
|
||||
eventChannel, err := ow.cadvisor.WatchEvents(&request)
|
||||
oomLog, err := oomparser.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outStream := make(chan *oomparser.OomInstance, 10)
|
||||
go oomLog.StreamOoms(outStream)
|
||||
|
||||
go func() {
|
||||
defer runtime.HandleCrash()
|
||||
|
||||
for event := range eventChannel.GetChannel() {
|
||||
klog.V(2).Infof("Got sys oom event from cadvisor: %v", event)
|
||||
ow.recorder.PastEventf(ref, metav1.Time{Time: event.Timestamp}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
|
||||
for event := range outStream {
|
||||
if event.ContainerName == "/" {
|
||||
klog.V(1).Infof("Got sys oom event: %v", event)
|
||||
ow.recorder.PastEventf(ref, metav1.Time{Time: event.TimeOfDeath}, v1.EventTypeWarning, systemOOMEvent, "System OOM encountered")
|
||||
}
|
||||
}
|
||||
klog.Errorf("Unexpectedly stopped receiving OOM notifications from cAdvisor")
|
||||
klog.Errorf("Unexpectedly stopped receiving OOM notifications")
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kubelet
|
||||
package oom
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
|
@ -23,14 +23,12 @@ import (
|
|||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
|
||||
)
|
||||
|
||||
func TestBasic(t *testing.T) {
|
||||
fakeRecorder := &record.FakeRecorder{}
|
||||
mockCadvisor := &cadvisortest.Fake{}
|
||||
node := &v1.ObjectReference{}
|
||||
oomWatcher := NewOOMWatcher(mockCadvisor, fakeRecorder)
|
||||
oomWatcher := NewOOMWatcher(fakeRecorder)
|
||||
assert.NoError(t, oomWatcher.Start(node))
|
||||
|
||||
// TODO: Improve this test once cadvisor exports events.EventChannel as an interface
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
// +build !linux
|
||||
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oom
|
||||
|
||||
import (
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/client-go/tools/record"
|
||||
)
|
||||
|
||||
type oomWatcherUnsupported struct{}
|
||||
|
||||
var _ OOMWatcher = new(oomWatcherUnsupported)
|
||||
|
||||
// NewOOMWatcher creates a fake one here
|
||||
func NewOOMWatcher(_ record.EventRecorder) OOMWatcher {
|
||||
return &oomWatcherUnsupported{}
|
||||
}
|
||||
|
||||
func (ow *oomWatcherUnsupported) Start(_ *v1.ObjectReference) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package oom
|
||||
|
||||
import (
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// OOMWatcher defines the interface of OOM watchers.
|
||||
type OOMWatcher interface {
|
||||
Start(ref *v1.ObjectReference) error
|
||||
}
|
||||
Loading…
Reference in New Issue