diff --git a/cmd/agent/main.go b/cmd/agent/main.go index dfb75dfe30..268995d32c 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/agent" @@ -16,8 +18,7 @@ func main() { cmds.NewAgentCommand(agent.Run), } - err := app.Run(configfilearg.MustParse(os.Args)) - if err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/cmd/etcdsnapshot/main.go b/cmd/etcdsnapshot/main.go index 3d55ab6181..0ad08000f6 100644 --- a/cmd/etcdsnapshot/main.go +++ b/cmd/etcdsnapshot/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/cmds" @@ -22,7 +24,7 @@ func main() { ), } - if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go index 036cea3da9..4d7014985e 100644 --- a/cmd/k3s/main.go +++ b/cmd/k3s/main.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "os" "os/exec" "path/filepath" @@ -53,7 +54,7 @@ func main() { ), } - if err := app.Run(os.Args); err != nil { + if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } @@ -90,7 +91,7 @@ func runCLIs(dataDir string) bool { progName := filepath.Base(os.Args[0]) switch progName { case "crictl", "ctr", "kubectl": - if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil { + if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } return true diff --git a/cmd/server/main.go b/cmd/server/main.go index bd01b22be2..9cc8fdbc88 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,6 +1,8 @@ package main import ( + "context" + "errors" "os" "path/filepath" @@ -52,8 +54,7 @@ func main() { ), } - err := app.Run(configfilearg.MustParse(os.Args)) - if err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/go.mod b/go.mod index f08a32340d..21e1d622bb 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,8 @@ replace ( k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.1-k3s1 k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.1-k3s1 k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.1-k3s1 + k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1 // k3s-release-1.x + k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 // k3s-main k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1 k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.1-k3s1 k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.1-k3s1 diff --git a/go.sum b/go.sum index 9b25a77cc6..34d7323f19 100644 --- a/go.sum +++ b/go.sum @@ -320,7 +320,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= @@ -575,6 +574,10 @@ github.com/k3s-io/helm-controller v0.11.3 h1:DSPAOCGHxF5pmF4vzQP5AgPT3tiGRJRZu+0 github.com/k3s-io/helm-controller v0.11.3/go.mod h1:z0ExsRRIkTO/QC//3/Esn5ItTD6AiQSluwzMaS7RI/4= github.com/k3s-io/kine v0.8.0 h1:k6T9bI9DID7lIbktukXxg1QfeFoAQK4EIvAHoyPAe08= github.com/k3s-io/kine v0.8.0/go.mod h1:gaezUQ9c8iw8vxDV/DI8vc93h2rCpTvY37kMdYPMsyc= +github.com/k3s-io/klog v1.0.0-k3s1 h1:Bg+gRta3s4sfbaYUSWbHcMEyVdxdaU1cJCRtWcaxjBE= +github.com/k3s-io/klog v1.0.0-k3s1/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +github.com/k3s-io/klog/v2 v2.9.0-k3s1 h1:q4DqcZgBG+D2TSTGx6JcP1cuXeoxSoixSLE3casDcSw= +github.com/k3s-io/klog/v2 v2.9.0-k3s1/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= github.com/k3s-io/kubernetes v1.22.1-k3s1 h1:evi+JkMnC0HDEJ8A7aLqPKZ+0cP/CK6udFquf7KVq1I= github.com/k3s-io/kubernetes v1.22.1-k3s1/go.mod h1:IGQZrV02n2IBp52+/YwLVMurCEQPKXJ/k8hU3mqEOuA= github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.22.1-k3s1 h1:BlTOjlJwCV9ZNgTgyByLJXn7emlDAQC/duN14pUtWLk= @@ -1301,15 +1304,6 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= diff --git a/main.go b/main.go index 8f9de71b21..d7afe195fe 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,8 @@ package main import ( + "context" + "errors" "os" "github.com/rancher/k3s/pkg/cli/agent" @@ -36,7 +38,7 @@ func main() { ), } - if err := app.Run(configfilearg.MustParse(os.Args)); err != nil { + if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) { logrus.Fatal(err) } } diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index 9cefd4c649..df6dba8831 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -2,6 +2,7 @@ package flannel import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -96,7 +97,9 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) e go func() { err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet) - logrus.Fatalf("flannel exited: %v", err) + if err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatalf("flannel exited: %v", err) + } }() return nil diff --git a/pkg/cluster/https.go b/pkg/cluster/https.go index 23ac04f0a8..b3e17b8f07 100644 --- a/pkg/cluster/https.go +++ b/pkg/cluster/https.go @@ -3,6 +3,7 @@ package cluster import ( "context" "crypto/tls" + "errors" "log" "net" "net/http" @@ -85,7 +86,9 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error { // Start the supervisor http server on the tls listener go func() { err := server.Serve(listener) - logrus.Fatalf("server stopped: %v", err) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + logrus.Fatalf("server stopped: %v", err) + } }() // Shutdown the http server when the context is closed diff --git a/pkg/daemons/executor/embed.go b/pkg/daemons/executor/embed.go index d89639c9a3..d919b9cdc6 100644 --- a/pkg/daemons/executor/embed.go +++ b/pkg/daemons/executor/embed.go @@ -43,6 +43,11 @@ func (Embedded) Kubelet(args []string) error { command.SetArgs(args) go func() { + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("kubelet panic: %v", err) + } + }() logrus.Fatalf("kubelet exited: %v", command.Execute()) }() @@ -54,6 +59,11 @@ func (Embedded) KubeProxy(args []string) error { command.SetArgs(args) go func() { + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("kube-proxy panic: %v", err) + } + }() logrus.Fatalf("kube-proxy exited: %v", command.Execute()) }() @@ -66,6 +76,11 @@ func (Embedded) APIServer(ctx context.Context, etcdReady <-chan struct{}, args [ command.SetArgs(args) go func() { + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("apiserver panic: %v", err) + } + }() logrus.Fatalf("apiserver exited: %v", command.Execute()) }() @@ -79,6 +94,11 @@ func (Embedded) Scheduler(apiReady <-chan struct{}, args []string) error { go func() { <-apiReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("scheduler panic: %v", err) + } + }() logrus.Fatalf("scheduler exited: %v", command.Execute()) }() @@ -91,6 +111,11 @@ func (Embedded) ControllerManager(apiReady <-chan struct{}, args []string) error go func() { <-apiReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("controller-manager panic: %v", err) + } + }() logrus.Fatalf("controller-manager exited: %v", command.Execute()) }() @@ -129,6 +154,11 @@ func (Embedded) CloudControllerManager(ccmRBACReady <-chan struct{}, args []stri go func() { <-ccmRBACReady + defer func() { + if err := recover(); err != nil { + logrus.Fatalf("cloud-controller-manager panic: %v", err) + } + }() logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute()) }() diff --git a/pkg/daemons/executor/etcd.go b/pkg/daemons/executor/etcd.go index b7954f1d54..307ee9a9ff 100644 --- a/pkg/daemons/executor/etcd.go +++ b/pkg/daemons/executor/etcd.go @@ -3,6 +3,7 @@ package executor import ( + "context" "errors" "io/ioutil" "path/filepath" @@ -17,7 +18,7 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) { return InitialOptions{}, nil } -func (e Embedded) ETCD(args ETCDConfig) error { +func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error { configFile, err := args.ToConfigFile() if err != nil { return err @@ -42,7 +43,9 @@ func (e Embedded) ETCD(args ETCDConfig) error { logrus.Infof("this node has been removed from the cluster please restart %s to rejoin the cluster", version.Program) return } - + case <-ctx.Done(): + logrus.Infof("stopping etcd") + etcd.Close() case <-etcd.Server.StopNotify(): logrus.Fatalf("etcd stopped") case err := <-etcd.Err(): diff --git a/pkg/daemons/executor/executor.go b/pkg/daemons/executor/executor.go index 01a21080bc..25286013b3 100644 --- a/pkg/daemons/executor/executor.go +++ b/pkg/daemons/executor/executor.go @@ -26,7 +26,7 @@ type Executor interface { Scheduler(apiReady <-chan struct{}, args []string) error ControllerManager(apiReady <-chan struct{}, args []string) error CurrentETCDOptions() (InitialOptions, error) - ETCD(args ETCDConfig) error + ETCD(ctx context.Context, args ETCDConfig) error CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error } @@ -113,8 +113,8 @@ func CurrentETCDOptions() (InitialOptions, error) { return executor.CurrentETCDOptions() } -func ETCD(args ETCDConfig) error { - return executor.ETCD(args) +func ETCD(ctx context.Context, args ETCDConfig) error { + return executor.ETCD(ctx, args) } func CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error { diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 87222f9fd8..0867124e77 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -594,7 +594,7 @@ func (e *ETCD) metricsURL(expose bool) string { // cluster returns ETCDConfig for a cluster func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.InitialOptions) error { - return executor.ETCD(executor.ETCDConfig{ + return executor.ETCD(ctx, executor.ETCDConfig{ Name: e.name, InitialOptions: options, ForceNewCluster: forceNew, diff --git a/pkg/server/server.go b/pkg/server/server.go index a99aa7407a..bc4e0dbe01 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -172,7 +172,9 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err go func() { start(ctx) <-ctx.Done() - logrus.Fatal("controllers exited") + if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) { + logrus.Fatalf("controllers exited: %v", err) + } }() } else { go leader.RunOrDie(ctx, "", version.Program, sc.K8s, start) diff --git a/vendor/k8s.io/klog/klog.go b/vendor/k8s.io/klog/klog.go index 2712ce0afc..4227983522 100644 --- a/vendor/k8s.io/klog/klog.go +++ b/vendor/k8s.io/klog/klog.go @@ -812,30 +812,7 @@ func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoTo } } if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. + panic(string(data)) } l.putBuffer(buf) l.mu.Unlock() diff --git a/vendor/k8s.io/klog/klog_file.go b/vendor/k8s.io/klog/klog_file.go index e4010ad4df..458456a4a5 100644 --- a/vendor/k8s.io/klog/klog_file.go +++ b/vendor/k8s.io/klog/klog_file.go @@ -24,6 +24,7 @@ import ( "os" "os/user" "path/filepath" + "runtime" "strings" "sync" "time" @@ -55,13 +56,31 @@ func init() { host = shortHostname(h) } - current, err := user.Current() - if err == nil { - userName = current.Username - } + // On Windows, the Go 'user' package requires netapi32.dll. + // This affects Windows Nano Server: + // https://github.com/golang/go/issues/21867 + // Fallback to using environment variables. + if runtime.GOOS == "windows" { + u := os.Getenv("USERNAME") + if len(u) == 0 { + return + } + // Sanitize the USERNAME since it may contain filepath separators. + u = strings.Replace(u, `\`, "_", -1) - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) + // user.Current().Username normally produces something like 'USERDOMAIN\USERNAME' + d := os.Getenv("USERDOMAIN") + if len(d) != 0 { + userName = d + "_" + u + } else { + userName = u + } + } else { + current, err := user.Current() + if err == nil { + userName = current.Username + } + } } // shortHostname returns its argument, truncating at the first period. diff --git a/vendor/k8s.io/klog/v2/klog.go b/vendor/k8s.io/klog/v2/klog.go index 1e187f7635..5d3993e63c 100644 --- a/vendor/k8s.io/klog/v2/klog.go +++ b/vendor/k8s.io/klog/v2/klog.go @@ -965,28 +965,7 @@ func (l *loggingT) output(s severity, log logr.Logger, buf *buffer, depth int, f } } if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - trace := stacks(true) - // Write the stack trace for all goroutines to the stderr. - if l.toStderr || l.alsoToStderr || s >= l.stderrThreshold.get() || alsoToStderr { - os.Stderr.Write(trace) - } - // Write the stack trace for all goroutines to the files. - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. + panic(string(data)) } l.putBuffer(buf) l.mu.Unlock() diff --git a/vendor/modules.txt b/vendor/modules.txt index 9d3bea88dc..989cce4c8d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2323,10 +2323,10 @@ k8s.io/gengo/generator k8s.io/gengo/namer k8s.io/gengo/parser k8s.io/gengo/types -# k8s.io/klog v1.0.0 +# k8s.io/klog v1.0.0 => github.com/k3s-io/klog v1.0.0-k3s1 ## explicit k8s.io/klog -# k8s.io/klog/v2 v2.9.0 +# k8s.io/klog/v2 v2.9.0 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 ## explicit k8s.io/klog/v2 # k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1 @@ -3432,6 +3432,8 @@ sigs.k8s.io/yaml # k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.1-k3s1 # k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.1-k3s1 # k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.1-k3s1 +# k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1 +# k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 # k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1 # k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.1-k3s1 # k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.1-k3s1