Replace klog with non-exiting fork

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

cmd/agent/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"os"
 
 	"github.com/rancher/k3s/pkg/cli/agent"
@@ -16,8 +18,7 @@ func main() {
 		cmds.NewAgentCommand(agent.Run),
 	}
 
-	err := app.Run(configfilearg.MustParse(os.Args))
-	if err != nil {
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
 }

cmd/etcdsnapshot/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"os"
 
 	"github.com/rancher/k3s/pkg/cli/cmds"
@@ -22,7 +24,7 @@ func main() {
 		),
 	}
 
-	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
 }

cmd/k3s/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -53,7 +54,7 @@ func main() {
 		),
 	}
 
-	if err := app.Run(os.Args); err != nil {
+	if err := app.Run(os.Args); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
 }
@@ -90,7 +91,7 @@ func runCLIs(dataDir string) bool {
 	progName := filepath.Base(os.Args[0])
 	switch progName {
 	case "crictl", "ctr", "kubectl":
-		if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil {
+		if err := externalCLI(progName, dataDir, os.Args[1:]); err != nil && !errors.Is(err, context.Canceled) {
 			logrus.Fatal(err)
 		}
 		return true

cmd/server/main.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"os"
 	"path/filepath"
 
@@ -52,8 +54,7 @@ func main() {
 		),
 	}
 
-	err := app.Run(configfilearg.MustParse(os.Args))
-	if err != nil {
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
 }

go.mod

@@ -53,6 +53,8 @@ replace (
 	k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.1-k3s1
 	k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.1-k3s1
 	k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.1-k3s1
+	k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1 // k3s-release-1.x
+	k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1 // k3s-main
 	k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1
 	k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.1-k3s1
 	k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.1-k3s1

go.sum

@@ -320,7 +320,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
-github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
@@ -575,6 +574,10 @@ github.com/k3s-io/helm-controller v0.11.3 h1:DSPAOCGHxF5pmF4vzQP5AgPT3tiGRJRZu+0
 github.com/k3s-io/helm-controller v0.11.3/go.mod h1:z0ExsRRIkTO/QC//3/Esn5ItTD6AiQSluwzMaS7RI/4=
 github.com/k3s-io/kine v0.8.0 h1:k6T9bI9DID7lIbktukXxg1QfeFoAQK4EIvAHoyPAe08=
 github.com/k3s-io/kine v0.8.0/go.mod h1:gaezUQ9c8iw8vxDV/DI8vc93h2rCpTvY37kMdYPMsyc=
+github.com/k3s-io/klog v1.0.0-k3s1 h1:Bg+gRta3s4sfbaYUSWbHcMEyVdxdaU1cJCRtWcaxjBE=
+github.com/k3s-io/klog v1.0.0-k3s1/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+github.com/k3s-io/klog/v2 v2.9.0-k3s1 h1:q4DqcZgBG+D2TSTGx6JcP1cuXeoxSoixSLE3casDcSw=
+github.com/k3s-io/klog/v2 v2.9.0-k3s1/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 github.com/k3s-io/kubernetes v1.22.1-k3s1 h1:evi+JkMnC0HDEJ8A7aLqPKZ+0cP/CK6udFquf7KVq1I=
 github.com/k3s-io/kubernetes v1.22.1-k3s1/go.mod h1:IGQZrV02n2IBp52+/YwLVMurCEQPKXJ/k8hU3mqEOuA=
 github.com/k3s-io/kubernetes/staging/src/k8s.io/api v1.22.1-k3s1 h1:BlTOjlJwCV9ZNgTgyByLJXn7emlDAQC/duN14pUtWLk=
@@ -1301,15 +1304,6 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
-k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
-k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
-k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
-k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
-k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
-k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
-k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=

main.go

@@ -7,6 +7,8 @@
 package main
 
 import (
+	"context"
+	"errors"
 	"os"
 
 	"github.com/rancher/k3s/pkg/cli/agent"
@@ -36,7 +38,7 @@ func main() {
 		),
 	}
 
-	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil {
+	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
 }

pkg/agent/flannel/setup.go

@@ -2,6 +2,7 @@ package flannel
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -96,7 +97,9 @@ func Run(ctx context.Context, nodeConfig *config.Node, nodes v1.NodeInterface) e
 
 	go func() {
 		err := flannel(ctx, nodeConfig.FlannelIface, nodeConfig.FlannelConf, nodeConfig.AgentConfig.KubeConfigKubelet)
-		logrus.Fatalf("flannel exited: %v", err)
+		if err != nil && !errors.Is(err, context.Canceled) {
+			logrus.Fatalf("flannel exited: %v", err)
+		}
 	}()
 
 	return nil

pkg/cluster/https.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"log"
 	"net"
 	"net/http"
@@ -85,7 +86,9 @@ func (c *Cluster) initClusterAndHTTPS(ctx context.Context) error {
 	// Start the supervisor http server on the tls listener
 	go func() {
 		err := server.Serve(listener)
-		logrus.Fatalf("server stopped: %v", err)
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			logrus.Fatalf("server stopped: %v", err)
+		}
 	}()
 
 	// Shutdown the http server when the context is closed

pkg/daemons/executor/embed.go

@@ -43,6 +43,11 @@ func (Embedded) Kubelet(args []string) error {
 	command.SetArgs(args)
 
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("kubelet panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("kubelet exited: %v", command.Execute())
 	}()
 
@@ -54,6 +59,11 @@ func (Embedded) KubeProxy(args []string) error {
 	command.SetArgs(args)
 
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("kube-proxy panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("kube-proxy exited: %v", command.Execute())
 	}()
 
@@ -66,6 +76,11 @@ func (Embedded) APIServer(ctx context.Context, etcdReady <-chan struct{}, args [
 	command.SetArgs(args)
 
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("apiserver panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("apiserver exited: %v", command.Execute())
 	}()
 
@@ -79,6 +94,11 @@ func (Embedded) Scheduler(apiReady <-chan struct{}, args []string) error {
 
 	go func() {
 		<-apiReady
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("scheduler panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("scheduler exited: %v", command.Execute())
 	}()
 
@@ -91,6 +111,11 @@ func (Embedded) ControllerManager(apiReady <-chan struct{}, args []string) error
 
 	go func() {
 		<-apiReady
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("controller-manager panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("controller-manager exited: %v", command.Execute())
 	}()
 
@@ -129,6 +154,11 @@ func (Embedded) CloudControllerManager(ccmRBACReady <-chan struct{}, args []stri
 
 	go func() {
 		<-ccmRBACReady
+		defer func() {
+			if err := recover(); err != nil {
+				logrus.Fatalf("cloud-controller-manager panic: %v", err)
+			}
+		}()
 		logrus.Fatalf("cloud-controller-manager exited: %v", command.Execute())
 	}()
 

pkg/daemons/executor/etcd.go

@@ -3,6 +3,7 @@
 package executor
 
 import (
+	"context"
 	"errors"
 	"io/ioutil"
 	"path/filepath"
@@ -17,7 +18,7 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) {
 	return InitialOptions{}, nil
 }
 
-func (e Embedded) ETCD(args ETCDConfig) error {
+func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error {
 	configFile, err := args.ToConfigFile()
 	if err != nil {
 		return err
@@ -42,7 +43,9 @@ func (e Embedded) ETCD(args ETCDConfig) error {
 				logrus.Infof("this node has been removed from the cluster please restart %s to rejoin the cluster", version.Program)
 				return
 			}
-
+		case <-ctx.Done():
+			logrus.Infof("stopping etcd")
+			etcd.Close()
 		case <-etcd.Server.StopNotify():
 			logrus.Fatalf("etcd stopped")
 		case err := <-etcd.Err():

pkg/daemons/executor/executor.go

@@ -26,7 +26,7 @@ type Executor interface {
 	Scheduler(apiReady <-chan struct{}, args []string) error
 	ControllerManager(apiReady <-chan struct{}, args []string) error
 	CurrentETCDOptions() (InitialOptions, error)
-	ETCD(args ETCDConfig) error
+	ETCD(ctx context.Context, args ETCDConfig) error
 	CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error
 }
 
@@ -113,8 +113,8 @@ func CurrentETCDOptions() (InitialOptions, error) {
 	return executor.CurrentETCDOptions()
 }
 
-func ETCD(args ETCDConfig) error {
-	return executor.ETCD(args)
+func ETCD(ctx context.Context, args ETCDConfig) error {
+	return executor.ETCD(ctx, args)
 }
 
 func CloudControllerManager(ccmRBACReady <-chan struct{}, args []string) error {

pkg/etcd/etcd.go

@@ -594,7 +594,7 @@ func (e *ETCD) metricsURL(expose bool) string {
 
 // cluster returns ETCDConfig for a cluster
 func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.InitialOptions) error {
-	return executor.ETCD(executor.ETCDConfig{
+	return executor.ETCD(ctx, executor.ETCDConfig{
 		Name:                e.name,
 		InitialOptions:      options,
 		ForceNewCluster:     forceNew,

pkg/server/server.go

@@ -172,7 +172,9 @@ func runControllers(ctx context.Context, wg *sync.WaitGroup, config *Config) err
 		go func() {
 			start(ctx)
 			<-ctx.Done()
-			logrus.Fatal("controllers exited")
+			if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) {
+				logrus.Fatalf("controllers exited: %v", err)
+			}
 		}()
 	} else {
 		go leader.RunOrDie(ctx, "", version.Program, sc.K8s, start)

vendor/k8s.io/klog/klog.go

@@ -812,30 +812,7 @@ func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoTo
 		}
 	}
 	if s == fatalLog {
-		// If we got here via Exit rather than Fatal, print no stacks.
-		if atomic.LoadUint32(&fatalNoStacks) > 0 {
-			l.mu.Unlock()
-			timeoutFlush(10 * time.Second)
-			os.Exit(1)
-		}
-		// Dump all goroutine stacks before exiting.
-		// First, make sure we see the trace for the current goroutine on standard error.
-		// If -logtostderr has been specified, the loop below will do that anyway
-		// as the first stack in the full dump.
-		if !l.toStderr {
-			os.Stderr.Write(stacks(false))
-		}
-		// Write the stack trace for all goroutines to the files.
-		trace := stacks(true)
-		logExitFunc = func(error) {} // If we get a write error, we'll still exit below.
-		for log := fatalLog; log >= infoLog; log-- {
-			if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set.
-				f.Write(trace)
-			}
-		}
-		l.mu.Unlock()
-		timeoutFlush(10 * time.Second)
-		os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway.
+		panic(string(data))
 	}
 	l.putBuffer(buf)
 	l.mu.Unlock()

vendor/k8s.io/klog/klog_file.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -55,13 +56,31 @@ func init() {
 		host = shortHostname(h)
 	}
 
-	current, err := user.Current()
-	if err == nil {
-		userName = current.Username
-	}
+	// On Windows, the Go 'user' package requires netapi32.dll.
+	// This affects Windows Nano Server:
+	//   https://github.com/golang/go/issues/21867
+	// Fallback to using environment variables.
+	if runtime.GOOS == "windows" {
+		u := os.Getenv("USERNAME")
+		if len(u) == 0 {
+			return
+		}
+		// Sanitize the USERNAME since it may contain filepath separators.
+		u = strings.Replace(u, `\`, "_", -1)
 
-	// Sanitize userName since it may contain filepath separators on Windows.
-	userName = strings.Replace(userName, `\`, "_", -1)
+		// user.Current().Username normally produces something like 'USERDOMAIN\USERNAME'
+		d := os.Getenv("USERDOMAIN")
+		if len(d) != 0 {
+			userName = d + "_" + u
+		} else {
+			userName = u
+		}
+	} else {
+		current, err := user.Current()
+		if err == nil {
+			userName = current.Username
+		}
+	}
 }
 
 // shortHostname returns its argument, truncating at the first period.

vendor/k8s.io/klog/v2/klog.go

@@ -965,28 +965,7 @@ func (l *loggingT) output(s severity, log logr.Logger, buf *buffer, depth int, f
 		}
 	}
 	if s == fatalLog {
-		// If we got here via Exit rather than Fatal, print no stacks.
-		if atomic.LoadUint32(&fatalNoStacks) > 0 {
-			l.mu.Unlock()
-			timeoutFlush(10 * time.Second)
-			os.Exit(1)
-		}
-		// Dump all goroutine stacks before exiting.
-		trace := stacks(true)
-		// Write the stack trace for all goroutines to the stderr.
-		if l.toStderr || l.alsoToStderr || s >= l.stderrThreshold.get() || alsoToStderr {
-			os.Stderr.Write(trace)
-		}
-		// Write the stack trace for all goroutines to the files.
-		logExitFunc = func(error) {} // If we get a write error, we'll still exit below.
-		for log := fatalLog; log >= infoLog; log-- {
-			if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set.
-				f.Write(trace)
-			}
-		}
-		l.mu.Unlock()
-		timeoutFlush(10 * time.Second)
-		os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway.
+		panic(string(data))
 	}
 	l.putBuffer(buf)
 	l.mu.Unlock()

vendor/modules.txt

@@ -2323,10 +2323,10 @@ k8s.io/gengo/generator
 k8s.io/gengo/namer
 k8s.io/gengo/parser
 k8s.io/gengo/types
-# k8s.io/klog v1.0.0
+# k8s.io/klog v1.0.0 => github.com/k3s-io/klog v1.0.0-k3s1
 ## explicit
 k8s.io/klog
-# k8s.io/klog/v2 v2.9.0
+# k8s.io/klog/v2 v2.9.0 => github.com/k3s-io/klog/v2 v2.9.0-k3s1
 ## explicit
 k8s.io/klog/v2
 # k8s.io/kube-aggregator v0.18.0 => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1
@@ -3432,6 +3432,8 @@ sigs.k8s.io/yaml
 # k8s.io/controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/controller-manager v1.22.1-k3s1
 # k8s.io/cri-api => github.com/k3s-io/kubernetes/staging/src/k8s.io/cri-api v1.22.1-k3s1
 # k8s.io/csi-translation-lib => github.com/k3s-io/kubernetes/staging/src/k8s.io/csi-translation-lib v1.22.1-k3s1
+# k8s.io/klog => github.com/k3s-io/klog v1.0.0-k3s1
+# k8s.io/klog/v2 => github.com/k3s-io/klog/v2 v2.9.0-k3s1
 # k8s.io/kube-aggregator => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-aggregator v1.22.1-k3s1
 # k8s.io/kube-controller-manager => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-controller-manager v1.22.1-k3s1
 # k8s.io/kube-proxy => github.com/k3s-io/kubernetes/staging/src/k8s.io/kube-proxy v1.22.1-k3s1