[master] Add etcd extra args support for K3s (#4463)

* Add etcd extra args support for K3s

Signed-off-by: Chris Kim <oats87g@gmail.com>

* Add etcd custom argument integration test

Signed-off-by: Chris Kim <oats87g@gmail.com>

* go generate

Signed-off-by: Chris Kim <oats87g@gmail.com>

pkg/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go

@@ -1,4 +1,3 @@
-//go:build !ignore_autogenerated
 // +build !ignore_autogenerated
 
 /*

pkg/cli/cmds/server.go

@@ -48,6 +48,7 @@ type Server struct {
 	TLSSan                   cli.StringSlice
 	BindAddress              string
 	ExtraAPIArgs             cli.StringSlice
+	ExtraEtcdArgs            cli.StringSlice
 	ExtraSchedulerArgs       cli.StringSlice
 	ExtraControllerArgs      cli.StringSlice
 	ExtraCloudControllerArgs cli.StringSlice
@@ -128,6 +129,11 @@ var (
 		Usage: "(flags) Customized flag for kube-apiserver process",
 		Value: &ServerConfig.ExtraAPIArgs,
 	}
+	ExtraEtcdArgs = cli.StringSliceFlag{
+		Name:  "etcd-arg",
+		Usage: "(flags) Customized flag for etcd process",
+		Value: &ServerConfig.ExtraEtcdArgs,
+	}
 	ExtraSchedulerArgs = cli.StringSliceFlag{
 		Name:  "kube-scheduler-arg",
 		Usage: "(flags) Customized flag for kube-scheduler process",
@@ -214,6 +220,7 @@ var ServerFlags = []cli.Flag{
 		EnvVar:      version.ProgramUpper + "_KUBECONFIG_MODE",
 	},
 	ExtraAPIArgs,
+	ExtraEtcdArgs,
 	ExtraControllerArgs,
 	ExtraSchedulerArgs,
 	cli.StringSliceFlag{

pkg/cli/server/server.go

@@ -117,6 +117,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 	serverConfig.ControlConfig.APIServerBindAddress = cfg.APIServerBindAddress
 	serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs
 	serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs
+	serverConfig.ControlConfig.ExtraEtcdArgs = cfg.ExtraEtcdArgs
 	serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs
 	serverConfig.ControlConfig.ClusterDomain = cfg.ClusterDomain
 	serverConfig.ControlConfig.Datastore.Endpoint = cfg.DatastoreEndpoint

pkg/cluster/bootstrap.go

@@ -74,7 +74,7 @@ func (c *Cluster) Bootstrap(ctx context.Context, snapshot bool) error {
 					ElectionTimeout:   5000,
 					LogOutputs:        []string{"stderr"},
 				}
-				configFile, err := args.ToConfigFile()
+				configFile, err := args.ToConfigFile(c.config.ExtraEtcdArgs)
 				if err != nil {
 					return err
 				}

pkg/daemons/config/types.go

@@ -135,6 +135,7 @@ type Control struct {
 	ExtraAPIArgs             []string
 	ExtraControllerArgs      []string
 	ExtraCloudControllerArgs []string
+	ExtraEtcdArgs            []string
 	ExtraSchedulerAPIArgs    []string
 	NoLeaderElect            bool
 	JoinURL                  string

pkg/daemons/executor/etcd.go

@@ -19,8 +19,8 @@ func (e Embedded) CurrentETCDOptions() (InitialOptions, error) {
 	return InitialOptions{}, nil
 }
 
-func (e Embedded) ETCD(ctx context.Context, args ETCDConfig) error {
+func (e Embedded) ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error {
-	configFile, err := args.ToConfigFile()
+	configFile, err := args.ToConfigFile(extraArgs)
 	if err != nil {
 		return err
 	}

pkg/daemons/executor/executor.go

@@ -6,12 +6,15 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-
+	"strconv"
-	"sigs.k8s.io/yaml"
+	"strings"
+	"time"
 
 	"github.com/rancher/k3s/pkg/cli/cmds"
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
+	yaml2 "gopkg.in/yaml.v2"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"sigs.k8s.io/yaml"
 )
 
 var (
@@ -27,7 +30,7 @@ type Executor interface {
 	Scheduler(ctx context.Context, apiReady <-chan struct{}, args []string) error
 	ControllerManager(ctx context.Context, apiReady <-chan struct{}, args []string) error
 	CurrentETCDOptions() (InitialOptions, error)
-	ETCD(ctx context.Context, args ETCDConfig) error
+	ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error
 	CloudControllerManager(ctx context.Context, ccmRBACReady <-chan struct{}, args []string) error
 }
 
@@ -69,13 +72,53 @@ type InitialOptions struct {
 	State            string `json:"initial-cluster-state,omitempty"`
 }
 
-func (e ETCDConfig) ToConfigFile() (string, error) {
+func (e ETCDConfig) ToConfigFile(extraArgs []string) (string, error) {
 	confFile := filepath.Join(e.DataDir, "config")
 	bytes, err := yaml.Marshal(&e)
 	if err != nil {
 		return "", err
 	}
 
+	if len(extraArgs) > 0 {
+		var s map[string]interface{}
+		if err := yaml2.Unmarshal(bytes, &s); err != nil {
+			return "", err
+		}
+
+		for _, v := range extraArgs {
+			extraArg := strings.SplitN(v, "=", 2)
+			// Depending on the argV, we have different types to handle.
+			// Source: https://github.com/etcd-io/etcd/blob/44b8ae145b505811775f5af915dd19198d556d55/server/config/config.go#L36-L190 and https://etcd.io/docs/v3.5/op-guide/configuration/#configuration-file
+			if len(extraArg) == 2 {
+				key := strings.TrimLeft(extraArg[0], "-")
+				lowerKey := strings.ToLower(key)
+				var stringArr []string
+				if i, err := strconv.Atoi(extraArg[1]); err == nil {
+					s[key] = i
+				} else if time, err := time.ParseDuration(extraArg[1]); err == nil && (strings.Contains(lowerKey, "time") || strings.Contains(lowerKey, "duration") || strings.Contains(lowerKey, "interval") || strings.Contains(lowerKey, "retention")) {
+					// auto-compaction-retention is either a time.Duration or int, depending on version. If it is an int, it will be caught above.
+					s[key] = time
+				} else if err := yaml.Unmarshal([]byte(extraArg[1]), &stringArr); err == nil {
+					s[key] = stringArr
+				} else {
+					switch strings.ToLower(extraArg[1]) {
+					case "true":
+						s[key] = true
+					case "false":
+						s[key] = false
+					default:
+						s[key] = extraArg[1]
+					}
+				}
+			}
+		}
+
+		bytes, err = yaml2.Marshal(&s)
+		if err != nil {
+			return "", err
+		}
+	}
+
 	if err := os.MkdirAll(e.DataDir, 0700); err != nil {
 		return "", err
 	}
@@ -118,8 +161,8 @@ func CurrentETCDOptions() (InitialOptions, error) {
 	return executor.CurrentETCDOptions()
 }
 
-func ETCD(ctx context.Context, args ETCDConfig) error {
+func ETCD(ctx context.Context, args ETCDConfig, extraArgs []string) error {
-	return executor.ETCD(ctx, args)
+	return executor.ETCD(ctx, args, extraArgs)
 }
 
 func CloudControllerManager(ctx context.Context, ccmRBACReady <-chan struct{}, args []string) error {

pkg/deploy/zz_generated_bindata.go

@@ -12,7 +12,6 @@
 // manifests/metrics-server/resource-reader.yaml
 // manifests/rolebindings.yaml
 // manifests/traefik.yaml
-//go:build !no_stage
 // +build !no_stage
 
 package deploy

pkg/etcd/etcd.go

@@ -669,7 +669,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
 		HeartbeatInterval: 500,
 		Logger:            "zap",
 		LogOutputs:        []string{"stderr"},
-	})
+	}, e.config.ExtraEtcdArgs)
 }
 
 // RemovePeer removes a peer from the cluster. The peer name and IP address must both match.

tests/integration/custometcdargs/custometcdargs_int_test.go

@@ -0,0 +1,63 @@
+package integration
+
+import (
+	"os/exec"
+	"strings"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	"github.com/onsi/ginkgo/reporters"
+	. "github.com/onsi/gomega"
+	testutil "github.com/rancher/k3s/tests/util"
+)
+
+var customEtcdArgsServer *testutil.K3sServer
+var customEtcdArgsServerArgs = []string{
+	"--cluster-init",
+	"--etcd-arg quota-backend-bytes=858993459",
+}
+var _ = BeforeSuite(func() {
+	if !testutil.IsExistingServer() {
+		var err error
+		customEtcdArgsServer, err = testutil.K3sStartServer(customEtcdArgsServerArgs...)
+		Expect(err).ToNot(HaveOccurred())
+	}
+})
+
+var _ = Describe("custom etcd args", func() {
+	BeforeEach(func() {
+		if testutil.IsExistingServer() && !testutil.ServerArgsPresent(customEtcdArgsServerArgs) {
+			Skip("Test needs k3s server with: " + strings.Join(customEtcdArgsServerArgs, " "))
+		}
+	})
+	When("a custom quota backend bytes is specified", func() {
+		It("renders a config file with the correct entry", func() {
+			Eventually(func() (string, error) {
+				var cmd *exec.Cmd
+				grepCmd := "grep"
+				grepCmdArgs := []string{"quota-backend-bytes", "/var/lib/rancher/k3s/server/db/etcd/config"}
+				if testutil.IsRoot() {
+					cmd = exec.Command(grepCmd, grepCmdArgs...)
+				} else {
+					fullGrepCmd := append([]string{grepCmd}, grepCmdArgs...)
+					cmd = exec.Command("sudo", fullGrepCmd...)
+				}
+				byteOut, err := cmd.CombinedOutput()
+				return string(byteOut), err
+			}, "45s", "5s").Should(MatchRegexp(".*quota-backend-bytes: 858993459.*"))
+		})
+	})
+})
+
+var _ = AfterSuite(func() {
+	if !testutil.IsExistingServer() {
+		Expect(testutil.K3sKillServer(customEtcdArgsServer)).To(Succeed())
+	}
+})
+
+func Test_IntegrationCustomEtcdArgs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecsWithDefaultAndCustomReporters(t, "Custom etcd Arguments", []Reporter{
+		reporters.NewJUnitReporter("/tmp/results/junit-ls.xml"),
+	})
+}