From 697f7e471aafb678ab5c664a3f1957df683a63f6 Mon Sep 17 00:00:00 2001 From: Brian Downs Date: Mon, 18 Oct 2021 10:45:47 -0700 Subject: [PATCH] [Engine-1.21] - Add etcd s3 timeout (#4207) (#4229) --- pkg/cli/cmds/etcd_snapshot.go | 13 +++++++++++++ pkg/cli/cmds/server.go | 14 ++++++++++++++ pkg/cli/etcdsnapshot/etcd_snapshot.go | 2 ++ pkg/cli/server/server.go | 2 ++ pkg/daemons/config/types.go | 3 +++ pkg/etcd/etcd.go | 2 +- pkg/etcd/s3.go | 11 ++++------- 7 files changed, 39 insertions(+), 8 deletions(-) diff --git a/pkg/cli/cmds/etcd_snapshot.go b/pkg/cli/cmds/etcd_snapshot.go index 566a7ed5f8..1b5a777172 100644 --- a/pkg/cli/cmds/etcd_snapshot.go +++ b/pkg/cli/cmds/etcd_snapshot.go @@ -1,6 +1,8 @@ package cmds import ( + "time" + "github.com/rancher/k3s/pkg/version" "github.com/urfave/cli" ) @@ -77,6 +79,17 @@ var EtcdSnapshotFlags = []cli.Flag{ Usage: "(db) S3 folder", Destination: &ServerConfig.EtcdS3Folder, }, + &cli.BoolFlag{ + Name: "s3-insecure,etcd-s3-insecure", + Usage: "(db) Disables S3 over HTTPS", + Destination: &ServerConfig.EtcdS3Insecure, + }, + &cli.DurationFlag{ + Name: "s3-timeout,etcd-s3-timeout", + Usage: "(db) S3 timeout", + Destination: &ServerConfig.EtcdS3Timeout, + Value: 30 * time.Second, + }, } func NewEtcdSnapshotCommand(action func(*cli.Context) error, subcommands []cli.Command) cli.Command { diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 5627df8d36..550cc8ce2b 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -3,6 +3,7 @@ package cmds import ( "context" "sync" + "time" "github.com/rancher/k3s/pkg/version" "github.com/urfave/cli" @@ -89,6 +90,8 @@ type Server struct { EtcdS3BucketName string EtcdS3Region string EtcdS3Folder string + EtcdS3Timeout time.Duration + EtcdS3Insecure bool } var ( @@ -331,6 +334,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Usage: "(db) S3 folder", Destination: &ServerConfig.EtcdS3Folder, }, + &cli.BoolFlag{ + Name: "etcd-s3-insecure", + Usage: "(db) Disables S3 over HTTPS", + Destination: &ServerConfig.EtcdS3Insecure, + }, + &cli.DurationFlag{ + Name: "etcd-s3-timeout", + Usage: "(db) S3 timeout", + Destination: &ServerConfig.EtcdS3Timeout, + Value: 30 * time.Second, + }, cli.StringFlag{ Name: "default-local-storage-path", Usage: "(storage) Default local storage path for local provisioner storage class", diff --git a/pkg/cli/etcdsnapshot/etcd_snapshot.go b/pkg/cli/etcdsnapshot/etcd_snapshot.go index c2d23fff96..3dd058456b 100644 --- a/pkg/cli/etcdsnapshot/etcd_snapshot.go +++ b/pkg/cli/etcdsnapshot/etcd_snapshot.go @@ -49,6 +49,8 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, sc *server.Config) (string sc.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName sc.ControlConfig.EtcdS3Region = cfg.EtcdS3Region sc.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder + sc.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure + sc.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout sc.ControlConfig.Runtime = &config.ControlRuntime{} return server.ResolveDataDir(cfg.DataDir) diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index be9a988553..ff310aedd0 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -149,6 +149,8 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont serverConfig.ControlConfig.EtcdS3BucketName = cfg.EtcdS3BucketName serverConfig.ControlConfig.EtcdS3Region = cfg.EtcdS3Region serverConfig.ControlConfig.EtcdS3Folder = cfg.EtcdS3Folder + serverConfig.ControlConfig.EtcdS3Insecure = cfg.EtcdS3Insecure + serverConfig.ControlConfig.EtcdS3Timeout = cfg.EtcdS3Timeout } else { logrus.Info("ETCD snapshots are disabled") } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 35aaacf14d..94a36cb0c5 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -8,6 +8,7 @@ import ( "net/http" "sort" "strings" + "time" "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/wrangler-api/pkg/generated/controllers/core" @@ -166,6 +167,8 @@ type Control struct { EtcdS3BucketName string EtcdS3Region string EtcdS3Folder string + EtcdS3Timeout time.Duration + EtcdS3Insecure bool ServerNodeName string BindAddress string diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index e1a5caa7ad..f2dd4ea261 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -1059,7 +1059,7 @@ func (e *ETCD) DeleteSnapshots(ctx context.Context, snapshots []string) error { objectsCh := make(chan minio.ObjectInfo) - ctx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout) + ctx, cancel := context.WithTimeout(ctx, e.config.EtcdS3Timeout) defer cancel() go func() { diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 6c0c024992..1b4339dab2 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -14,7 +14,6 @@ import ( "path/filepath" "sort" "strings" - "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -23,8 +22,6 @@ import ( "github.com/sirupsen/logrus" ) -const defaultS3OpTimeout = time.Second * 30 - // S3 maintains state for S3 functionality. type S3 struct { config *config.Control @@ -71,7 +68,7 @@ func NewS3(ctx context.Context, config *config.Control) (*S3, error) { logrus.Infof("Checking if S3 bucket %s exists", config.EtcdS3BucketName) - ctx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout) + ctx, cancel := context.WithTimeout(ctx, config.EtcdS3Timeout) defer cancel() exists, err := c.BucketExists(ctx, config.EtcdS3BucketName) @@ -100,7 +97,7 @@ func (s *S3) upload(ctx context.Context, snapshot string) error { snapshotFileName = basename } - toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout) + toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) defer cancel() opts := minio.PutObjectOptions{ ContentType: "application/zip", @@ -124,7 +121,7 @@ func (s *S3) Download(ctx context.Context) error { } logrus.Debugf("retrieving snapshot: %s", remotePath) - toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout) + toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) defer cancel() r, err := s.client.GetObject(toCtx, s.config.EtcdS3BucketName, remotePath, minio.GetObjectOptions{}) @@ -178,7 +175,7 @@ func (s *S3) snapshotPrefix() string { func (s *S3) snapshotRetention(ctx context.Context) error { var snapshotFiles []minio.ObjectInfo - toCtx, cancel := context.WithTimeout(ctx, defaultS3OpTimeout) + toCtx, cancel := context.WithTimeout(ctx, s.config.EtcdS3Timeout) defer cancel() loo := minio.ListObjectsOptions{