Galal hussein etcd backup restore (#2154)

* Add etcd snapshot and restore Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix error logs Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * goimports Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * fix flag describtion Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * Add disable snapshot and retention Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * use creation time for snapshot retention Signed-off-by: galal-hussein <hussein.galal.ahmed.11@gmail.com> * unexport method, update var name Signed-off-by: Brian Downs <brian.downs@gmail.com> * adjust snapshot flags Signed-off-by: Brian Downs <brian.downs@gmail.com> * update var name, string concat Signed-off-by: Brian Downs <brian.downs@gmail.com> * revert previous change, create constants Signed-off-by: Brian Downs <brian.downs@gmail.com> * update Signed-off-by: Brian Downs <brian.downs@gmail.com> * updates Signed-off-by: Brian Downs <brian.downs@gmail.com> * type assertion error checking Signed-off-by: Brian Downs <brian.downs@gmail.com> * update Signed-off-by: Brian Downs <brian.downs@gmail.com> * update Signed-off-by: Brian Downs <brian.downs@gmail.com> * update Signed-off-by: Brian Downs <brian.downs@gmail.com> * pr remediation Signed-off-by: Brian Downs <brian.downs@gmail.com> * pr remediation Signed-off-by: Brian Downs <brian.downs@gmail.com> * pr remediation Signed-off-by: Brian Downs <brian.downs@gmail.com> * pr remediation Signed-off-by: Brian Downs <brian.downs@gmail.com> * pr remediation Signed-off-by: Brian Downs <brian.downs@gmail.com> * updates Signed-off-by: Brian Downs <brian.downs@gmail.com> * updates Signed-off-by: Brian Downs <brian.downs@gmail.com> * simplify logic, remove unneeded function Signed-off-by: Brian Downs <brian.downs@gmail.com> * update flags Signed-off-by: Brian Downs <brian.downs@gmail.com> * update flags Signed-off-by: Brian Downs <brian.downs@gmail.com> * add comment Signed-off-by: Brian Downs <brian.downs@gmail.com> * exit on restore completion, update flag names, move retention check Signed-off-by: Brian Downs <brian.downs@gmail.com> * exit on restore completion, update flag names, move retention check Signed-off-by: Brian Downs <brian.downs@gmail.com> * exit on restore completion, update flag names, move retention check Signed-off-by: Brian Downs <brian.downs@gmail.com> * update disable snapshots flag and field names Signed-off-by: Brian Downs <brian.downs@gmail.com> * move function Signed-off-by: Brian Downs <brian.downs@gmail.com> * update field names Signed-off-by: Brian Downs <brian.downs@gmail.com> * update var and field names Signed-off-by: Brian Downs <brian.downs@gmail.com> * update var and field names Signed-off-by: Brian Downs <brian.downs@gmail.com> * update defaultSnapshotIntervalMinutes to 12 like rke Signed-off-by: Brian Downs <brian.downs@gmail.com> * update directory perms Signed-off-by: Brian Downs <brian.downs@gmail.com> * update etc-snapshot-dir usage Signed-off-by: Brian Downs <brian.downs@gmail.com> * update interval to 12 hours Signed-off-by: Brian Downs <brian.downs@gmail.com> * fix usage typo Signed-off-by: Brian Downs <brian.downs@gmail.com> * add cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * add cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * add cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * wire in cron Signed-off-by: Brian Downs <brian.downs@gmail.com> * update deps target to work, add build/data target for creation, and generate Signed-off-by: Brian Downs <brian.downs@gmail.com> * remove dead make targets Signed-off-by: Brian Downs <brian.downs@gmail.com> * error handling, cluster reset functionality Signed-off-by: Brian Downs <brian.downs@gmail.com> * error handling, cluster reset functionality Signed-off-by: Brian Downs <brian.downs@gmail.com> * update Signed-off-by: Brian Downs <brian.downs@gmail.com> * remove intermediate dapper file Signed-off-by: Brian Downs <brian.downs@gmail.com> Co-authored-by: galal-hussein <hussein.galal.ahmed.11@gmail.com>
2020-08-28 16:57:40 -07:00 · 2020-08-28 16:57:40 -07:00 · 866dc94cea
parent cdce2b7e9a
commit 866dc94cea
27 changed files with 2429 additions and 17 deletions
--- a/19
+++ b/19
@ -10,13 +10,10 @@ TARGETS := $(shell ls scripts | grep -v \\.sh)
 $(TARGETS): .dapper
 	./.dapper $@

-trash: .dapper
-	./.dapper -m bind trash
-
-trash-keep: .dapper
-	./.dapper -m bind trash -k
-
-deps: trash
+.PHONY: deps
+deps:
+	go mod vendor
+	go mod tidy

 release:
 	./scripts/release.sh
@ -24,3 +21,11 @@ release:
 .DEFAULT_GOAL := ci

 .PHONY: $(TARGETS)
+
+.PHONY: generate
+generate: build/data 
+	./scripts/download
+	go generate
+
+build/data:
+	mkdir -p $@
--- a/go.mod
+++ b/go.mod
@ -103,6 +103,7 @@ require (
 	github.com/rancher/spur v0.0.0-20200617165101-8702c8e4ce7a
 	github.com/rancher/wrangler v0.6.1
 	github.com/rancher/wrangler-api v0.6.0
+	github.com/robfig/cron/v3 v3.0.1
 	github.com/rootless-containers/rootlesskit v0.10.0
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/pflag v1.0.5
--- a/go.sum
+++ b/go.sum
@ -712,6 +712,8 @@ github.com/rancher/wrangler-api v0.6.0/go.mod h1:RbuDkPNHhxcXuwAbLVvEAhH+UPAh+MI
 github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
 github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY=
 github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
+github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
+github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
--- a/pkg/agent/netpol/netpol_windows.go
+++ b/pkg/agent/netpol/netpol_windows.go
@ -2,6 +2,7 @@ package netpol

 import (
 	"context"
+
 	daemonconfig "github.com/rancher/k3s/pkg/daemons/config"
 )

--- a/pkg/cli/cmds/server.go
+++ b/pkg/cli/cmds/server.go
@ -11,6 +11,9 @@ import (

 const (
 	DisableItems = "coredns, servicelb, traefik, local-storage, metrics-server"
+
+	defaultSnapshotRentention    = 5
+	defaultSnapshotIntervalHours = 12
 )

 type Server struct {
@ -56,8 +59,13 @@ type Server struct {
 	DisableKubeProxy         bool
 	ClusterInit              bool
 	ClusterReset             bool
+	ClusterResetRestorePath  string
 	EncryptSecrets           bool
 	StartupHooks             []func(context.Context, config.Control) error
+	EtcdDisableSnapshots     bool
+	EtcdSnapshotDir          string
+	EtcdSnapshotCron         string
+	EtcdSnapshotRetention    int
 }

 var ServerConfig Server
@ -210,6 +218,28 @@ func NewServerCommand(action func(*cli.Context) error) *cli.Command {
 				Destination: &ServerConfig.DatastoreKeyFile,
 				EnvVars:     []string{version.ProgramUpper + "_DATASTORE_KEYFILE"},
 			},
+			&cli.BoolFlag{
+				Name:        "etcd-disable-snapshots",
+				Usage:       "(db) Disable automatic etcd snapshots",
+				Destination: &ServerConfig.EtcdDisableSnapshots,
+			},
+			&cli.StringFlag{
+				Name:        "etcd-snapshot-schedule-cron",
+				Usage:       "(db) Snapshot interval time in cron spec. eg. every 5 hours '* */5 * * *'",
+				Destination: &ServerConfig.EtcdSnapshotCron,
+				Value:       "0 */12 * * *",
+			},
+			&cli.IntFlag{
+				Name:        "etcd-snapshot-retention",
+				Usage:       "(db) Number of snapshots to retain",
+				Destination: &ServerConfig.EtcdSnapshotRetention,
+				Value:       defaultSnapshotRentention,
+			},
+			&cli.StringFlag{
+				Name:        "etcd-snapshot-dir",
+				Usage:       "(db) Directory to save db snapshots. (Default location: ${data-dir}/db/snapshots)",
+				Destination: &ServerConfig.EtcdSnapshotDir,
+			},
 			&cli.StringFlag{
 				Name:        "default-local-storage-path",
 				Usage:       "(storage) Default local storage path for local provisioner storage class",
@ -294,6 +324,11 @@ func NewServerCommand(action func(*cli.Context) error) *cli.Command {
 				EnvVars:     []string{version.ProgramUpper + "_CLUSTER_RESET"},
 				Destination: &ServerConfig.ClusterReset,
 			},
+			&cli.StringFlag{
+				Name:        "cluster-reset-restore-path",
+				Usage:       "(db) Path to snapshot file to be restored",
+				Destination: &ServerConfig.ClusterResetRestorePath,
+			},
 			&cli.BoolFlag{
 				Name:        "secrets-encryption",
 				Usage:       "(experimental) Enable Secret encryption at rest",
--- a/pkg/cli/server/server.go
+++ b/pkg/cli/server/server.go
@ -107,8 +107,18 @@ func run(app *cli.Context, cfg *cmds.Server) error {
 	serverConfig.ControlConfig.DisableNPC = cfg.DisableNPC
 	serverConfig.ControlConfig.DisableKubeProxy = cfg.DisableKubeProxy
 	serverConfig.ControlConfig.ClusterInit = cfg.ClusterInit
-	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
 	serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
+	serverConfig.ControlConfig.EtcdSnapshotCron = cfg.EtcdSnapshotCron
+	serverConfig.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
+	serverConfig.ControlConfig.EtcdSnapshotRetention = cfg.EtcdSnapshotRetention
+	serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
+
+	if cfg.ClusterResetRestorePath != "" && !cfg.ClusterReset {
+		return errors.New("Invalid flag use. --cluster-reset required with --cluster-reset-restore-path")
+	}
+
+	serverConfig.ControlConfig.ClusterReset = cfg.ClusterReset
+	serverConfig.ControlConfig.ClusterResetRestorePath = cfg.ClusterResetRestorePath

 	if serverConfig.ControlConfig.SupervisorPort == 0 {
 		serverConfig.ControlConfig.SupervisorPort = serverConfig.ControlConfig.HTTPSPort
--- a/pkg/cluster/etcd.go
+++ b/pkg/cluster/etcd.go
@ -8,5 +8,5 @@ import (
 )

 func init() {
-	managed.RegisterDriver(&etcd.ETCD{})
+	managed.RegisterDriver(etcd.NewETCD())
 }
--- a/pkg/cluster/managed/drivers.go
+++ b/pkg/cluster/managed/drivers.go
@ -20,6 +20,7 @@ type Driver interface {
 	Reset(ctx context.Context, clientAccessInfo *clientaccess.Info) error
 	Start(ctx context.Context, clientAccessInfo *clientaccess.Info) error
 	Test(ctx context.Context, clientAccessInfo *clientaccess.Info) error
+	Restore(ctx context.Context) error
 	EndpointName() string
 }

--- a/pkg/daemons/config/types.go
+++ b/pkg/daemons/config/types.go
@ -126,9 +126,14 @@ type Control struct {
 	DisableKubeProxy         bool
 	ClusterInit              bool
 	ClusterReset             bool
+	ClusterResetRestorePath  string
 	EncryptSecrets           bool
 	TLSMinVersion            uint16
 	TLSCipherSuites          []uint16
+	EtcdDisableSnapshots     bool
+	EtcdSnapshotDir          string
+	EtcdSnapshotCron         string
+	EtcdSnapshotRetention    int

 	BindAddress string
 	SANs        []string
--- a/pkg/etcd/etcd.go
+++ b/pkg/etcd/etcd.go
@ -11,6 +11,8 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"time"

@ -21,8 +23,10 @@ import (
 	"github.com/rancher/k3s/pkg/clientaccess"
 	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/k3s/pkg/daemons/executor"
+	"github.com/robfig/cron/v3"
 	"github.com/sirupsen/logrus"
 	etcd "go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3/snapshot"
 	"go.etcd.io/etcd/etcdserver/etcdserverpb"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 )
@ -33,20 +37,39 @@ type ETCD struct {
 	name    string
 	runtime *config.ControlRuntime
 	address string
+	cron    *cron.Cron
 }

+// NewETCD creates a new value of type
+// ETCD with an initialized cron value.
+func NewETCD() *ETCD {
+	return &ETCD{
+		cron: cron.New(),
+	}
+}
+
+const (
+	snapshotPrefix = "etcd-snapshot-"
+	endpoint       = "https://127.0.0.1:2379"
+
+	testTimeout = time.Second * 10
+)
+
+// Members contains a slice that holds all
+// members of the cluster.
 type Members struct {
 	Members []*etcdserverpb.Member `json:"members"`
 }

+// EndpointName returns the name of the endpoint.
 func (e *ETCD) EndpointName() string {
 	return "etcd"
 }

 func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) error {
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	ctx, cancel := context.WithTimeout(ctx, testTimeout)
 	defer cancel()
-	status, err := e.client.Status(ctx, "https://127.0.0.1:2379")
+	status, err := e.client.Status(ctx, endpoint)
 	if err != nil {
 		return err
 	}
@ -61,7 +84,7 @@ func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) er
 		return err
 	}

-	var cluster []string
+	var memberNameUrls []string
 	for _, member := range members.Members {
 		for _, peerURL := range member.PeerURLs {
 			if peerURL == e.peerURL() && e.name == member.Name {
@ -69,10 +92,10 @@ func (e *ETCD) Test(ctx context.Context, clientAccessInfo *clientaccess.Info) er
 			}
 		}
 		if len(member.PeerURLs) > 0 {
-			cluster = append(cluster, fmt.Sprintf("%s=%s", member.Name, member.PeerURLs[0]))
+			memberNameUrls = append(memberNameUrls, member.Name+"="+member.PeerURLs[0])
 		}
 	}
-	msg := fmt.Sprintf("This server is a not a member of the etcd cluster "+"found %v and expecting to contain %s=%s", cluster, e.name, e.address)
+	msg := fmt.Sprintf("This server is a not a member of the etcd cluster. Found %v, expect: %s=%s", memberNameUrls, e.name, e.address)
 	logrus.Error(msg)
 	return fmt.Errorf(msg)
 }
@ -117,6 +140,16 @@ func (e *ETCD) Reset(ctx context.Context, clientAccessInfo *clientaccess.Info) e
 			}
 		}
 	}()
+	if e.config.ClusterResetRestorePath != "" {
+		info, err := os.Stat(e.config.ClusterResetRestorePath)
+		if os.IsNotExist(err) {
+			return fmt.Errorf("etcd: snapshot path does not exist: %s", e.config.ClusterResetRestorePath)
+		}
+		if info.IsDir() {
+			return fmt.Errorf("etcd: snapshot path is directory: %s", e.config.ClusterResetRestorePath)
+		}
+		return e.Restore(ctx)
+	}
 	return e.newCluster(ctx, true)
 }

@ -131,6 +164,11 @@ func (e *ETCD) Start(ctx context.Context, clientAccessInfo *clientaccess.Info) e
 		return nil
 	}

+	if !e.config.EtcdDisableSnapshots {
+		e.setSnapshotFunction(ctx)
+		e.cron.Start()
+	}
+
 	if existingCluster {
 		opt, err := executor.CurrentETCDOptions()
 		if err != nil {
@ -229,7 +267,7 @@ func (e *ETCD) Register(ctx context.Context, config *config.Control, l net.Liste
 	}
 	e.address = address

-	e.config.Datastore.Endpoint = "https://127.0.0.1:2379"
+	e.config.Datastore.Endpoint = endpoint
 	e.config.Datastore.Config.CAFile = e.runtime.ETCDServerCA
 	e.config.Datastore.Config.CertFile = e.runtime.ClientETCDCert
 	e.config.Datastore.Config.KeyFile = e.runtime.ClientETCDKey
@ -317,7 +355,7 @@ func newClient(ctx context.Context, runtime *config.ControlRuntime) (*etcd.Clien

 	cfg := etcd.Config{
 		Context:   ctx,
-		Endpoints: []string{"https://127.0.0.1:2379"},
+		Endpoints: []string{endpoint},
 		TLS:       tlsConfig,
 	}

@ -376,7 +414,7 @@ func (e *ETCD) cluster(ctx context.Context, forceNew bool, options executor.Init
 		InitialOptions:      options,
 		ForceNewCluster:     forceNew,
 		ListenClientURLs:    fmt.Sprintf(e.clientURL() + ",https://127.0.0.1:2379"),
-		ListenMetricsURLs:   fmt.Sprintf("http://127.0.0.1:2381"),
+		ListenMetricsURLs:   "http://127.0.0.1:2381",
 		ListenPeerURLs:      e.peerURL(),
 		AdvertiseClientURLs: e.clientURL(),
 		DataDir:             dataDir(e.config),
@ -480,3 +518,130 @@ func (e *ETCD) clientURLs(ctx context.Context, clientAccessInfo *clientaccess.In
 	}
 	return clientURLs, memberList, nil
 }
+
+func snapshotDir(config *config.Control) (string, error) {
+	if config.EtcdSnapshotDir == "" {
+		// we have to create the snapshot dir if we are using
+		// the default snapshot dir if it doesn't exist
+		defaultSnapshotDir := filepath.Join(config.DataDir, "db", "snapshots")
+		s, err := os.Stat(defaultSnapshotDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				if err := os.MkdirAll(defaultSnapshotDir, 0700); err != nil {
+					return "", err
+				}
+				return defaultSnapshotDir, nil
+			}
+			return "", err
+		}
+		if s.IsDir() {
+			return defaultSnapshotDir, nil
+		}
+	}
+	return config.EtcdSnapshotDir, nil
+}
+
+func (e *ETCD) snapshot(ctx context.Context) {
+	snapshotTime := time.Now()
+	logrus.Infof("Snapshot retention check")
+	snapshotDir, err := snapshotDir(e.config)
+	if err != nil {
+		logrus.Errorf("failed to get the snapshot dir: %v", err)
+		return
+	}
+	logrus.Infof("Taking etcd snapshot at %s", snapshotTime.String())
+	sManager := snapshot.NewV3(nil)
+	tlsConfig, err := toTLSConfig(e.runtime)
+	if err != nil {
+		logrus.Errorf("failed to get tls config for etcd: %v", err)
+		return
+	}
+	etcdConfig := etcd.Config{
+		Context:   ctx,
+		Endpoints: []string{endpoint},
+		TLS:       tlsConfig,
+	}
+	snapshotPath := filepath.Join(snapshotDir, snapshotPrefix+strconv.Itoa(int(snapshotTime.Unix())))
+
+	if err := sManager.Save(ctx, etcdConfig, snapshotPath); err != nil {
+		logrus.Errorf("failed to save snapshot %s: %v", snapshotPath, err)
+		return
+	}
+	if err := snapshotRetention(e.config.EtcdSnapshotRetention, snapshotDir); err != nil {
+		logrus.Errorf("failed to apply snapshot retention: %v", err)
+		return
+	}
+}
+
+// snapshot performs an ETCD snapshot at the given interval and
+// saves the file to either the default snapshot directory or
+// the user provided directory.
+func (e *ETCD) setSnapshotFunction(ctx context.Context) {
+	e.cron.AddFunc(e.config.EtcdSnapshotCron, func() { e.snapshot(ctx) })
+}
+
+// Restore performs a restore of the ETCD datastore from
+// the given snapshot path. This operation exists upon
+// completion.
+func (e *ETCD) Restore(ctx context.Context) error {
+	// check the old etcd data dir
+	oldDataDir := dataDir(e.config) + "-old"
+	if s, err := os.Stat(oldDataDir); err == nil && s.IsDir() {
+		logrus.Infof("etcd already restored from a snapshot. Restart without --snapshot-restore-path flag. Backup and delete ${datadir}/server/db on each peer etcd server and rejoin the nodes")
+		os.Exit(0)
+	} else if os.IsNotExist(err) {
+		if e.config.ClusterResetRestorePath == "" {
+			return errors.New("no etcd restore path was specified")
+		}
+		// make sure snapshot exists before restoration
+		if _, err := os.Stat(e.config.ClusterResetRestorePath); err != nil {
+			return err
+		}
+		// move the data directory to a temp path
+		if err := os.Rename(dataDir(e.config), oldDataDir); err != nil {
+			return err
+		}
+		sManager := snapshot.NewV3(nil)
+		if err := sManager.Restore(snapshot.RestoreConfig{
+			SnapshotPath:   e.config.ClusterResetRestorePath,
+			Name:           e.name,
+			OutputDataDir:  dataDir(e.config),
+			OutputWALDir:   walDir(e.config),
+			PeerURLs:       []string{e.peerURL()},
+			InitialCluster: e.name + "=" + e.peerURL(),
+		}); err != nil {
+			return err
+		}
+	} else {
+		return err
+	}
+	if err := e.setName(); err != nil {
+		return err
+	}
+
+	return e.newCluster(ctx, true)
+}
+
+// snapshotRetention iterates through the snapshots and removes the oldest
+// leaving the desired number of snapshots.
+func snapshotRetention(retention int, snapshotDir string) error {
+	var snapshotFiles []os.FileInfo
+	if err := filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if strings.HasPrefix(info.Name(), snapshotPrefix) {
+			snapshotFiles = append(snapshotFiles, info)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+	if len(snapshotFiles) <= retention {
+		return nil
+	}
+	sort.Slice(snapshotFiles, func(i, j int) bool {
+		return snapshotFiles[i].Name() < snapshotFiles[j].Name()
+	})
+	return os.Remove(filepath.Join(snapshotDir, snapshotFiles[0].Name()))
+}
--- a/vendor/github.com/robfig/cron/v3/.gitignore
+++ b/vendor/github.com/robfig/cron/v3/.gitignore
@ -0,0 +1,22 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
--- a/vendor/github.com/robfig/cron/v3/.travis.yml
+++ b/vendor/github.com/robfig/cron/v3/.travis.yml
@ -0,0 +1 @@
+language: go
--- a/vendor/github.com/robfig/cron/v3/LICENSE
+++ b/vendor/github.com/robfig/cron/v3/LICENSE
@ -0,0 +1,21 @@
+Copyright (C) 2012 Rob Figueiredo
+All Rights Reserved.
+
+MIT LICENSE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--- a/vendor/github.com/robfig/cron/v3/README.md
+++ b/vendor/github.com/robfig/cron/v3/README.md
@ -0,0 +1,125 @@
+[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron)
+[![Build Status](https://travis-ci.org/robfig/cron.svg?branch=master)](https://travis-ci.org/robfig/cron)
+
+# cron
+
+Cron V3 has been released!
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Refer to the documentation here:
+http://godoc.org/github.com/robfig/cron
+
+The rest of this document describes the the advances in v3 and a list of
+breaking changes for users that wish to upgrade from an earlier version.
+
+## Upgrading to v3 (June 2019)
+
+cron v3 is a major upgrade to the library that addresses all outstanding bugs,
+feature requests, and rough edges. It is based on a merge of master which
+contains various fixes to issues found over the years and the v2 branch which
+contains some backwards-incompatible features like the ability to remove cron
+jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like
+the timezone support, and fixes a number of bugs.
+
+New features:
+
+- Support for Go modules. Callers must now import this library as
+  `github.com/robfig/cron/v3`, instead of `gopkg.in/...`
+
+- Fixed bugs:
+  - 0f01e6b parser: fix combining of Dow and Dom (#70)
+  - dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157)
+  - eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144)
+  - 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97)
+  - 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206)
+
+- Standard cron spec parsing by default (first field is "minute"), with an easy
+  way to opt into the seconds field (quartz-compatible). Although, note that the
+  year field (optional in Quartz) is not supported.
+
+- Extensible, key/value logging via an interface that complies with
+  the https://github.com/go-logr/logr project.
+
+- The new Chain & JobWrapper types allow you to install "interceptors" to add
+  cross-cutting behavior like the following:
+  - Recover any panics from jobs
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+  - Notification when jobs are completed
+
+It is backwards incompatible with both v1 and v2. These updates are required:
+
+- The v1 branch accepted an optional seconds field at the beginning of the cron
+  spec. This is non-standard and has led to a lot of confusion. The new default
+  parser conforms to the standard as described by [the Cron wikipedia page].
+
+  UPDATING: To retain the old behavior, construct your Cron with a custom
+  parser:
+
+      // Seconds field, required
+      cron.New(cron.WithSeconds())
+
+      // Seconds field, optional
+      cron.New(
+          cron.WithParser(
+              cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
+
+- The Cron type now accepts functional options on construction rather than the
+  previous ad-hoc behavior modification mechanisms (setting a field, calling a setter).
+
+  UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be
+  updated to provide those values on construction.
+
+- CRON_TZ is now the recommended way to specify the timezone of a single
+  schedule, which is sanctioned by the specification. The legacy "TZ=" prefix
+  will continue to be supported since it is unambiguous and easy to do so.
+
+  UPDATING: No update is required.
+
+- By default, cron will no longer recover panics in jobs that it runs.
+  Recovering can be surprising (see issue #192) and seems to be at odds with
+  typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option
+  has been removed to accommodate the more general JobWrapper type.
+
+  UPDATING: To opt into panic recovery and configure the panic logger:
+
+      cron.New(cron.WithChain(
+          cron.Recover(logger),  // or use cron.DefaultLogger
+      ))
+
+- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was
+  removed, since it is duplicative with the leveled logging.
+
+  UPDATING: Callers should use `WithLogger` and specify a logger that does not
+  discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`:
+
+      cron.New(
+          cron.WithLogger(cron.VerbosePrintfLogger(logger)))
+
+
+### Background - Cron spec format
+
+There are two cron spec formats in common usage:
+
+- The "standard" cron format, described on [the Cron wikipedia page] and used by
+  the cron Linux system utility.
+
+- The cron format used by [the Quartz Scheduler], commonly used for scheduled
+  jobs in Java software
+
+[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron
+[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html
+
+The original version of this package included an optional "seconds" field, which
+made it incompatible with both of these formats. Now, the "standard" format is
+the default format accepted, and the Quartz format is opt-in.
--- a/vendor/github.com/robfig/cron/v3/chain.go
+++ b/vendor/github.com/robfig/cron/v3/chain.go
@ -0,0 +1,92 @@
+package cron
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"time"
+)
+
+// JobWrapper decorates the given Job with some behavior.
+type JobWrapper func(Job) Job
+
+// Chain is a sequence of JobWrappers that decorates submitted jobs with
+// cross-cutting behaviors like logging or synchronization.
+type Chain struct {
+	wrappers []JobWrapper
+}
+
+// NewChain returns a Chain consisting of the given JobWrappers.
+func NewChain(c ...JobWrapper) Chain {
+	return Chain{c}
+}
+
+// Then decorates the given job with all JobWrappers in the chain.
+//
+// This:
+//     NewChain(m1, m2, m3).Then(job)
+// is equivalent to:
+//     m1(m2(m3(job)))
+func (c Chain) Then(j Job) Job {
+	for i := range c.wrappers {
+		j = c.wrappers[len(c.wrappers)-i-1](j)
+	}
+	return j
+}
+
+// Recover panics in wrapped jobs and log them with the provided logger.
+func Recover(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		return FuncJob(func() {
+			defer func() {
+				if r := recover(); r != nil {
+					const size = 64 << 10
+					buf := make([]byte, size)
+					buf = buf[:runtime.Stack(buf, false)]
+					err, ok := r.(error)
+					if !ok {
+						err = fmt.Errorf("%v", r)
+					}
+					logger.Error(err, "panic", "stack", "...\n"+string(buf))
+				}
+			}()
+			j.Run()
+		})
+	}
+}
+
+// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
+// previous one is complete. Jobs running after a delay of more than a minute
+// have the delay logged at Info.
+func DelayIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var mu sync.Mutex
+		return FuncJob(func() {
+			start := time.Now()
+			mu.Lock()
+			defer mu.Unlock()
+			if dur := time.Since(start); dur > time.Minute {
+				logger.Info("delay", "duration", dur)
+			}
+			j.Run()
+		})
+	}
+}
+
+// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
+// still running. It logs skips to the given logger at Info level.
+func SkipIfStillRunning(logger Logger) JobWrapper {
+	return func(j Job) Job {
+		var ch = make(chan struct{}, 1)
+		ch <- struct{}{}
+		return FuncJob(func() {
+			select {
+			case v := <-ch:
+				j.Run()
+				ch <- v
+			default:
+				logger.Info("skip")
+			}
+		})
+	}
+}
--- a/vendor/github.com/robfig/cron/v3/constantdelay.go
+++ b/vendor/github.com/robfig/cron/v3/constantdelay.go
@ -0,0 +1,27 @@
+package cron
+
+import "time"
+
+// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes".
+// It does not support jobs more frequent than once a second.
+type ConstantDelaySchedule struct {
+	Delay time.Duration
+}
+
+// Every returns a crontab Schedule that activates once every duration.
+// Delays of less than a second are not supported (will round up to 1 second).
+// Any fields less than a Second are truncated.
+func Every(duration time.Duration) ConstantDelaySchedule {
+	if duration < time.Second {
+		duration = time.Second
+	}
+	return ConstantDelaySchedule{
+		Delay: duration - time.Duration(duration.Nanoseconds())%time.Second,
+	}
+}
+
+// Next returns the next time this should be run.
+// This rounds so that the next activation time will be on the second.
+func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time {
+	return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond)
+}
--- a/vendor/github.com/robfig/cron/v3/cron.go
+++ b/vendor/github.com/robfig/cron/v3/cron.go
@ -0,0 +1,355 @@
+package cron
+
+import (
+	"context"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Cron keeps track of any number of entries, invoking the associated func as
+// specified by the schedule. It may be started, stopped, and the entries may
+// be inspected while running.
+type Cron struct {
+	entries   []*Entry
+	chain     Chain
+	stop      chan struct{}
+	add       chan *Entry
+	remove    chan EntryID
+	snapshot  chan chan []Entry
+	running   bool
+	logger    Logger
+	runningMu sync.Mutex
+	location  *time.Location
+	parser    ScheduleParser
+	nextID    EntryID
+	jobWaiter sync.WaitGroup
+}
+
+// ScheduleParser is an interface for schedule spec parsers that return a Schedule
+type ScheduleParser interface {
+	Parse(spec string) (Schedule, error)
+}
+
+// Job is an interface for submitted cron jobs.
+type Job interface {
+	Run()
+}
+
+// Schedule describes a job's duty cycle.
+type Schedule interface {
+	// Next returns the next activation time, later than the given time.
+	// Next is invoked initially, and then each time the job is run.
+	Next(time.Time) time.Time
+}
+
+// EntryID identifies an entry within a Cron instance
+type EntryID int
+
+// Entry consists of a schedule and the func to execute on that schedule.
+type Entry struct {
+	// ID is the cron-assigned ID of this entry, which may be used to look up a
+	// snapshot or remove it.
+	ID EntryID
+
+	// Schedule on which this job should be run.
+	Schedule Schedule
+
+	// Next time the job will run, or the zero time if Cron has not been
+	// started or this entry's schedule is unsatisfiable
+	Next time.Time
+
+	// Prev is the last time this job was run, or the zero time if never.
+	Prev time.Time
+
+	// WrappedJob is the thing to run when the Schedule is activated.
+	WrappedJob Job
+
+	// Job is the thing that was submitted to cron.
+	// It is kept around so that user code that needs to get at the job later,
+	// e.g. via Entries() can do so.
+	Job Job
+}
+
+// Valid returns true if this is not the zero entry.
+func (e Entry) Valid() bool { return e.ID != 0 }
+
+// byTime is a wrapper for sorting the entry array by time
+// (with zero time at the end).
+type byTime []*Entry
+
+func (s byTime) Len() int      { return len(s) }
+func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byTime) Less(i, j int) bool {
+	// Two zero times should return false.
+	// Otherwise, zero is "greater" than any other time.
+	// (To sort it at the end of the list.)
+	if s[i].Next.IsZero() {
+		return false
+	}
+	if s[j].Next.IsZero() {
+		return true
+	}
+	return s[i].Next.Before(s[j].Next)
+}
+
+// New returns a new Cron job runner, modified by the given options.
+//
+// Available Settings
+//
+//   Time Zone
+//     Description: The time zone in which schedules are interpreted
+//     Default:     time.Local
+//
+//   Parser
+//     Description: Parser converts cron spec strings into cron.Schedules.
+//     Default:     Accepts this spec: https://en.wikipedia.org/wiki/Cron
+//
+//   Chain
+//     Description: Wrap submitted jobs to customize behavior.
+//     Default:     A chain that recovers panics and logs them to stderr.
+//
+// See "cron.With*" to modify the default behavior.
+func New(opts ...Option) *Cron {
+	c := &Cron{
+		entries:   nil,
+		chain:     NewChain(),
+		add:       make(chan *Entry),
+		stop:      make(chan struct{}),
+		snapshot:  make(chan chan []Entry),
+		remove:    make(chan EntryID),
+		running:   false,
+		runningMu: sync.Mutex{},
+		logger:    DefaultLogger,
+		location:  time.Local,
+		parser:    standardParser,
+	}
+	for _, opt := range opts {
+		opt(c)
+	}
+	return c
+}
+
+// FuncJob is a wrapper that turns a func() into a cron.Job
+type FuncJob func()
+
+func (f FuncJob) Run() { f() }
+
+// AddFunc adds a func to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) {
+	return c.AddJob(spec, FuncJob(cmd))
+}
+
+// AddJob adds a Job to the Cron to be run on the given schedule.
+// The spec is parsed using the time zone of this Cron instance as the default.
+// An opaque ID is returned that can be used to later remove it.
+func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) {
+	schedule, err := c.parser.Parse(spec)
+	if err != nil {
+		return 0, err
+	}
+	return c.Schedule(schedule, cmd), nil
+}
+
+// Schedule adds a Job to the Cron to be run on the given schedule.
+// The job is wrapped with the configured Chain.
+func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	c.nextID++
+	entry := &Entry{
+		ID:         c.nextID,
+		Schedule:   schedule,
+		WrappedJob: c.chain.Then(cmd),
+		Job:        cmd,
+	}
+	if !c.running {
+		c.entries = append(c.entries, entry)
+	} else {
+		c.add <- entry
+	}
+	return entry.ID
+}
+
+// Entries returns a snapshot of the cron entries.
+func (c *Cron) Entries() []Entry {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		replyChan := make(chan []Entry, 1)
+		c.snapshot <- replyChan
+		return <-replyChan
+	}
+	return c.entrySnapshot()
+}
+
+// Location gets the time zone location
+func (c *Cron) Location() *time.Location {
+	return c.location
+}
+
+// Entry returns a snapshot of the given entry, or nil if it couldn't be found.
+func (c *Cron) Entry(id EntryID) Entry {
+	for _, entry := range c.Entries() {
+		if id == entry.ID {
+			return entry
+		}
+	}
+	return Entry{}
+}
+
+// Remove an entry from being run in the future.
+func (c *Cron) Remove(id EntryID) {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.remove <- id
+	} else {
+		c.removeEntry(id)
+	}
+}
+
+// Start the cron scheduler in its own goroutine, or no-op if already started.
+func (c *Cron) Start() {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		return
+	}
+	c.running = true
+	go c.run()
+}
+
+// Run the cron scheduler, or no-op if already running.
+func (c *Cron) Run() {
+	c.runningMu.Lock()
+	if c.running {
+		c.runningMu.Unlock()
+		return
+	}
+	c.running = true
+	c.runningMu.Unlock()
+	c.run()
+}
+
+// run the scheduler.. this is private just due to the need to synchronize
+// access to the 'running' state variable.
+func (c *Cron) run() {
+	c.logger.Info("start")
+
+	// Figure out the next activation times for each entry.
+	now := c.now()
+	for _, entry := range c.entries {
+		entry.Next = entry.Schedule.Next(now)
+		c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next)
+	}
+
+	for {
+		// Determine the next entry to run.
+		sort.Sort(byTime(c.entries))
+
+		var timer *time.Timer
+		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
+			// If there are no entries yet, just sleep - it still handles new entries
+			// and stop requests.
+			timer = time.NewTimer(100000 * time.Hour)
+		} else {
+			timer = time.NewTimer(c.entries[0].Next.Sub(now))
+		}
+
+		for {
+			select {
+			case now = <-timer.C:
+				now = now.In(c.location)
+				c.logger.Info("wake", "now", now)
+
+				// Run every entry whose next time was less than now
+				for _, e := range c.entries {
+					if e.Next.After(now) || e.Next.IsZero() {
+						break
+					}
+					c.startJob(e.WrappedJob)
+					e.Prev = e.Next
+					e.Next = e.Schedule.Next(now)
+					c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next)
+				}
+
+			case newEntry := <-c.add:
+				timer.Stop()
+				now = c.now()
+				newEntry.Next = newEntry.Schedule.Next(now)
+				c.entries = append(c.entries, newEntry)
+				c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next)
+
+			case replyChan := <-c.snapshot:
+				replyChan <- c.entrySnapshot()
+				continue
+
+			case <-c.stop:
+				timer.Stop()
+				c.logger.Info("stop")
+				return
+
+			case id := <-c.remove:
+				timer.Stop()
+				now = c.now()
+				c.removeEntry(id)
+				c.logger.Info("removed", "entry", id)
+			}
+
+			break
+		}
+	}
+}
+
+// startJob runs the given job in a new goroutine.
+func (c *Cron) startJob(j Job) {
+	c.jobWaiter.Add(1)
+	go func() {
+		defer c.jobWaiter.Done()
+		j.Run()
+	}()
+}
+
+// now returns current time in c location
+func (c *Cron) now() time.Time {
+	return time.Now().In(c.location)
+}
+
+// Stop stops the cron scheduler if it is running; otherwise it does nothing.
+// A context is returned so the caller can wait for running jobs to complete.
+func (c *Cron) Stop() context.Context {
+	c.runningMu.Lock()
+	defer c.runningMu.Unlock()
+	if c.running {
+		c.stop <- struct{}{}
+		c.running = false
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		c.jobWaiter.Wait()
+		cancel()
+	}()
+	return ctx
+}
+
+// entrySnapshot returns a copy of the current cron entry list.
+func (c *Cron) entrySnapshot() []Entry {
+	var entries = make([]Entry, len(c.entries))
+	for i, e := range c.entries {
+		entries[i] = *e
+	}
+	return entries
+}
+
+func (c *Cron) removeEntry(id EntryID) {
+	var entries []*Entry
+	for _, e := range c.entries {
+		if e.ID != id {
+			entries = append(entries, e)
+		}
+	}
+	c.entries = entries
+}
--- a/vendor/github.com/robfig/cron/v3/doc.go
+++ b/vendor/github.com/robfig/cron/v3/doc.go
@ -0,0 +1,231 @@
+/*
+Package cron implements a cron spec parser and job runner.
+
+Installation
+
+To download the specific tagged release, run:
+
+	go get github.com/robfig/cron/v3@v3.0.0
+
+Import it in your program as:
+
+	import "github.com/robfig/cron/v3"
+
+It requires Go 1.11 or later due to usage of Go Modules.
+
+Usage
+
+Callers may register Funcs to be invoked on a given schedule.  Cron will run
+them in their own goroutines.
+
+	c := cron.New()
+	c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") })
+	c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") })
+	c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") })
+	c.AddFunc("@hourly",      func() { fmt.Println("Every hour, starting an hour from now") })
+	c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") })
+	c.Start()
+	..
+	// Funcs are invoked in their own goroutine, asynchronously.
+	...
+	// Funcs may also be added to a running Cron
+	c.AddFunc("@daily", func() { fmt.Println("Every day") })
+	..
+	// Inspect the cron job entries' next and previous run times.
+	inspect(c.Entries())
+	..
+	c.Stop()  // Stop the scheduler (does not stop any jobs already running).
+
+CRON Expression Format
+
+A cron expression represents a set of times, using 5 space-separated fields.
+
+	Field name   | Mandatory? | Allowed values  | Allowed special characters
+	----------   | ---------- | --------------  | --------------------------
+	Minutes      | Yes        | 0-59            | * / , -
+	Hours        | Yes        | 0-23            | * / , -
+	Day of month | Yes        | 1-31            | * / , - ?
+	Month        | Yes        | 1-12 or JAN-DEC | * / , -
+	Day of week  | Yes        | 0-6 or SUN-SAT  | * / , - ?
+
+Month and Day-of-week field values are case insensitive.  "SUN", "Sun", and
+"sun" are equally accepted.
+
+The specific interpretation of the format is based on the Cron Wikipedia page:
+https://en.wikipedia.org/wiki/Cron
+
+Alternative Formats
+
+Alternative Cron expression formats support other fields like seconds. You can
+implement that by creating a custom Parser as follows.
+
+	cron.New(
+		cron.WithParser(
+			cron.NewParser(
+				cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)))
+
+Since adding Seconds is the most common modification to the standard cron spec,
+cron provides a builtin function to do that, which is equivalent to the custom
+parser you saw earlier, except that its seconds field is REQUIRED:
+
+	cron.New(cron.WithSeconds())
+
+That emulates Quartz, the most popular alternative Cron schedule format:
+http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
+
+Special Characters
+
+Asterisk ( * )
+
+The asterisk indicates that the cron expression will match for all values of the
+field; e.g., using an asterisk in the 5th field (month) would indicate every
+month.
+
+Slash ( / )
+
+Slashes are used to describe increments of ranges. For example 3-59/15 in the
+1st field (minutes) would indicate the 3rd minute of the hour and every 15
+minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...",
+that is, an increment over the largest possible range of the field.  The form
+"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the
+increment until the end of that specific range.  It does not wrap around.
+
+Comma ( , )
+
+Commas are used to separate items of a list. For example, using "MON,WED,FRI" in
+the 5th field (day of week) would mean Mondays, Wednesdays and Fridays.
+
+Hyphen ( - )
+
+Hyphens are used to define ranges. For example, 9-17 would indicate every
+hour between 9am and 5pm inclusive.
+
+Question mark ( ? )
+
+Question mark may be used instead of '*' for leaving either day-of-month or
+day-of-week blank.
+
+Predefined schedules
+
+You may use one of several pre-defined schedules in place of a cron expression.
+
+	Entry                  | Description                                | Equivalent To
+	-----                  | -----------                                | -------------
+	@yearly (or @annually) | Run once a year, midnight, Jan. 1st        | 0 0 1 1 *
+	@monthly               | Run once a month, midnight, first of month | 0 0 1 * *
+	@weekly                | Run once a week, midnight between Sat/Sun  | 0 0 * * 0
+	@daily (or @midnight)  | Run once a day, midnight                   | 0 0 * * *
+	@hourly                | Run once an hour, beginning of hour        | 0 * * * *
+
+Intervals
+
+You may also schedule a job to execute at fixed intervals, starting at the time it's added
+or cron is run. This is supported by formatting the cron spec like this:
+
+    @every <duration>
+
+where "duration" is a string accepted by time.ParseDuration
+(http://golang.org/pkg/time/#ParseDuration).
+
+For example, "@every 1h30m10s" would indicate a schedule that activates after
+1 hour, 30 minutes, 10 seconds, and then every interval after that.
+
+Note: The interval does not take the job runtime into account.  For example,
+if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes,
+it will have only 2 minutes of idle time between each run.
+
+Time zones
+
+By default, all interpretation and scheduling is done in the machine's local
+time zone (time.Local). You can specify a different time zone on construction:
+
+      cron.New(
+          cron.WithLocation(time.UTC))
+
+Individual cron schedules may also override the time zone they are to be
+interpreted in by providing an additional space-separated field at the beginning
+of the cron spec, of the form "CRON_TZ=Asia/Tokyo".
+
+For example:
+
+	# Runs at 6am in time.Local
+	cron.New().AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in America/New_York
+	nyc, _ := time.LoadLocation("America/New_York")
+	c := cron.New(cron.WithLocation(nyc))
+	c.AddFunc("0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+	# Runs at 6am in Asia/Tokyo
+	c := cron.New(cron.WithLocation(nyc))
+	c.SetLocation("America/New_York")
+	c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
+
+The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility.
+
+Be aware that jobs scheduled during daylight-savings leap-ahead transitions will
+not be run!
+
+Job Wrappers
+
+A Cron runner may be configured with a chain of job wrappers to add
+cross-cutting functionality to all submitted jobs. For example, they may be used
+to achieve the following effects:
+
+  - Recover any panics from jobs (activated by default)
+  - Delay a job's execution if the previous run hasn't completed yet
+  - Skip a job's execution if the previous run hasn't completed yet
+  - Log each job's invocations
+
+Install wrappers for all jobs added to a cron using the `cron.WithChain` option:
+
+	cron.New(cron.WithChain(
+		cron.SkipIfStillRunning(logger),
+	))
+
+Install wrappers for individual jobs by explicitly wrapping them:
+
+	job = cron.NewChain(
+		cron.SkipIfStillRunning(logger),
+	).Then(job)
+
+Thread safety
+
+Since the Cron service runs concurrently with the calling code, some amount of
+care must be taken to ensure proper synchronization.
+
+All cron methods are designed to be correctly synchronized as long as the caller
+ensures that invocations have a clear happens-before ordering between them.
+
+Logging
+
+Cron defines a Logger interface that is a subset of the one defined in
+github.com/go-logr/logr. It has two logging levels (Info and Error), and
+parameters are key/value pairs. This makes it possible for cron logging to plug
+into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided
+to wrap the standard library *log.Logger.
+
+For additional insight into Cron operations, verbose logging may be activated
+which will record job runs, scheduling decisions, and added or removed jobs.
+Activate it with a one-off logger as follows:
+
+	cron.New(
+		cron.WithLogger(
+			cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))))
+
+
+Implementation
+
+Cron entries are stored in an array, sorted by their next activation time.  Cron
+sleeps until the next job is due to be run.
+
+Upon waking:
+ - it runs each entry that is active on that second
+ - it calculates the next run times for the jobs that were run
+ - it re-sorts the array of entries by next activation time.
+ - it goes to sleep until the soonest job.
+*/
+package cron
--- a/vendor/github.com/robfig/cron/v3/go.mod
+++ b/vendor/github.com/robfig/cron/v3/go.mod
@ -0,0 +1,3 @@
+module github.com/robfig/cron/v3
+
+go 1.12
--- a/vendor/github.com/robfig/cron/v3/logger.go
+++ b/vendor/github.com/robfig/cron/v3/logger.go
@ -0,0 +1,86 @@
+package cron
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"time"
+)
+
+// DefaultLogger is used by Cron if none is specified.
+var DefaultLogger Logger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))
+
+// DiscardLogger can be used by callers to discard all log messages.
+var DiscardLogger Logger = PrintfLogger(log.New(ioutil.Discard, "", 0))
+
+// Logger is the interface used in this package for logging, so that any backend
+// can be plugged in. It is a subset of the github.com/go-logr/logr interface.
+type Logger interface {
+	// Info logs routine messages about cron's operation.
+	Info(msg string, keysAndValues ...interface{})
+	// Error logs an error condition.
+	Error(err error, msg string, keysAndValues ...interface{})
+}
+
+// PrintfLogger wraps a Printf-based logger (such as the standard library "log")
+// into an implementation of the Logger interface which logs errors only.
+func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, false}
+}
+
+// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library
+// "log") into an implementation of the Logger interface which logs everything.
+func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
+	return printfLogger{l, true}
+}
+
+type printfLogger struct {
+	logger  interface{ Printf(string, ...interface{}) }
+	logInfo bool
+}
+
+func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) {
+	if pl.logInfo {
+		keysAndValues = formatTimes(keysAndValues)
+		pl.logger.Printf(
+			formatString(len(keysAndValues)),
+			append([]interface{}{msg}, keysAndValues...)...)
+	}
+}
+
+func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) {
+	keysAndValues = formatTimes(keysAndValues)
+	pl.logger.Printf(
+		formatString(len(keysAndValues)+2),
+		append([]interface{}{msg, "error", err}, keysAndValues...)...)
+}
+
+// formatString returns a logfmt-like format string for the number of
+// key/values.
+func formatString(numKeysAndValues int) string {
+	var sb strings.Builder
+	sb.WriteString("%s")
+	if numKeysAndValues > 0 {
+		sb.WriteString(", ")
+	}
+	for i := 0; i < numKeysAndValues/2; i++ {
+		if i > 0 {
+			sb.WriteString(", ")
+		}
+		sb.WriteString("%v=%v")
+	}
+	return sb.String()
+}
+
+// formatTimes formats any time.Time values as RFC3339.
+func formatTimes(keysAndValues []interface{}) []interface{} {
+	var formattedArgs []interface{}
+	for _, arg := range keysAndValues {
+		if t, ok := arg.(time.Time); ok {
+			arg = t.Format(time.RFC3339)
+		}
+		formattedArgs = append(formattedArgs, arg)
+	}
+	return formattedArgs
+}
--- a/vendor/github.com/robfig/cron/v3/option.go
+++ b/vendor/github.com/robfig/cron/v3/option.go
@ -0,0 +1,45 @@
+package cron
+
+import (
+	"time"
+)
+
+// Option represents a modification to the default behavior of a Cron.
+type Option func(*Cron)
+
+// WithLocation overrides the timezone of the cron instance.
+func WithLocation(loc *time.Location) Option {
+	return func(c *Cron) {
+		c.location = loc
+	}
+}
+
+// WithSeconds overrides the parser used for interpreting job schedules to
+// include a seconds field as the first one.
+func WithSeconds() Option {
+	return WithParser(NewParser(
+		Second | Minute | Hour | Dom | Month | Dow | Descriptor,
+	))
+}
+
+// WithParser overrides the parser used for interpreting job schedules.
+func WithParser(p ScheduleParser) Option {
+	return func(c *Cron) {
+		c.parser = p
+	}
+}
+
+// WithChain specifies Job wrappers to apply to all jobs added to this cron.
+// Refer to the Chain* functions in this package for provided wrappers.
+func WithChain(wrappers ...JobWrapper) Option {
+	return func(c *Cron) {
+		c.chain = NewChain(wrappers...)
+	}
+}
+
+// WithLogger uses the provided logger.
+func WithLogger(logger Logger) Option {
+	return func(c *Cron) {
+		c.logger = logger
+	}
+}
--- a/vendor/github.com/robfig/cron/v3/parser.go
+++ b/vendor/github.com/robfig/cron/v3/parser.go
@ -0,0 +1,434 @@
+package cron
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Configuration options for creating a parser. Most options specify which
+// fields should be included, while others enable features. If a field is not
+// included the parser will assume a default value. These options do not change
+// the order fields are parse in.
+type ParseOption int
+
+const (
+	Second         ParseOption = 1 << iota // Seconds field, default 0
+	SecondOptional                         // Optional seconds field, default 0
+	Minute                                 // Minutes field, default 0
+	Hour                                   // Hours field, default 0
+	Dom                                    // Day of month field, default *
+	Month                                  // Month field, default *
+	Dow                                    // Day of week field, default *
+	DowOptional                            // Optional day of week field, default *
+	Descriptor                             // Allow descriptors such as @monthly, @weekly, etc.
+)
+
+var places = []ParseOption{
+	Second,
+	Minute,
+	Hour,
+	Dom,
+	Month,
+	Dow,
+}
+
+var defaults = []string{
+	"0",
+	"0",
+	"0",
+	"*",
+	"*",
+	"*",
+}
+
+// A custom Parser that can be configured.
+type Parser struct {
+	options ParseOption
+}
+
+// NewParser creates a Parser with custom options.
+//
+// It panics if more than one Optional is given, since it would be impossible to
+// correctly infer which optional is provided or missing in general.
+//
+// Examples
+//
+//  // Standard parser without descriptors
+//  specParser := NewParser(Minute | Hour | Dom | Month | Dow)
+//  sched, err := specParser.Parse("0 0 15 */3 *")
+//
+//  // Same as above, just excludes time fields
+//  subsParser := NewParser(Dom | Month | Dow)
+//  sched, err := specParser.Parse("15 */3 *")
+//
+//  // Same as above, just makes Dow optional
+//  subsParser := NewParser(Dom | Month | DowOptional)
+//  sched, err := specParser.Parse("15 */3")
+//
+func NewParser(options ParseOption) Parser {
+	optionals := 0
+	if options&DowOptional > 0 {
+		optionals++
+	}
+	if options&SecondOptional > 0 {
+		optionals++
+	}
+	if optionals > 1 {
+		panic("multiple optionals may not be configured")
+	}
+	return Parser{options}
+}
+
+// Parse returns a new crontab schedule representing the given spec.
+// It returns a descriptive error if the spec is not valid.
+// It accepts crontab specs and features configured by NewParser.
+func (p Parser) Parse(spec string) (Schedule, error) {
+	if len(spec) == 0 {
+		return nil, fmt.Errorf("empty spec string")
+	}
+
+	// Extract timezone if present
+	var loc = time.Local
+	if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") {
+		var err error
+		i := strings.Index(spec, " ")
+		eq := strings.Index(spec, "=")
+		if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil {
+			return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err)
+		}
+		spec = strings.TrimSpace(spec[i:])
+	}
+
+	// Handle named schedules (descriptors), if configured
+	if strings.HasPrefix(spec, "@") {
+		if p.options&Descriptor == 0 {
+			return nil, fmt.Errorf("parser does not accept descriptors: %v", spec)
+		}
+		return parseDescriptor(spec, loc)
+	}
+
+	// Split on whitespace.
+	fields := strings.Fields(spec)
+
+	// Validate & fill in any omitted or optional fields
+	var err error
+	fields, err = normalizeFields(fields, p.options)
+	if err != nil {
+		return nil, err
+	}
+
+	field := func(field string, r bounds) uint64 {
+		if err != nil {
+			return 0
+		}
+		var bits uint64
+		bits, err = getField(field, r)
+		return bits
+	}
+
+	var (
+		second     = field(fields[0], seconds)
+		minute     = field(fields[1], minutes)
+		hour       = field(fields[2], hours)
+		dayofmonth = field(fields[3], dom)
+		month      = field(fields[4], months)
+		dayofweek  = field(fields[5], dow)
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return &SpecSchedule{
+		Second:   second,
+		Minute:   minute,
+		Hour:     hour,
+		Dom:      dayofmonth,
+		Month:    month,
+		Dow:      dayofweek,
+		Location: loc,
+	}, nil
+}
+
+// normalizeFields takes a subset set of the time fields and returns the full set
+// with defaults (zeroes) populated for unset fields.
+//
+// As part of performing this function, it also validates that the provided
+// fields are compatible with the configured options.
+func normalizeFields(fields []string, options ParseOption) ([]string, error) {
+	// Validate optionals & add their field to options
+	optionals := 0
+	if options&SecondOptional > 0 {
+		options |= Second
+		optionals++
+	}
+	if options&DowOptional > 0 {
+		options |= Dow
+		optionals++
+	}
+	if optionals > 1 {
+		return nil, fmt.Errorf("multiple optionals may not be configured")
+	}
+
+	// Figure out how many fields we need
+	max := 0
+	for _, place := range places {
+		if options&place > 0 {
+			max++
+		}
+	}
+	min := max - optionals
+
+	// Validate number of fields
+	if count := len(fields); count < min || count > max {
+		if min == max {
+			return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields)
+		}
+		return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields)
+	}
+
+	// Populate the optional field if not provided
+	if min < max && len(fields) == min {
+		switch {
+		case options&DowOptional > 0:
+			fields = append(fields, defaults[5]) // TODO: improve access to default
+		case options&SecondOptional > 0:
+			fields = append([]string{defaults[0]}, fields...)
+		default:
+			return nil, fmt.Errorf("unknown optional field")
+		}
+	}
+
+	// Populate all fields not part of options with their defaults
+	n := 0
+	expandedFields := make([]string, len(places))
+	copy(expandedFields, defaults)
+	for i, place := range places {
+		if options&place > 0 {
+			expandedFields[i] = fields[n]
+			n++
+		}
+	}
+	return expandedFields, nil
+}
+
+var standardParser = NewParser(
+	Minute | Hour | Dom | Month | Dow | Descriptor,
+)
+
+// ParseStandard returns a new crontab schedule representing the given
+// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries
+// representing: minute, hour, day of month, month and day of week, in that
+// order. It returns a descriptive error if the spec is not valid.
+//
+// It accepts
+//   - Standard crontab specs, e.g. "* * * * ?"
+//   - Descriptors, e.g. "@midnight", "@every 1h30m"
+func ParseStandard(standardSpec string) (Schedule, error) {
+	return standardParser.Parse(standardSpec)
+}
+
+// getField returns an Int with the bits set representing all of the times that
+// the field represents or error parsing field value.  A "field" is a comma-separated
+// list of "ranges".
+func getField(field string, r bounds) (uint64, error) {
+	var bits uint64
+	ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
+	for _, expr := range ranges {
+		bit, err := getRange(expr, r)
+		if err != nil {
+			return bits, err
+		}
+		bits |= bit
+	}
+	return bits, nil
+}
+
+// getRange returns the bits indicated by the given expression:
+//   number | number "-" number [ "/" number ]
+// or error parsing range.
+func getRange(expr string, r bounds) (uint64, error) {
+	var (
+		start, end, step uint
+		rangeAndStep     = strings.Split(expr, "/")
+		lowAndHigh       = strings.Split(rangeAndStep[0], "-")
+		singleDigit      = len(lowAndHigh) == 1
+		err              error
+	)
+
+	var extra uint64
+	if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
+		start = r.min
+		end = r.max
+		extra = starBit
+	} else {
+		start, err = parseIntOrName(lowAndHigh[0], r.names)
+		if err != nil {
+			return 0, err
+		}
+		switch len(lowAndHigh) {
+		case 1:
+			end = start
+		case 2:
+			end, err = parseIntOrName(lowAndHigh[1], r.names)
+			if err != nil {
+				return 0, err
+			}
+		default:
+			return 0, fmt.Errorf("too many hyphens: %s", expr)
+		}
+	}
+
+	switch len(rangeAndStep) {
+	case 1:
+		step = 1
+	case 2:
+		step, err = mustParseInt(rangeAndStep[1])
+		if err != nil {
+			return 0, err
+		}
+
+		// Special handling: "N/step" means "N-max/step".
+		if singleDigit {
+			end = r.max
+		}
+		if step > 1 {
+			extra = 0
+		}
+	default:
+		return 0, fmt.Errorf("too many slashes: %s", expr)
+	}
+
+	if start < r.min {
+		return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
+	}
+	if end > r.max {
+		return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
+	}
+	if start > end {
+		return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
+	}
+	if step == 0 {
+		return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
+	}
+
+	return getBits(start, end, step) | extra, nil
+}
+
+// parseIntOrName returns the (possibly-named) integer contained in expr.
+func parseIntOrName(expr string, names map[string]uint) (uint, error) {
+	if names != nil {
+		if namedInt, ok := names[strings.ToLower(expr)]; ok {
+			return namedInt, nil
+		}
+	}
+	return mustParseInt(expr)
+}
+
+// mustParseInt parses the given expression as an int or returns an error.
+func mustParseInt(expr string) (uint, error) {
+	num, err := strconv.Atoi(expr)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
+	}
+	if num < 0 {
+		return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
+	}
+
+	return uint(num), nil
+}
+
+// getBits sets all bits in the range [min, max], modulo the given step size.
+func getBits(min, max, step uint) uint64 {
+	var bits uint64
+
+	// If step is 1, use shifts.
+	if step == 1 {
+		return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
+	}
+
+	// Else, use a simple loop.
+	for i := min; i <= max; i += step {
+		bits |= 1 << i
+	}
+	return bits
+}
+
+// all returns all bits within the given bounds.  (plus the star bit)
+func all(r bounds) uint64 {
+	return getBits(r.min, r.max, 1) | starBit
+}
+
+// parseDescriptor returns a predefined schedule for the expression, or error if none matches.
+func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) {
+	switch descriptor {
+	case "@yearly", "@annually":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      1 << dom.min,
+			Month:    1 << months.min,
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@monthly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      1 << dom.min,
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@weekly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      1 << dow.min,
+			Location: loc,
+		}, nil
+
+	case "@daily", "@midnight":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     1 << hours.min,
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	case "@hourly":
+		return &SpecSchedule{
+			Second:   1 << seconds.min,
+			Minute:   1 << minutes.min,
+			Hour:     all(hours),
+			Dom:      all(dom),
+			Month:    all(months),
+			Dow:      all(dow),
+			Location: loc,
+		}, nil
+
+	}
+
+	const every = "@every "
+	if strings.HasPrefix(descriptor, every) {
+		duration, err := time.ParseDuration(descriptor[len(every):])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err)
+		}
+		return Every(duration), nil
+	}
+
+	return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor)
+}
--- a/vendor/github.com/robfig/cron/v3/spec.go
+++ b/vendor/github.com/robfig/cron/v3/spec.go
@ -0,0 +1,188 @@
+package cron
+
+import "time"
+
+// SpecSchedule specifies a duty cycle (to the second granularity), based on a
+// traditional crontab specification. It is computed initially and stored as bit sets.
+type SpecSchedule struct {
+	Second, Minute, Hour, Dom, Month, Dow uint64
+
+	// Override location for this schedule.
+	Location *time.Location
+}
+
+// bounds provides a range of acceptable values (plus a map of name to value).
+type bounds struct {
+	min, max uint
+	names    map[string]uint
+}
+
+// The bounds for each field.
+var (
+	seconds = bounds{0, 59, nil}
+	minutes = bounds{0, 59, nil}
+	hours   = bounds{0, 23, nil}
+	dom     = bounds{1, 31, nil}
+	months  = bounds{1, 12, map[string]uint{
+		"jan": 1,
+		"feb": 2,
+		"mar": 3,
+		"apr": 4,
+		"may": 5,
+		"jun": 6,
+		"jul": 7,
+		"aug": 8,
+		"sep": 9,
+		"oct": 10,
+		"nov": 11,
+		"dec": 12,
+	}}
+	dow = bounds{0, 6, map[string]uint{
+		"sun": 0,
+		"mon": 1,
+		"tue": 2,
+		"wed": 3,
+		"thu": 4,
+		"fri": 5,
+		"sat": 6,
+	}}
+)
+
+const (
+	// Set the top bit if a star was included in the expression.
+	starBit = 1 << 63
+)
+
+// Next returns the next time this schedule is activated, greater than the given
+// time.  If no time can be found to satisfy the schedule, return the zero time.
+func (s *SpecSchedule) Next(t time.Time) time.Time {
+	// General approach
+	//
+	// For Month, Day, Hour, Minute, Second:
+	// Check if the time value matches.  If yes, continue to the next field.
+	// If the field doesn't match the schedule, then increment the field until it matches.
+	// While incrementing the field, a wrap-around brings it back to the beginning
+	// of the field list (since it is necessary to re-verify previous field
+	// values)
+
+	// Convert the given time into the schedule's timezone, if one is specified.
+	// Save the original timezone so we can convert back after we find a time.
+	// Note that schedules without a time zone specified (time.Local) are treated
+	// as local to the time provided.
+	origLocation := t.Location()
+	loc := s.Location
+	if loc == time.Local {
+		loc = t.Location()
+	}
+	if s.Location != time.Local {
+		t = t.In(s.Location)
+	}
+
+	// Start at the earliest possible time (the upcoming second).
+	t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond)
+
+	// This flag indicates whether a field has been incremented.
+	added := false
+
+	// If no time is found within five years, return zero.
+	yearLimit := t.Year() + 5
+
+WRAP:
+	if t.Year() > yearLimit {
+		return time.Time{}
+	}
+
+	// Find the first applicable month.
+	// If it's this month, then do nothing.
+	for 1<<uint(t.Month())&s.Month == 0 {
+		// If we have to add a month, reset the other parts to 0.
+		if !added {
+			added = true
+			// Otherwise, set the date at the beginning (since the current time is irrelevant).
+			t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 1, 0)
+
+		// Wrapped around.
+		if t.Month() == time.January {
+			goto WRAP
+		}
+	}
+
+	// Now get a day in that month.
+	//
+	// NOTE: This causes issues for daylight savings regimes where midnight does
+	// not exist.  For example: Sao Paulo has DST that transforms midnight on
+	// 11/3 into 1am. Handle that by noticing when the Hour ends up != 0.
+	for !dayMatches(s, t) {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
+		}
+		t = t.AddDate(0, 0, 1)
+		// Notice if the hour is no longer midnight due to DST.
+		// Add an hour if it's 23, subtract an hour if it's 1.
+		if t.Hour() != 0 {
+			if t.Hour() > 12 {
+				t = t.Add(time.Duration(24-t.Hour()) * time.Hour)
+			} else {
+				t = t.Add(time.Duration(-t.Hour()) * time.Hour)
+			}
+		}
+
+		if t.Day() == 1 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Hour())&s.Hour == 0 {
+		if !added {
+			added = true
+			t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc)
+		}
+		t = t.Add(1 * time.Hour)
+
+		if t.Hour() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Minute())&s.Minute == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Minute)
+		}
+		t = t.Add(1 * time.Minute)
+
+		if t.Minute() == 0 {
+			goto WRAP
+		}
+	}
+
+	for 1<<uint(t.Second())&s.Second == 0 {
+		if !added {
+			added = true
+			t = t.Truncate(time.Second)
+		}
+		t = t.Add(1 * time.Second)
+
+		if t.Second() == 0 {
+			goto WRAP
+		}
+	}
+
+	return t.In(origLocation)
+}
+
+// dayMatches returns true if the schedule's day-of-week and day-of-month
+// restrictions are satisfied by the given time.
+func dayMatches(s *SpecSchedule, t time.Time) bool {
+	var (
+		domMatch bool = 1<<uint(t.Day())&s.Dom > 0
+		dowMatch bool = 1<<uint(t.Weekday())&s.Dow > 0
+	)
+	if s.Dom&starBit > 0 || s.Dow&starBit > 0 {
+		return domMatch && dowMatch
+	}
+	return domMatch || dowMatch
+}
--- a/vendor/go.etcd.io/etcd/clientv3/snapshot/doc.go
+++ b/vendor/go.etcd.io/etcd/clientv3/snapshot/doc.go
@ -0,0 +1,16 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package snapshot implements utilities around etcd snapshot.
+package snapshot
--- a/vendor/go.etcd.io/etcd/clientv3/snapshot/util.go
+++ b/vendor/go.etcd.io/etcd/clientv3/snapshot/util.go
@ -0,0 +1,35 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapshot
+
+import "encoding/binary"
+
+type revision struct {
+	main int64
+	sub  int64
+}
+
+func bytesToRev(bytes []byte) revision {
+	return revision{
+		main: int64(binary.BigEndian.Uint64(bytes[0:8])),
+		sub:  int64(binary.BigEndian.Uint64(bytes[9:])),
+	}
+}
+
+// initIndex implements ConsistentIndexGetter so the snapshot won't block
+// the new raft instance by waiting for a future raft index.
+type initIndex int
+
+func (i *initIndex) ConsistentIndex() uint64 { return uint64(*i) }
--- a/vendor/go.etcd.io/etcd/clientv3/snapshot/v3_snapshot.go
+++ b/vendor/go.etcd.io/etcd/clientv3/snapshot/v3_snapshot.go
@ -0,0 +1,503 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snapshot
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"math"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	bolt "go.etcd.io/bbolt"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/etcdserver"
+	"go.etcd.io/etcd/etcdserver/api/membership"
+	"go.etcd.io/etcd/etcdserver/api/snap"
+	"go.etcd.io/etcd/etcdserver/api/v2store"
+	"go.etcd.io/etcd/etcdserver/etcdserverpb"
+	"go.etcd.io/etcd/lease"
+	"go.etcd.io/etcd/mvcc"
+	"go.etcd.io/etcd/mvcc/backend"
+	"go.etcd.io/etcd/pkg/fileutil"
+	"go.etcd.io/etcd/pkg/traceutil"
+	"go.etcd.io/etcd/pkg/types"
+	"go.etcd.io/etcd/raft"
+	"go.etcd.io/etcd/raft/raftpb"
+	"go.etcd.io/etcd/wal"
+	"go.etcd.io/etcd/wal/walpb"
+	"go.uber.org/zap"
+)
+
+// Manager defines snapshot methods.
+type Manager interface {
+	// Save fetches snapshot from remote etcd server and saves data
+	// to target path. If the context "ctx" is canceled or timed out,
+	// snapshot save stream will error out (e.g. context.Canceled,
+	// context.DeadlineExceeded). Make sure to specify only one endpoint
+	// in client configuration. Snapshot API must be requested to a
+	// selected node, and saved snapshot is the point-in-time state of
+	// the selected node.
+	Save(ctx context.Context, cfg clientv3.Config, dbPath string) error
+
+	// Status returns the snapshot file information.
+	Status(dbPath string) (Status, error)
+
+	// Restore restores a new etcd data directory from given snapshot
+	// file. It returns an error if specified data directory already
+	// exists, to prevent unintended data directory overwrites.
+	Restore(cfg RestoreConfig) error
+}
+
+// NewV3 returns a new snapshot Manager for v3.x snapshot.
+func NewV3(lg *zap.Logger) Manager {
+	if lg == nil {
+		lg = zap.NewExample()
+	}
+	return &v3Manager{lg: lg}
+}
+
+type v3Manager struct {
+	lg *zap.Logger
+
+	name    string
+	dbPath  string
+	walDir  string
+	snapDir string
+	cl      *membership.RaftCluster
+
+	skipHashCheck bool
+}
+
+// hasChecksum returns "true" if the file size "n"
+// has appended sha256 hash digest.
+func hasChecksum(n int64) bool {
+	// 512 is chosen because it's a minimum disk sector size
+	// smaller than (and multiplies to) OS page size in most systems
+	return (n % 512) == sha256.Size
+}
+
+// Save fetches snapshot from remote etcd server and saves data to target path.
+func (s *v3Manager) Save(ctx context.Context, cfg clientv3.Config, dbPath string) error {
+	if len(cfg.Endpoints) != 1 {
+		return fmt.Errorf("snapshot must be requested to one selected node, not multiple %v", cfg.Endpoints)
+	}
+	cli, err := clientv3.New(cfg)
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+
+	partpath := dbPath + ".part"
+	defer os.RemoveAll(partpath)
+
+	var f *os.File
+	f, err = os.OpenFile(partpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileutil.PrivateFileMode)
+	if err != nil {
+		return fmt.Errorf("could not open %s (%v)", partpath, err)
+	}
+	s.lg.Info("created temporary db file", zap.String("path", partpath))
+
+	now := time.Now()
+	var rd io.ReadCloser
+	rd, err = cli.Snapshot(ctx)
+	if err != nil {
+		return err
+	}
+	s.lg.Info("fetching snapshot", zap.String("endpoint", cfg.Endpoints[0]))
+	var size int64
+	size, err = io.Copy(f, rd)
+	if err != nil {
+		return err
+	}
+	if !hasChecksum(size) {
+		return fmt.Errorf("sha256 checksum not found [bytes: %d]", size)
+	}
+	if err = fileutil.Fsync(f); err != nil {
+		return err
+	}
+	if err = f.Close(); err != nil {
+		return err
+	}
+	s.lg.Info(
+		"fetched snapshot",
+		zap.String("endpoint", cfg.Endpoints[0]),
+		zap.String("size", humanize.Bytes(uint64(size))),
+		zap.Duration("took", time.Since(now)),
+	)
+
+	if err = os.Rename(partpath, dbPath); err != nil {
+		return fmt.Errorf("could not rename %s to %s (%v)", partpath, dbPath, err)
+	}
+	s.lg.Info("saved", zap.String("path", dbPath))
+	return nil
+}
+
+// Status is the snapshot file status.
+type Status struct {
+	Hash      uint32 `json:"hash"`
+	Revision  int64  `json:"revision"`
+	TotalKey  int    `json:"totalKey"`
+	TotalSize int64  `json:"totalSize"`
+}
+
+// Status returns the snapshot file information.
+func (s *v3Manager) Status(dbPath string) (ds Status, err error) {
+	if _, err = os.Stat(dbPath); err != nil {
+		return ds, err
+	}
+
+	db, err := bolt.Open(dbPath, 0400, &bolt.Options{ReadOnly: true})
+	if err != nil {
+		return ds, err
+	}
+	defer db.Close()
+
+	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
+
+	if err = db.View(func(tx *bolt.Tx) error {
+		// check snapshot file integrity first
+		var dbErrStrings []string
+		for dbErr := range tx.Check() {
+			dbErrStrings = append(dbErrStrings, dbErr.Error())
+		}
+		if len(dbErrStrings) > 0 {
+			return fmt.Errorf("snapshot file integrity check failed. %d errors found.\n"+strings.Join(dbErrStrings, "\n"), len(dbErrStrings))
+		}
+		ds.TotalSize = tx.Size()
+		c := tx.Cursor()
+		for next, _ := c.First(); next != nil; next, _ = c.Next() {
+			b := tx.Bucket(next)
+			if b == nil {
+				return fmt.Errorf("cannot get hash of bucket %s", string(next))
+			}
+			h.Write(next)
+			iskeyb := (string(next) == "key")
+			b.ForEach(func(k, v []byte) error {
+				h.Write(k)
+				h.Write(v)
+				if iskeyb {
+					rev := bytesToRev(k)
+					ds.Revision = rev.main
+				}
+				ds.TotalKey++
+				return nil
+			})
+		}
+		return nil
+	}); err != nil {
+		return ds, err
+	}
+
+	ds.Hash = h.Sum32()
+	return ds, nil
+}
+
+// RestoreConfig configures snapshot restore operation.
+type RestoreConfig struct {
+	// SnapshotPath is the path of snapshot file to restore from.
+	SnapshotPath string
+
+	// Name is the human-readable name of this member.
+	Name string
+
+	// OutputDataDir is the target data directory to save restored data.
+	// OutputDataDir should not conflict with existing etcd data directory.
+	// If OutputDataDir already exists, it will return an error to prevent
+	// unintended data directory overwrites.
+	// If empty, defaults to "[Name].etcd" if not given.
+	OutputDataDir string
+	// OutputWALDir is the target WAL data directory.
+	// If empty, defaults to "[OutputDataDir]/member/wal" if not given.
+	OutputWALDir string
+
+	// PeerURLs is a list of member's peer URLs to advertise to the rest of the cluster.
+	PeerURLs []string
+
+	// InitialCluster is the initial cluster configuration for restore bootstrap.
+	InitialCluster string
+	// InitialClusterToken is the initial cluster token for etcd cluster during restore bootstrap.
+	InitialClusterToken string
+
+	// SkipHashCheck is "true" to ignore snapshot integrity hash value
+	// (required if copied from data directory).
+	SkipHashCheck bool
+}
+
+// Restore restores a new etcd data directory from given snapshot file.
+func (s *v3Manager) Restore(cfg RestoreConfig) error {
+	pURLs, err := types.NewURLs(cfg.PeerURLs)
+	if err != nil {
+		return err
+	}
+	var ics types.URLsMap
+	ics, err = types.NewURLsMap(cfg.InitialCluster)
+	if err != nil {
+		return err
+	}
+
+	srv := etcdserver.ServerConfig{
+		Logger:              s.lg,
+		Name:                cfg.Name,
+		PeerURLs:            pURLs,
+		InitialPeerURLsMap:  ics,
+		InitialClusterToken: cfg.InitialClusterToken,
+	}
+	if err = srv.VerifyBootstrap(); err != nil {
+		return err
+	}
+
+	s.cl, err = membership.NewClusterFromURLsMap(s.lg, cfg.InitialClusterToken, ics)
+	if err != nil {
+		return err
+	}
+
+	dataDir := cfg.OutputDataDir
+	if dataDir == "" {
+		dataDir = cfg.Name + ".etcd"
+	}
+	if fileutil.Exist(dataDir) {
+		return fmt.Errorf("data-dir %q exists", dataDir)
+	}
+
+	walDir := cfg.OutputWALDir
+	if walDir == "" {
+		walDir = filepath.Join(dataDir, "member", "wal")
+	} else if fileutil.Exist(walDir) {
+		return fmt.Errorf("wal-dir %q exists", walDir)
+	}
+
+	s.name = cfg.Name
+	s.dbPath = cfg.SnapshotPath
+	s.walDir = walDir
+	s.snapDir = filepath.Join(dataDir, "member", "snap")
+	s.skipHashCheck = cfg.SkipHashCheck
+
+	s.lg.Info(
+		"restoring snapshot",
+		zap.String("path", s.dbPath),
+		zap.String("wal-dir", s.walDir),
+		zap.String("data-dir", dataDir),
+		zap.String("snap-dir", s.snapDir),
+	)
+	if err = s.saveDB(); err != nil {
+		return err
+	}
+	if err = s.saveWALAndSnap(); err != nil {
+		return err
+	}
+	s.lg.Info(
+		"restored snapshot",
+		zap.String("path", s.dbPath),
+		zap.String("wal-dir", s.walDir),
+		zap.String("data-dir", dataDir),
+		zap.String("snap-dir", s.snapDir),
+	)
+
+	return nil
+}
+
+// saveDB copies the database snapshot to the snapshot directory
+func (s *v3Manager) saveDB() error {
+	f, ferr := os.OpenFile(s.dbPath, os.O_RDONLY, 0600)
+	if ferr != nil {
+		return ferr
+	}
+	defer f.Close()
+
+	// get snapshot integrity hash
+	if _, err := f.Seek(-sha256.Size, io.SeekEnd); err != nil {
+		return err
+	}
+	sha := make([]byte, sha256.Size)
+	if _, err := f.Read(sha); err != nil {
+		return err
+	}
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+
+	if err := fileutil.CreateDirAll(s.snapDir); err != nil {
+		return err
+	}
+
+	dbpath := filepath.Join(s.snapDir, "db")
+	db, dberr := os.OpenFile(dbpath, os.O_RDWR|os.O_CREATE, 0600)
+	if dberr != nil {
+		return dberr
+	}
+	if _, err := io.Copy(db, f); err != nil {
+		return err
+	}
+
+	// truncate away integrity hash, if any.
+	off, serr := db.Seek(0, io.SeekEnd)
+	if serr != nil {
+		return serr
+	}
+	hasHash := hasChecksum(off)
+	if hasHash {
+		if err := db.Truncate(off - sha256.Size); err != nil {
+			return err
+		}
+	}
+
+	if !hasHash && !s.skipHashCheck {
+		return fmt.Errorf("snapshot missing hash but --skip-hash-check=false")
+	}
+
+	if hasHash && !s.skipHashCheck {
+		// check for match
+		if _, err := db.Seek(0, io.SeekStart); err != nil {
+			return err
+		}
+		h := sha256.New()
+		if _, err := io.Copy(h, db); err != nil {
+			return err
+		}
+		dbsha := h.Sum(nil)
+		if !reflect.DeepEqual(sha, dbsha) {
+			return fmt.Errorf("expected sha256 %v, got %v", sha, dbsha)
+		}
+	}
+
+	// db hash is OK, can now modify DB so it can be part of a new cluster
+	db.Close()
+
+	commit := len(s.cl.Members())
+
+	// update consistentIndex so applies go through on etcdserver despite
+	// having a new raft instance
+	be := backend.NewDefaultBackend(dbpath)
+
+	// a lessor never timeouts leases
+	lessor := lease.NewLessor(s.lg, be, lease.LessorConfig{MinLeaseTTL: math.MaxInt64})
+
+	mvs := mvcc.NewStore(s.lg, be, lessor, (*initIndex)(&commit), mvcc.StoreConfig{CompactionBatchLimit: math.MaxInt32})
+	txn := mvs.Write(traceutil.TODO())
+	btx := be.BatchTx()
+	del := func(k, v []byte) error {
+		txn.DeleteRange(k, nil)
+		return nil
+	}
+
+	// delete stored members from old cluster since using new members
+	btx.UnsafeForEach([]byte("members"), del)
+
+	// todo: add back new members when we start to deprecate old snap file.
+	btx.UnsafeForEach([]byte("members_removed"), del)
+
+	// trigger write-out of new consistent index
+	txn.End()
+
+	mvs.Commit()
+	mvs.Close()
+	be.Close()
+
+	return nil
+}
+
+// saveWALAndSnap creates a WAL for the initial cluster
+func (s *v3Manager) saveWALAndSnap() error {
+	if err := fileutil.CreateDirAll(s.walDir); err != nil {
+		return err
+	}
+
+	// add members again to persist them to the store we create.
+	st := v2store.New(etcdserver.StoreClusterPrefix, etcdserver.StoreKeysPrefix)
+	s.cl.SetStore(st)
+	for _, m := range s.cl.Members() {
+		s.cl.AddMember(m)
+	}
+
+	m := s.cl.MemberByName(s.name)
+	md := &etcdserverpb.Metadata{NodeID: uint64(m.ID), ClusterID: uint64(s.cl.ID())}
+	metadata, merr := md.Marshal()
+	if merr != nil {
+		return merr
+	}
+	w, walerr := wal.Create(s.lg, s.walDir, metadata)
+	if walerr != nil {
+		return walerr
+	}
+	defer w.Close()
+
+	peers := make([]raft.Peer, len(s.cl.MemberIDs()))
+	for i, id := range s.cl.MemberIDs() {
+		ctx, err := json.Marshal((*s.cl).Member(id))
+		if err != nil {
+			return err
+		}
+		peers[i] = raft.Peer{ID: uint64(id), Context: ctx}
+	}
+
+	ents := make([]raftpb.Entry, len(peers))
+	nodeIDs := make([]uint64, len(peers))
+	for i, p := range peers {
+		nodeIDs[i] = p.ID
+		cc := raftpb.ConfChange{
+			Type:    raftpb.ConfChangeAddNode,
+			NodeID:  p.ID,
+			Context: p.Context,
+		}
+		d, err := cc.Marshal()
+		if err != nil {
+			return err
+		}
+		ents[i] = raftpb.Entry{
+			Type:  raftpb.EntryConfChange,
+			Term:  1,
+			Index: uint64(i + 1),
+			Data:  d,
+		}
+	}
+
+	commit, term := uint64(len(ents)), uint64(1)
+	if err := w.Save(raftpb.HardState{
+		Term:   term,
+		Vote:   peers[0].ID,
+		Commit: commit,
+	}, ents); err != nil {
+		return err
+	}
+
+	b, berr := st.Save()
+	if berr != nil {
+		return berr
+	}
+	raftSnap := raftpb.Snapshot{
+		Data: b,
+		Metadata: raftpb.SnapshotMetadata{
+			Index: commit,
+			Term:  term,
+			ConfState: raftpb.ConfState{
+				Voters: nodeIDs,
+			},
+		},
+	}
+	sn := snap.New(s.lg, s.snapDir)
+	if err := sn.SaveSnap(raftSnap); err != nil {
+		return err
+	}
+	return w.SaveSnapshot(walpb.Snapshot{Index: commit, Term: term})
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@ -812,6 +812,8 @@ github.com/rancher/wrangler-api/pkg/generated/controllers/rbac
 github.com/rancher/wrangler-api/pkg/generated/controllers/rbac/v1
 # github.com/robfig/cron v1.1.0
 github.com/robfig/cron
+# github.com/robfig/cron/v3 v3.0.1
+github.com/robfig/cron/v3
 # github.com/rootless-containers/rootlesskit v0.10.0
 github.com/rootless-containers/rootlesskit/pkg/api
 github.com/rootless-containers/rootlesskit/pkg/api/client
@ -919,6 +921,7 @@ go.etcd.io/etcd/clientv3/balancer/picker
 go.etcd.io/etcd/clientv3/balancer/resolver/endpoint
 go.etcd.io/etcd/clientv3/concurrency
 go.etcd.io/etcd/clientv3/credentials
+go.etcd.io/etcd/clientv3/snapshot
 go.etcd.io/etcd/embed
 go.etcd.io/etcd/etcdserver
 go.etcd.io/etcd/etcdserver/api