2019-10-27 05:53:25 +00:00
|
|
|
package cluster
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2021-02-12 15:35:57 +00:00
|
|
|
"net/url"
|
2021-06-04 22:05:47 +00:00
|
|
|
"runtime"
|
2019-11-11 22:18:26 +00:00
|
|
|
"strings"
|
2019-10-27 05:53:25 +00:00
|
|
|
|
2022-03-02 23:47:27 +00:00
|
|
|
"github.com/k3s-io/k3s/pkg/clientaccess"
|
|
|
|
"github.com/k3s-io/k3s/pkg/cluster/managed"
|
|
|
|
"github.com/k3s-io/k3s/pkg/daemons/config"
|
|
|
|
"github.com/k3s-io/k3s/pkg/etcd"
|
2020-11-30 23:45:22 +00:00
|
|
|
"github.com/k3s-io/kine/pkg/endpoint"
|
2019-12-16 18:44:13 +00:00
|
|
|
"github.com/pkg/errors"
|
2021-03-01 22:19:57 +00:00
|
|
|
"github.com/sirupsen/logrus"
|
2019-10-27 05:53:25 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
type Cluster struct {
|
|
|
|
clientAccessInfo *clientaccess.Info
|
|
|
|
config *config.Control
|
2020-05-05 21:59:15 +00:00
|
|
|
managedDB managed.Driver
|
2019-11-11 22:18:26 +00:00
|
|
|
joining bool
|
2021-10-07 19:47:00 +00:00
|
|
|
storageStarted bool
|
2019-11-11 22:18:26 +00:00
|
|
|
saveBootstrap bool
|
2021-10-07 19:47:00 +00:00
|
|
|
shouldBootstrap bool
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// Start creates the dynamic tls listener, http request handler,
|
|
|
|
// handles starting and writing/reading bootstrap data, and returns a channel
|
2021-03-11 20:07:40 +00:00
|
|
|
// that will be closed when datastore is ready. If embedded etcd is in use,
|
|
|
|
// a secondary call to Cluster.save is made.
|
2020-05-05 21:59:15 +00:00
|
|
|
func (c *Cluster) Start(ctx context.Context) (<-chan struct{}, error) {
|
2020-09-24 06:29:25 +00:00
|
|
|
// Set up the dynamiclistener and http request handlers
|
2020-05-05 21:59:15 +00:00
|
|
|
if err := c.initClusterAndHTTPS(ctx); err != nil {
|
2020-09-24 06:29:25 +00:00
|
|
|
return nil, errors.Wrap(err, "init cluster datastore and https")
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
|
2021-02-12 15:35:57 +00:00
|
|
|
if c.config.DisableETCD {
|
|
|
|
ready := make(chan struct{})
|
|
|
|
defer close(ready)
|
|
|
|
|
|
|
|
// try to get /db/info urls first before attempting to use join url
|
2021-03-01 21:50:50 +00:00
|
|
|
clientURLs, _, err := etcd.ClientURLs(ctx, c.clientAccessInfo, c.config.PrivateIP)
|
2021-02-12 15:35:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if len(clientURLs) < 1 {
|
|
|
|
clientURL, err := url.Parse(c.config.JoinURL)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
clientURL.Host = clientURL.Hostname() + ":2379"
|
|
|
|
clientURLs = append(clientURLs, clientURL.String())
|
|
|
|
}
|
2021-03-08 22:10:00 +00:00
|
|
|
etcdProxy, err := etcd.NewETCDProxy(ctx, true, c.config.DataDir, clientURLs[0])
|
2021-02-12 15:35:57 +00:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
c.setupEtcdProxy(ctx, etcdProxy)
|
2021-03-01 21:50:50 +00:00
|
|
|
|
|
|
|
// remove etcd member if it exists
|
|
|
|
if err := c.managedDB.RemoveSelf(ctx); err != nil {
|
2021-03-01 22:19:57 +00:00
|
|
|
logrus.Warnf("Failed to remove this node from etcd members")
|
2021-03-01 21:50:50 +00:00
|
|
|
}
|
|
|
|
|
2022-03-08 17:04:31 +00:00
|
|
|
c.config.Runtime.EtcdConfig.Endpoints = strings.Split(c.config.Datastore.Endpoint, ",")
|
|
|
|
c.config.Runtime.EtcdConfig.TLSConfig = c.config.Datastore.BackendTLSConfig
|
|
|
|
|
2021-02-12 15:35:57 +00:00
|
|
|
return ready, nil
|
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// start managed database (if necessary)
|
2020-05-05 21:59:15 +00:00
|
|
|
if err := c.start(ctx); err != nil {
|
2020-09-24 06:29:25 +00:00
|
|
|
return nil, errors.Wrap(err, "start managed database")
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// get the wait channel for testing managed database readiness
|
2020-05-05 21:59:15 +00:00
|
|
|
ready, err := c.testClusterDB(ctx)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
|
2021-11-10 12:33:42 +00:00
|
|
|
if err := c.startStorage(ctx); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// if necessary, store bootstrap data to datastore
|
2019-11-11 22:18:26 +00:00
|
|
|
if c.saveBootstrap {
|
2022-02-25 17:26:13 +00:00
|
|
|
if err := Save(ctx, c.config, false); err != nil {
|
2020-05-05 21:59:15 +00:00
|
|
|
return nil, err
|
2019-11-11 22:18:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-17 21:33:00 +00:00
|
|
|
// at this point, if etcd is in use, it's bootstrapping is complete
|
|
|
|
// so save the bootstrap data. We will need for etcd to be up. If
|
|
|
|
// the save call returns an error, we panic since subsequent etcd
|
|
|
|
// snapshots will be empty.
|
2021-03-11 20:07:40 +00:00
|
|
|
if c.managedDB != nil {
|
2021-03-17 21:33:00 +00:00
|
|
|
go func() {
|
2021-06-04 22:05:47 +00:00
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case <-ready:
|
2022-02-25 17:26:13 +00:00
|
|
|
if err := Save(ctx, c.config, false); err != nil {
|
2021-06-04 22:05:47 +00:00
|
|
|
panic(err)
|
|
|
|
}
|
|
|
|
|
2021-07-09 17:22:49 +00:00
|
|
|
if !c.config.EtcdDisableSnapshots {
|
2021-11-29 18:30:04 +00:00
|
|
|
if err := c.managedDB.ReconcileSnapshotData(ctx); err != nil {
|
2021-07-09 17:22:49 +00:00
|
|
|
logrus.Errorf("Failed to record snapshots for cluster: %v", err)
|
|
|
|
}
|
2021-06-04 22:05:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return
|
|
|
|
default:
|
|
|
|
runtime.Gosched()
|
2021-03-17 21:33:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2021-03-11 20:07:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return ready, nil
|
2019-11-11 22:18:26 +00:00
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// startStorage starts the kine listener and configures the endpoints, if necessary.
|
|
|
|
// This calls into the kine endpoint code, which sets up the database client
|
|
|
|
// and unix domain socket listener if using an external database. In the case of an etcd
|
|
|
|
// backend it just returns the user-provided etcd endpoints and tls config.
|
2019-11-11 22:18:26 +00:00
|
|
|
func (c *Cluster) startStorage(ctx context.Context) error {
|
|
|
|
if c.storageStarted {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
c.storageStarted = true
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// start listening on the kine socket as an etcd endpoint, or return the external etcd endpoints
|
2019-11-16 00:12:27 +00:00
|
|
|
etcdConfig, err := endpoint.Listen(ctx, c.config.Datastore)
|
2019-11-11 22:18:26 +00:00
|
|
|
if err != nil {
|
2019-12-16 18:44:13 +00:00
|
|
|
return errors.Wrap(err, "creating storage endpoint")
|
2019-10-31 02:05:40 +00:00
|
|
|
}
|
|
|
|
|
2020-09-24 06:29:25 +00:00
|
|
|
// Persist the returned etcd configuration. We decide if we're doing leader election for embedded controllers
|
|
|
|
// based on what the kine wrapper tells us about the datastore. Single-node datastores like sqlite don't require
|
|
|
|
// leader election, while basically all others (etcd, external database, etc) do since they allow multiple servers.
|
2022-02-25 17:26:13 +00:00
|
|
|
c.config.Runtime.EtcdConfig = etcdConfig
|
2021-08-30 20:43:25 +00:00
|
|
|
c.config.Datastore.BackendTLSConfig = etcdConfig.TLSConfig
|
2019-11-16 00:12:27 +00:00
|
|
|
c.config.Datastore.Endpoint = strings.Join(etcdConfig.Endpoints, ",")
|
2019-11-11 22:18:26 +00:00
|
|
|
c.config.NoLeaderElect = !etcdConfig.LeaderElect
|
|
|
|
return nil
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
|
2021-10-07 19:47:00 +00:00
|
|
|
// New creates an initial cluster using the provided configuration.
|
2019-10-27 05:53:25 +00:00
|
|
|
func New(config *config.Control) *Cluster {
|
|
|
|
return &Cluster{
|
2022-02-24 19:01:14 +00:00
|
|
|
config: config,
|
2019-10-27 05:53:25 +00:00
|
|
|
}
|
|
|
|
}
|