diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 9d2da18552..1feb56b70b 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -27,14 +27,6 @@ RUN if [ "${ARCH}" = 'amd64' ]; then \ ARG SELINUX=true ENV SELINUX $SELINUX -ARG DQLITE=true -ENV DQLITE $DQLITE -COPY --from=rancher/dqlite-build:v1.4.1-r1 /dist/artifacts /usr/src/ -RUN if [ "$DQLITE" = true ]; then \ - tar xzf /usr/src/dqlite.tgz -C / && \ - apk add --allow-untrusted /usr/local/packages/*.apk \ - ;fi - ENV GO111MODULE off ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH diff --git a/pkg/cluster/dqlite.go b/pkg/cluster/dqlite.go deleted file mode 100644 index 9019ea5a82..0000000000 --- a/pkg/cluster/dqlite.go +++ /dev/null @@ -1,143 +0,0 @@ -// +build dqlite - -package cluster - -import ( - "context" - "crypto/tls" - "encoding/json" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/canonical/go-dqlite/client" - "github.com/rancher/dynamiclistener/factory" - "github.com/rancher/k3s/pkg/clientaccess" - "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/k3s/pkg/dqlite" - "github.com/rancher/kine/pkg/endpoint" - v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" - "github.com/sirupsen/logrus" -) - -func (c *Cluster) testClusterDB(ctx context.Context) error { - if !c.dqliteEnabled() { - return nil - } - - dqlite := c.db.(*dqlite.DQLite) - for { - if err := dqlite.Test(ctx); err != nil { - logrus.Infof("Failed to test dqlite connection: %v", err) - } else { - return nil - } - - select { - case <-time.After(2 * time.Second): - case <-ctx.Done(): - return ctx.Err() - } - } -} - -func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) { - if !c.dqliteEnabled() { - return l, handler, nil - } - - dqlite := dqlite.New(c.config.DataDir, c.config.AdvertiseIP, c.config.AdvertisePort, func() v1.NodeController { - if c.runtime.Core == nil { - return nil - } - return c.runtime.Core.Core().V1().Node() - }) - - certs, err := toGetCerts(c.runtime) - if err != nil { - return nil, nil, err - } - - handler, err = dqlite.Start(ctx, c.config.ClusterInit, c.config.ClusterReset, certs, handler) - if err != nil { - return nil, nil, err - } - - if c.config.ClusterReset { - if err := dqlite.Reset(ctx); err == nil { - logrus.Info("Cluster reset successful, now rejoin members") - os.Exit(0) - } else { - logrus.Fatalf("Cluster reset failed: %v", err) - } - } - - c.db = dqlite - if !strings.HasPrefix(c.config.Datastore.Endpoint, "dqlite://") { - c.config.Datastore = endpoint.Config{ - Endpoint: dqlite.StorageEndpoint, - } - } - - return l, handler, err -} - -func (c *Cluster) dqliteEnabled() bool { - stamp := filepath.Join(dqlite.GetDBDir(c.config.DataDir)) - if _, err := os.Stat(stamp); err == nil { - return true - } - - driver, _ := endpoint.ParseStorageEndpoint(c.config.Datastore.Endpoint) - if driver == endpoint.DQLiteBackend { - return true - } - - return c.config.Datastore.Endpoint == "" && (c.config.ClusterInit || (c.config.Token != "" && c.config.JoinURL != "")) -} - -func (c *Cluster) postJoin(ctx context.Context) error { - if !c.dqliteEnabled() { - return nil - } - - resp, err := clientaccess.Get("/db/info", c.clientAccessInfo) - if err != nil { - return err - } - - dqlite := c.db.(*dqlite.DQLite) - var nodes []client.NodeInfo - - if err := json.Unmarshal(resp, &nodes); err != nil { - return err - } - - return dqlite.Join(ctx, nodes) -} - -func toGetCerts(runtime *config.ControlRuntime) (*dqlite.Certs, error) { - clientCA, _, err := factory.LoadCerts(runtime.ClientCA, runtime.ClientCAKey) - if err != nil { - return nil, err - } - - ca, _, err := factory.LoadCerts(runtime.ServerCA, runtime.ServerCAKey) - if err != nil { - return nil, err - } - - clientCert, err := tls.LoadX509KeyPair(runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey) - if err != nil { - return nil, err - } - - return &dqlite.Certs{ - ServerTrust: ca, - ClientTrust: clientCA, - ClientCert: clientCert, - }, nil -} diff --git a/pkg/dqlite/controller/client/controller.go b/pkg/dqlite/controller/client/controller.go deleted file mode 100644 index a58b3834c8..0000000000 --- a/pkg/dqlite/controller/client/controller.go +++ /dev/null @@ -1,185 +0,0 @@ -package client - -import ( - "context" - "fmt" - "strconv" - - "github.com/canonical/go-dqlite/client" - "github.com/canonical/go-dqlite/driver" - "github.com/rancher/k3s/pkg/version" - controllerv1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" - "github.com/sirupsen/logrus" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" -) - -var ( - nodeID = "cluster." + version.Program + ".cattle.io/node-id" - nodeAddress = "cluster." + version.Program + ".cattle.io/node-address" -) - -const ( - allKey = "_all_" - master = "node-role.kubernetes.io/master" -) - -func Register(ctx context.Context, nodeName string, nodeInfo client.NodeInfo, - nodeStore client.NodeStore, nodes controllerv1.NodeController, opts []client.Option) { - h := &handler{ - nodeStore: nodeStore, - nodeController: nodes, - nodeName: nodeName, - id: strconv.FormatUint(nodeInfo.ID, 10), - address: nodeInfo.Address, - ctx: ctx, - opts: opts, - } - nodes.OnChange(ctx, "dqlite-client", h.sync) - nodes.OnRemove(ctx, "dqlite-client", h.onRemove) -} - -type handler struct { - nodeStore client.NodeStore - nodeController controllerv1.NodeController - nodeName string - id string - address string - ctx context.Context - opts []client.Option -} - -func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) { - if key == allKey { - return nil, h.updateNodeStore() - } - - if node == nil { - return nil, nil - } - - if key == h.nodeName { - return h.handleSelf(node) - } - - if node.Labels[master] == "true" { - h.nodeController.Enqueue(allKey) - } - - return node, nil -} - -func (h *handler) ensureExists(address string) error { - c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...) - if err == driver.ErrNoAvailableLeader { - logrus.Fatalf("no dqlite leader found: %v", err) - } else if err != nil { - return err - } - defer c.Close() - - members, err := c.Cluster(h.ctx) - if err != nil { - return err - } - - for _, member := range members { - if member.Address == address { - return nil - } - } - - logrus.Fatalf("Address %s is not member of the cluster", address) - return nil -} - -func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) { - if node.Annotations[nodeID] == h.id && node.Annotations[nodeAddress] == h.address { - return node, h.ensureExists(h.address) - } - - node = node.DeepCopy() - if node.Annotations == nil { - node.Annotations = map[string]string{} - } - node.Annotations[nodeID] = h.id - node.Annotations[nodeAddress] = h.address - - return h.nodeController.Update(node) -} - -func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) { - address := node.Annotations[nodeAddress] - if address == "" { - return node, nil - } - return node, h.delete(address) -} - -func (h *handler) delete(address string) error { - c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...) - if err != nil { - return err - } - defer c.Close() - - members, err := c.Cluster(h.ctx) - if err != nil { - return err - } - - for _, member := range members { - if member.Address == address { - logrus.Infof("Removing %s %d from dqlite", member.Address, member.ID) - return c.Remove(h.ctx, member.ID) - } - } - - return nil -} - -func (h *handler) updateNodeStore() error { - nodes, err := h.nodeController.Cache().List(labels.SelectorFromSet(labels.Set{ - master: "true", - })) - if err != nil { - return err - } - - var ( - nodeInfos []client.NodeInfo - seen = map[string]bool{} - ) - - for _, node := range nodes { - address, ok := node.Annotations[nodeAddress] - if !ok { - continue - } - - nodeIDStr, ok := node.Annotations[nodeID] - if !ok { - continue - } - - id, err := strconv.ParseUint(nodeIDStr, 10, 64) - if err != nil { - logrus.Errorf("invalid %s=%s, must be a number: %v", nodeID, nodeIDStr, err) - continue - } - - if !seen[address] { - nodeInfos = append(nodeInfos, client.NodeInfo{ - ID: id, - Address: address, - }) - seen[address] = true - } - } - - if len(nodeInfos) == 0 { - return fmt.Errorf("not setting dqlient NodeStore len to 0") - } - - return h.nodeStore.Set(h.ctx, nodeInfos) -} diff --git a/pkg/dqlite/dialer/dial.go b/pkg/dqlite/dialer/dial.go deleted file mode 100644 index 87b801a7dc..0000000000 --- a/pkg/dqlite/dialer/dial.go +++ /dev/null @@ -1,36 +0,0 @@ -package dialer - -import ( - "context" - "crypto/tls" - "fmt" - "net" - - "github.com/canonical/go-dqlite/client" - "github.com/rancher/k3s/pkg/dqlite/pipe" -) - -func NewHTTPDialer(advertiseAddress, bindAddress string, tls *tls.Config) (client.DialFunc, error) { - d := &dialer{ - advertiseAddress: advertiseAddress, - bindAddress: bindAddress, - tls: tls, - } - - return d.dial, nil -} - -type dialer struct { - advertiseAddress string - bindAddress string - tls *tls.Config -} - -func (d *dialer) dial(ctx context.Context, address string) (net.Conn, error) { - if address == d.advertiseAddress { - return net.Dial("unix", d.bindAddress) - } - - url := fmt.Sprintf("https://%s/db/connect", address) - return pipe.ToHTTP(ctx, url, d.tls) -} diff --git a/pkg/dqlite/join.go b/pkg/dqlite/join.go deleted file mode 100644 index d6648d2850..0000000000 --- a/pkg/dqlite/join.go +++ /dev/null @@ -1,124 +0,0 @@ -package dqlite - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" - - "github.com/canonical/go-dqlite/client" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -func (d *DQLite) Test(ctx context.Context) error { - var ips []string - peers, err := d.NodeStore.Get(ctx) - if err != nil { - return err - } - - for _, peer := range peers { - ips = append(ips, peer.Address) - } - - logrus.Infof("Testing connection to peers %v", ips) - if err := d.Join(ctx, nil); err != nil { - return err - } - logrus.Infof("Connection OK to peers %v", ips) - return nil -} - -func nodeIDsEqual(testID, currentID uint64) bool { - // this is a test for a bug in v1.0.0. In future versions we don't - // generate node ID higher than 1<<20 so this doesn't matter. But - // basically just ignore the first 32 bits. - return uint32(testID) == uint32(currentID) -} - -func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error { - if len(nodes) > 0 { - if err := d.NodeStore.Set(ctx, nodes); err != nil { - return err - } - } - - client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...) - if err != nil { - return err - } - defer client.Close() - - current, err := client.Cluster(ctx) - if err != nil { - return err - } - - nodeID, err := getClusterID(false, d.DataDir) - if err != nil { - return errors.Wrap(err, "get cluster ID") - } - for _, testNode := range current { - if testNode.Address == d.NodeInfo.Address { - if !nodeIDsEqual(testNode.ID, nodeID) { - if err := d.node.Close(); err != nil { - return errors.Wrap(err, "node close for id reset") - } - if err := writeClusterID(testNode.ID, d.DataDir); err != nil { - return errors.Wrap(err, "restart node to reset ID") - } - logrus.Fatalf("resetting node ID from %d to %d, please restart", nodeID, testNode.ID) - } - return nil - } else if nodeIDsEqual(testNode.ID, nodeID) { - deleteClusterID(d.DataDir) - logrus.Fatalf("node ID %d is in use, please restart", nodeID) - } - } - - if found, err := cleanDir(d.DataDir, true); err != nil { - return err - } else if found { - if err := d.node.Close(); err != nil { - return errors.Wrap(err, "node close for cleaning") - } - _, _ = cleanDir(d.DataDir, false) - return fmt.Errorf("cleaned DB directory, now restart and join") - } - - logrus.Infof("Joining dqlite cluster as address=%s, id=%d", d.NodeInfo.Address, d.NodeInfo.ID) - return client.Add(ctx, d.NodeInfo) -} - -func cleanDir(dataDir string, check bool) (bool, error) { - dbDir := GetDBDir(dataDir) - backupDir := filepath.Join(dbDir, fmt.Sprintf(".backup-%d", time.Now().Unix())) - files, err := ioutil.ReadDir(dbDir) - if err != nil { - return false, errors.Wrap(err, "cleaning dqlite DB dir") - } - - for _, file := range files { - if file.IsDir() || strings.HasPrefix(file.Name(), ".") || ignoreFile[file.Name()] { - continue - } - if check { - return true, nil - } - if err := os.MkdirAll(backupDir, 0700); err != nil { - return false, errors.Wrapf(err, "creating backup dir %s", backupDir) - } - oldName := filepath.Join(dbDir, file.Name()) - newName := filepath.Join(backupDir, file.Name()) - logrus.Infof("Backing up %s => %s", oldName, newName) - if err := os.Rename(oldName, newName); err != nil { - return false, errors.Wrapf(err, "backup %s", oldName) - } - } - - return false, nil -} diff --git a/pkg/dqlite/log.go b/pkg/dqlite/log.go deleted file mode 100644 index bbd07c8017..0000000000 --- a/pkg/dqlite/log.go +++ /dev/null @@ -1,27 +0,0 @@ -package dqlite - -import ( - "strings" - - "github.com/canonical/go-dqlite/client" - "github.com/sirupsen/logrus" -) - -func log() client.LogFunc { - return func(level client.LogLevel, s string, i ...interface{}) { - switch level { - case client.LogDebug: - logrus.Debugf(s, i...) - case client.LogError: - logrus.Errorf(s, i...) - case client.LogInfo: - if strings.HasPrefix(s, "connected") { - logrus.Debugf(s, i...) - } else { - logrus.Infof(s, i...) - } - case client.LogWarn: - logrus.Warnf(s, i...) - } - } -} diff --git a/pkg/dqlite/pipe/http.go b/pkg/dqlite/pipe/http.go deleted file mode 100644 index 036d4c69a2..0000000000 --- a/pkg/dqlite/pipe/http.go +++ /dev/null @@ -1,56 +0,0 @@ -package pipe - -import ( - "bufio" - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - - "github.com/pkg/errors" -) - -func ToHTTP(ctx context.Context, url string, tlsConfig *tls.Config) (net.Conn, error) { - request, err := http.NewRequest(http.MethodPost, url, nil) - if err != nil { - return nil, err - } - - request = request.WithContext(ctx) - netDial := &net.Dialer{} - - if deadline, ok := ctx.Deadline(); ok { - netDial.Deadline = deadline - } - - conn, err := tls.DialWithDialer(netDial, "tcp", request.URL.Host, tlsConfig) - if err != nil { - return nil, errors.Wrap(err, "tls dial") - } - - err = request.Write(conn) - if err != nil { - return nil, errors.Wrap(err, "request write") - } - - response, err := http.ReadResponse(bufio.NewReader(conn), request) - if err != nil { - return nil, errors.Wrap(err, "read request") - } - if response.StatusCode != http.StatusSwitchingProtocols { - return nil, fmt.Errorf("expected 101 response, got: %d", response.StatusCode) - } - - listener, err := net.Listen("unix", "") - if err != nil { - return nil, errors.Wrap(err, "Failed to create unix listener") - } - defer listener.Close() - - if err := Unix(conn, listener.Addr().String()); err != nil { - return nil, err - } - - return listener.Accept() -} diff --git a/pkg/dqlite/pipe/pipe.go b/pkg/dqlite/pipe/pipe.go deleted file mode 100644 index fc00031318..0000000000 --- a/pkg/dqlite/pipe/pipe.go +++ /dev/null @@ -1,46 +0,0 @@ -package pipe - -import ( - "io" - "net" - - "github.com/lxc/lxd/shared/eagain" - "github.com/sirupsen/logrus" -) - -func UnixPiper(srcs <-chan net.Conn, bindAddress string) { - for src := range srcs { - go Unix(src, bindAddress) - } -} - -func Unix(src net.Conn, target string) error { - dst, err := net.Dial("unix", target) - if err != nil { - src.Close() - return err - } - - Connect(src, dst) - return nil -} - -func Connect(src net.Conn, dst net.Conn) { - go func() { - _, err := io.Copy(eagain.Writer{Writer: dst}, eagain.Reader{Reader: src}) - if err != nil && err != io.EOF { - logrus.Debugf("copy pipe src->dst closed: %v", err) - } - src.Close() - dst.Close() - }() - - go func() { - _, err := io.Copy(eagain.Writer{Writer: src}, eagain.Reader{Reader: dst}) - if err != nil { - logrus.Debugf("copy pipe dst->src closed: %v", err) - } - src.Close() - dst.Close() - }() -} diff --git a/pkg/dqlite/proxy.go b/pkg/dqlite/proxy.go deleted file mode 100644 index 0387ad023e..0000000000 --- a/pkg/dqlite/proxy.go +++ /dev/null @@ -1,53 +0,0 @@ -package dqlite - -import ( - "context" - "net" - "net/http" - - "github.com/pkg/errors" - "github.com/rancher/k3s/pkg/dqlite/pipe" -) - -var ( - upgradeResponse = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: dqlite\r\n\r\n") -) - -type proxy struct { - conns chan net.Conn -} - -func newProxy(ctx context.Context, bindAddress string) http.Handler { - p := &proxy{ - conns: make(chan net.Conn, 100), - } - go func() { - <-ctx.Done() - close(p.conns) - }() - go pipe.UnixPiper(p.conns, bindAddress) - - return p -} - -func (h *proxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - hijacker, ok := rw.(http.Hijacker) - if !ok { - http.Error(rw, "failed to hijack", http.StatusInternalServerError) - return - } - - conn, _, err := hijacker.Hijack() - if err != nil { - err := errors.Wrap(err, "Hijack connection") - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - - if n, err := conn.Write(upgradeResponse); err != nil || n != len(upgradeResponse) { - conn.Close() - return - } - - h.conns <- conn -} diff --git a/pkg/dqlite/reset.go b/pkg/dqlite/reset.go deleted file mode 100644 index d9a379897f..0000000000 --- a/pkg/dqlite/reset.go +++ /dev/null @@ -1,15 +0,0 @@ -package dqlite - -import ( - "context" - - "github.com/canonical/go-dqlite/client" - "github.com/sirupsen/logrus" -) - -func (d *DQLite) Reset(ctx context.Context) error { - logrus.Infof("Resetting cluster to single master") - return d.node.Recover([]client.NodeInfo{ - d.NodeInfo, - }) -} diff --git a/pkg/dqlite/router.go b/pkg/dqlite/router.go deleted file mode 100644 index a88ce07e4a..0000000000 --- a/pkg/dqlite/router.go +++ /dev/null @@ -1,80 +0,0 @@ -package dqlite - -import ( - "context" - "crypto/x509" - "encoding/json" - "net/http" - - "github.com/canonical/go-dqlite" - "github.com/canonical/go-dqlite/client" - "github.com/gorilla/mux" -) - -func router(ctx context.Context, next http.Handler, nodeInfo dqlite.NodeInfo, clientCA *x509.Certificate, clientCN string, bindAddress string) http.Handler { - mux := mux.NewRouter() - mux.Handle("/db/connect", newChecker(newProxy(ctx, bindAddress), clientCA, clientCN)) - mux.Handle("/db/info", infoHandler(ctx, nodeInfo, bindAddress)) - mux.NotFoundHandler = next - return mux -} - -func infoHandler(ctx context.Context, nodeInfo dqlite.NodeInfo, bindAddress string) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - client, err := client.New(ctx, bindAddress, client.WithLogFunc(log())) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - defer client.Close() - - info, err := client.Cluster(ctx) - if err != nil { - http.Error(rw, err.Error(), http.StatusInternalServerError) - return - } - - rw.Header().Set("Content-Type", "application/json") - json.NewEncoder(rw).Encode(info) - }) -} - -type checker struct { - next http.Handler - verify x509.VerifyOptions - cn string -} - -func newChecker(next http.Handler, ca *x509.Certificate, cn string) http.Handler { - pool := x509.NewCertPool() - pool.AddCert(ca) - return &checker{ - next: next, - verify: x509.VerifyOptions{ - Roots: pool, - KeyUsages: []x509.ExtKeyUsage{ - x509.ExtKeyUsageClientAuth, - }, - DNSName: cn, - }, - cn: cn, - } -} - -func (c *checker) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - if !c.check(req) { - http.Error(rw, "unauthorized", http.StatusUnauthorized) - return - } - c.next.ServeHTTP(rw, req) -} - -func (c *checker) check(r *http.Request) bool { - for _, cert := range r.TLS.PeerCertificates { - _, err := cert.Verify(c.verify) - if err == nil { - return cert.Subject.CommonName == c.cn - } - } - return false -} diff --git a/pkg/dqlite/server.go b/pkg/dqlite/server.go deleted file mode 100644 index fd7db0f4c5..0000000000 --- a/pkg/dqlite/server.go +++ /dev/null @@ -1,245 +0,0 @@ -package dqlite - -import ( - "context" - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/canonical/go-dqlite" - "github.com/canonical/go-dqlite/client" - "github.com/pkg/errors" - controllerclient "github.com/rancher/k3s/pkg/dqlite/controller/client" - "github.com/rancher/k3s/pkg/dqlite/dialer" - dqlitedriver "github.com/rancher/kine/pkg/drivers/dqlite" - v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/util/net" -) - -const ( - PeersFile = "peers.db" - NodeIDFile = "node-id" -) - -var ( - ignoreFile = map[string]bool{ - PeersFile: true, - NodeIDFile: true, - } -) - -type Certs struct { - ServerTrust *x509.Certificate - ClientTrust *x509.Certificate - ClientCert tls.Certificate -} - -type DQLite struct { - ClientCA string - ClientCAKey string - ClientCert string - ClientCertKey string - ServerCA string - ServerCAKey string - AdvertiseIP string - AdvertisePort int - DataDir string - NodeStore client.NodeStore - NodeInfo client.NodeInfo - node *dqlite.Node - StorageEndpoint string - NodeControllerGetter NodeControllerGetter - clientOpts []client.Option -} - -type NodeControllerGetter func() v1.NodeController - -func New(dataDir, advertiseIP string, advertisePort int, getter NodeControllerGetter) *DQLite { - return &DQLite{ - AdvertiseIP: advertiseIP, - AdvertisePort: advertisePort, - DataDir: dataDir, - NodeControllerGetter: getter, - } -} - -func (d *DQLite) Start(ctx context.Context, initCluster, resetCluster bool, certs *Certs, next http.Handler) (http.Handler, error) { - bindAddress := d.getBindAddress() - - clientTLSConfig, err := getClientTLSConfig(certs.ClientCert, certs.ServerTrust) - if err != nil { - return nil, err - } - - advertise, err := getAdvertiseAddress(d.AdvertiseIP, d.AdvertisePort) - if err != nil { - return nil, errors.Wrap(err, "get advertise address") - } - - dial, err := getDialer(advertise, bindAddress, clientTLSConfig) - if err != nil { - return nil, err - } - - dqlitedriver.Dialer = dial - dqlitedriver.Logger = log() - - d.clientOpts = append(d.clientOpts, client.WithDialFunc(dial), client.WithLogFunc(log())) - - nodeInfo, node, err := getNode(d.DataDir, advertise, bindAddress, initCluster, dial) - if err != nil { - return nil, err - } - - d.NodeInfo = nodeInfo - d.node = node - - go func() { - <-ctx.Done() - node.Close() - }() - - if err := d.nodeStore(ctx, initCluster); err != nil { - return nil, err - } - - go d.startController(ctx) - - if !resetCluster { - if err := node.Start(); err != nil { - return nil, err - } - } - - return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), nil -} - -func (d *DQLite) startController(ctx context.Context) { - for { - if nc := d.NodeControllerGetter(); nc != nil { - if os.Getenv("NODE_NAME") == "" { - logrus.Errorf("--disable-agent is not compatible with dqlite") - } else { - break - } - } - time.Sleep(time.Second) - } - - controllerclient.Register(ctx, os.Getenv("NODE_NAME"), d.NodeInfo, d.NodeStore, d.NodeControllerGetter(), d.clientOpts) -} - -func (d *DQLite) nodeStore(ctx context.Context, initCluster bool) error { - peerDB := filepath.Join(GetDBDir(d.DataDir), PeersFile) - ns, err := client.DefaultNodeStore(peerDB) - if err != nil { - return err - } - d.NodeStore = ns - d.StorageEndpoint = fmt.Sprintf("dqlite://?peer-file=%s", peerDB) - if initCluster { - if err := dqlitedriver.AddPeers(ctx, d.NodeStore, d.NodeInfo); err != nil { - return err - } - } - return nil -} - -func getAdvertiseAddress(advertiseIP string, advertisePort int) (string, error) { - ip := advertiseIP - if ip == "" { - ipAddr, err := net.ChooseHostInterface() - if err != nil { - return "", err - } - ip = ipAddr.String() - } - - return fmt.Sprintf("%s:%d", ip, advertisePort), nil -} - -func getClientTLSConfig(cert tls.Certificate, ca *x509.Certificate) (*tls.Config, error) { - tlsConfig := &tls.Config{ - RootCAs: x509.NewCertPool(), - Certificates: []tls.Certificate{ - cert, - }, - ServerName: "kubernetes", - } - tlsConfig.RootCAs.AddCert(ca) - - return tlsConfig, nil -} - -func getDialer(advertiseAddress, bindAddress string, tlsConfig *tls.Config) (client.DialFunc, error) { - return dialer.NewHTTPDialer(advertiseAddress, bindAddress, tlsConfig) -} - -func GetDBDir(dataDir string) string { - return filepath.Join(dataDir, "db", "state.dqlite") -} - -func getNode(dataDir string, advertiseAddress, bindAddress string, initCluster bool, dial client.DialFunc) (dqlite.NodeInfo, *dqlite.Node, error) { - id, err := getClusterID(initCluster, dataDir) - if err != nil { - return dqlite.NodeInfo{}, nil, errors.Wrap(err, "reading cluster id") - } - - dbDir := GetDBDir(dataDir) - - node, err := dqlite.New(id, advertiseAddress, dbDir, - dqlite.WithBindAddress(bindAddress), - dqlite.WithDialFunc(dial), - dqlite.WithNetworkLatency(20*time.Millisecond)) - return dqlite.NodeInfo{ - ID: id, - Address: advertiseAddress, - }, node, err -} - -func writeClusterID(id uint64, dataDir string) error { - idFile := filepath.Join(GetDBDir(dataDir), NodeIDFile) - if err := os.MkdirAll(filepath.Dir(idFile), 0700); err != nil { - return err - } - return ioutil.WriteFile(idFile, []byte(strconv.FormatUint(id, 10)), 0644) -} - -func deleteClusterID(dataDir string) { - os.Remove(filepath.Join(GetDBDir(dataDir), NodeIDFile)) -} - -func getClusterID(initCluster bool, dataDir string) (uint64, error) { - idFile := filepath.Join(GetDBDir(dataDir), NodeIDFile) - content, err := ioutil.ReadFile(idFile) - if os.IsNotExist(err) { - content = nil - } else if err != nil { - return 0, err - } - - idStr := strings.TrimSpace(string(content)) - if idStr == "" { - id := uint64(rand.Intn(1 << 20)) - if initCluster { - id = 1 - } - return id, writeClusterID(id, dataDir) - } - - return strconv.ParseUint(idStr, 10, 64) -} - -func (d *DQLite) getBindAddress() string { - // only anonymous works??? - return "@" + filepath.Join(GetDBDir(d.DataDir), "dqlite.sock") -} diff --git a/scripts/build b/scripts/build index 4ca6333c4a..66416ca627 100755 --- a/scripts/build +++ b/scripts/build @@ -39,15 +39,10 @@ STATIC=" -extldflags '-static' " -if [ "$DQLITE" = "true" ]; then - DQLITE_TAGS="dqlite" - DQLITE_STATIC_SQLITE="-luv -lraft -lco" -fi - STATIC_SQLITE=" - -extldflags '-static -lm -ldl -lz -lpthread $DQLITE_STATIC_SQLITE' + -extldflags '-static -lm -ldl -lz -lpthread' " -TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless $DQLITE_TAGS" +TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless" RUNC_TAGS="apparmor seccomp" RUNC_STATIC="static" diff --git a/scripts/package-cli b/scripts/package-cli index ee55910187..d73fc1ec3f 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -56,9 +56,6 @@ LDFLAGS=" -w -s " STATIC="-extldflags '-static'" -if [ "$DQLITE" = 'true' ]; then - DQLITE_TAGS='dqlite' -fi -CGO_ENABLED=0 "${GO}" build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go +CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go ./scripts/build-upload ${CMD_NAME} ${COMMIT} diff --git a/scripts/provision/generic/alpine310/vagrant b/scripts/provision/generic/alpine310/vagrant index ecd6898989..aec65b1749 100755 --- a/scripts/provision/generic/alpine310/vagrant +++ b/scripts/provision/generic/alpine310/vagrant @@ -3,11 +3,9 @@ set -ve apk add -q -f curl libc6-compat tzdata download_go -download_dqlite # --- cat </etc/profile.d/build.sh export SELINUX=true -export DQLITE=true export STATIC_BUILD=true EOF . /etc/profile.d/build.sh @@ -28,4 +26,4 @@ EOF else echo "Using host docker server v$(cat /tmp/docker-server-version)" fi -) \ No newline at end of file +) diff --git a/scripts/provision/generic/centos7/vagrant b/scripts/provision/generic/centos7/vagrant index 448d59edd4..bf88a92b62 100755 --- a/scripts/provision/generic/centos7/vagrant +++ b/scripts/provision/generic/centos7/vagrant @@ -5,7 +5,6 @@ download_go # --- cat </etc/profile.d/build.sh export SELINUX=true -# export DQLITE=true # export STATIC_BUILD=true EOF . /etc/profile.d/build.sh diff --git a/scripts/provision/generic/ubuntu1804/vagrant b/scripts/provision/generic/ubuntu1804/vagrant index 4cb6b0c771..fd784a05cc 100755 --- a/scripts/provision/generic/ubuntu1804/vagrant +++ b/scripts/provision/generic/ubuntu1804/vagrant @@ -5,7 +5,6 @@ download_go # --- cat </etc/profile.d/build.sh export SELINUX=true -# export DQLITE=true # export STATIC_BUILD=true EOF . /etc/profile.d/build.sh diff --git a/scripts/provision/vagrant b/scripts/provision/vagrant index 7bb0e1ec0f..70adbe3a61 100755 --- a/scripts/provision/vagrant +++ b/scripts/provision/vagrant @@ -79,18 +79,6 @@ download_go() { curl -sL https://storage.googleapis.com/golang/go${goversion}.linux-${ARCH}.tar.gz | tar -xzf - -C /usr/local } -# --- Utility function to download dqlite -download_dqlite() { - dqliteURL="https://github.com/$(grep dqlite-build Dockerfile.dapper | sed -e 's/^.*--from=\([^ ]*\).*$/\1/' -e 's|:|/releases/download/|')/dqlite-$ARCH.tgz" - if [ -z "$dqliteURL" ]; then - echo 'Cannot find dqlite URL to fetch' - return 1 - fi - mkdir -p /usr/src/ - echo "Downloading DQLITE from $dqliteURL" - curl -sL $dqliteURL -o /usr/src/dqlite.tgz -} - # --- Run vagrant provision script if available if [ ! -f "${PROVISION}" ]; then echo "WARNING: Unable to execute provision script \"${PROVISION}\"" diff --git a/scripts/test b/scripts/test index 16335434c5..795eefc2d7 100755 --- a/scripts/test +++ b/scripts/test @@ -27,4 +27,3 @@ E2E_OUTPUT=$artifacts test-run-sonobuoy test-run-sonobuoy mysql test-run-sonobuoy postgres -# test-run-sonobuoy dqlite \ No newline at end of file