github
/
k3s
mirror of https://github.com/k3s-io/k3s
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Delete dqlite

pull/1770/head
Darren Shepherd 2020-05-05 16:26:05 -07:00
parent 7e59c0801e
commit 4317a91b96
19 changed files with 4 additions and 1047 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile.dapper
View File

@ -27,14 +27,6 @@ RUN if [ "${ARCH}" = 'amd64' ]; then \
ARG SELINUX=true
ENV SELINUX $SELINUX
ARG DQLITE=true
ENV DQLITE $DQLITE
COPY --from=rancher/dqlite-build:v1.4.1-r1 /dist/artifacts /usr/src/
RUN if [ "$DQLITE" = true ]; then \
tar xzf /usr/src/dqlite.tgz -C / && \
apk add --allow-untrusted /usr/local/packages/*.apk \
;fi
ENV GO111MODULE off
ENV DAPPER_RUN_ARGS --privileged -v k3s-cache:/go/src/github.com/rancher/k3s/.cache
ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME SKIP_VALIDATE GCLOUD_AUTH

143
pkg/cluster/dqlite.go
View File

@ -1,143 +0,0 @@
// +build dqlite
package cluster
import (
"context"
"crypto/tls"
"encoding/json"
"net"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/canonical/go-dqlite/client"
"github.com/rancher/dynamiclistener/factory"
"github.com/rancher/k3s/pkg/clientaccess"
"github.com/rancher/k3s/pkg/daemons/config"
"github.com/rancher/k3s/pkg/dqlite"
"github.com/rancher/kine/pkg/endpoint"
v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
)
func (c *Cluster) testClusterDB(ctx context.Context) error {
if !c.dqliteEnabled() {
return nil
}
dqlite := c.db.(*dqlite.DQLite)
for {
if err := dqlite.Test(ctx); err != nil {
logrus.Infof("Failed to test dqlite connection: %v", err)
} else {
return nil
}
select {
case <-time.After(2 * time.Second):
case <-ctx.Done():
return ctx.Err()
}
}
}
func (c *Cluster) initClusterDB(ctx context.Context, l net.Listener, handler http.Handler) (net.Listener, http.Handler, error) {
if !c.dqliteEnabled() {
return l, handler, nil
}
dqlite := dqlite.New(c.config.DataDir, c.config.AdvertiseIP, c.config.AdvertisePort, func() v1.NodeController {
if c.runtime.Core == nil {
return nil
}
return c.runtime.Core.Core().V1().Node()
})
certs, err := toGetCerts(c.runtime)
if err != nil {
return nil, nil, err
}
handler, err = dqlite.Start(ctx, c.config.ClusterInit, c.config.ClusterReset, certs, handler)
if err != nil {
return nil, nil, err
}
if c.config.ClusterReset {
if err := dqlite.Reset(ctx); err == nil {
logrus.Info("Cluster reset successful, now rejoin members")
os.Exit(0)
} else {
logrus.Fatalf("Cluster reset failed: %v", err)
}
}
c.db = dqlite
if !strings.HasPrefix(c.config.Datastore.Endpoint, "dqlite://") {
c.config.Datastore = endpoint.Config{
Endpoint: dqlite.StorageEndpoint,
}
}
return l, handler, err
}
func (c *Cluster) dqliteEnabled() bool {
stamp := filepath.Join(dqlite.GetDBDir(c.config.DataDir))
if _, err := os.Stat(stamp); err == nil {
return true
}
driver, _ := endpoint.ParseStorageEndpoint(c.config.Datastore.Endpoint)
if driver == endpoint.DQLiteBackend {
return true
}
return c.config.Datastore.Endpoint == "" && (c.config.ClusterInit || (c.config.Token != "" && c.config.JoinURL != ""))
}
func (c *Cluster) postJoin(ctx context.Context) error {
if !c.dqliteEnabled() {
return nil
}
resp, err := clientaccess.Get("/db/info", c.clientAccessInfo)
if err != nil {
return err
}
dqlite := c.db.(*dqlite.DQLite)
var nodes []client.NodeInfo
if err := json.Unmarshal(resp, &nodes); err != nil {
return err
}
return dqlite.Join(ctx, nodes)
}
func toGetCerts(runtime *config.ControlRuntime) (*dqlite.Certs, error) {
clientCA, _, err := factory.LoadCerts(runtime.ClientCA, runtime.ClientCAKey)
if err != nil {
return nil, err
}
ca, _, err := factory.LoadCerts(runtime.ServerCA, runtime.ServerCAKey)
if err != nil {
return nil, err
}
clientCert, err := tls.LoadX509KeyPair(runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey)
if err != nil {
return nil, err
}
return &dqlite.Certs{
ServerTrust: ca,
ClientTrust: clientCA,
ClientCert: clientCert,
}, nil
}

185
pkg/dqlite/controller/client/controller.go
View File

@ -1,185 +0,0 @@
package client
import (
"context"
"fmt"
"strconv"
"github.com/canonical/go-dqlite/client"
"github.com/canonical/go-dqlite/driver"
"github.com/rancher/k3s/pkg/version"
controllerv1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
)
var (
nodeID = "cluster." + version.Program + ".cattle.io/node-id"
nodeAddress = "cluster." + version.Program + ".cattle.io/node-address"
)
const (
allKey = "_all_"
master = "node-role.kubernetes.io/master"
)
func Register(ctx context.Context, nodeName string, nodeInfo client.NodeInfo,
nodeStore client.NodeStore, nodes controllerv1.NodeController, opts []client.Option) {
h := &handler{
nodeStore: nodeStore,
nodeController: nodes,
nodeName: nodeName,
id: strconv.FormatUint(nodeInfo.ID, 10),
address: nodeInfo.Address,
ctx: ctx,
opts: opts,
}
nodes.OnChange(ctx, "dqlite-client", h.sync)
nodes.OnRemove(ctx, "dqlite-client", h.onRemove)
}
type handler struct {
nodeStore client.NodeStore
nodeController controllerv1.NodeController
nodeName string
id string
address string
ctx context.Context
opts []client.Option
}
func (h *handler) sync(key string, node *v1.Node) (*v1.Node, error) {
if key == allKey {
return nil, h.updateNodeStore()
}
if node == nil {
return nil, nil
}
if key == h.nodeName {
return h.handleSelf(node)
}
if node.Labels[master] == "true" {
h.nodeController.Enqueue(allKey)
}
return node, nil
}
func (h *handler) ensureExists(address string) error {
c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...)
if err == driver.ErrNoAvailableLeader {
logrus.Fatalf("no dqlite leader found: %v", err)
} else if err != nil {
return err
}
defer c.Close()
members, err := c.Cluster(h.ctx)
if err != nil {
return err
}
for _, member := range members {
if member.Address == address {
return nil
}
}
logrus.Fatalf("Address %s is not member of the cluster", address)
return nil
}
func (h *handler) handleSelf(node *v1.Node) (*v1.Node, error) {
if node.Annotations[nodeID] == h.id && node.Annotations[nodeAddress] == h.address {
return node, h.ensureExists(h.address)
}
node = node.DeepCopy()
if node.Annotations == nil {
node.Annotations = map[string]string{}
}
node.Annotations[nodeID] = h.id
node.Annotations[nodeAddress] = h.address
return h.nodeController.Update(node)
}
func (h *handler) onRemove(key string, node *v1.Node) (*v1.Node, error) {
address := node.Annotations[nodeAddress]
if address == "" {
return node, nil
}
return node, h.delete(address)
}
func (h *handler) delete(address string) error {
c, err := client.FindLeader(h.ctx, h.nodeStore, h.opts...)
if err != nil {
return err
}
defer c.Close()
members, err := c.Cluster(h.ctx)
if err != nil {
return err
}
for _, member := range members {
if member.Address == address {
logrus.Infof("Removing %s %d from dqlite", member.Address, member.ID)
return c.Remove(h.ctx, member.ID)
}
}
return nil
}
func (h *handler) updateNodeStore() error {
nodes, err := h.nodeController.Cache().List(labels.SelectorFromSet(labels.Set{
master: "true",
}))
if err != nil {
return err
}
var (
nodeInfos []client.NodeInfo
seen = map[string]bool{}
)
for _, node := range nodes {
address, ok := node.Annotations[nodeAddress]
if !ok {
continue
}
nodeIDStr, ok := node.Annotations[nodeID]
if !ok {
continue
}
id, err := strconv.ParseUint(nodeIDStr, 10, 64)
if err != nil {
logrus.Errorf("invalid %s=%s, must be a number: %v", nodeID, nodeIDStr, err)
continue
}
if !seen[address] {
nodeInfos = append(nodeInfos, client.NodeInfo{
ID: id,
Address: address,
})
seen[address] = true
}
}
if len(nodeInfos) == 0 {
return fmt.Errorf("not setting dqlient NodeStore len to 0")
}
return h.nodeStore.Set(h.ctx, nodeInfos)
}

36
pkg/dqlite/dialer/dial.go
View File

@ -1,36 +0,0 @@
package dialer
import (
"context"
"crypto/tls"
"fmt"
"net"
"github.com/canonical/go-dqlite/client"
"github.com/rancher/k3s/pkg/dqlite/pipe"
)
func NewHTTPDialer(advertiseAddress, bindAddress string, tls *tls.Config) (client.DialFunc, error) {
d := &dialer{
advertiseAddress: advertiseAddress,
bindAddress: bindAddress,
tls: tls,
}
return d.dial, nil
}
type dialer struct {
advertiseAddress string
bindAddress string
tls *tls.Config
}
func (d *dialer) dial(ctx context.Context, address string) (net.Conn, error) {
if address == d.advertiseAddress {
return net.Dial("unix", d.bindAddress)
}
url := fmt.Sprintf("https://%s/db/connect", address)
return pipe.ToHTTP(ctx, url, d.tls)
}

124
pkg/dqlite/join.go
View File

@ -1,124 +0,0 @@
package dqlite
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
"github.com/canonical/go-dqlite/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
func (d *DQLite) Test(ctx context.Context) error {
var ips []string
peers, err := d.NodeStore.Get(ctx)
if err != nil {
return err
}
for _, peer := range peers {
ips = append(ips, peer.Address)
}
logrus.Infof("Testing connection to peers %v", ips)
if err := d.Join(ctx, nil); err != nil {
return err
}
logrus.Infof("Connection OK to peers %v", ips)
return nil
}
func nodeIDsEqual(testID, currentID uint64) bool {
// this is a test for a bug in v1.0.0. In future versions we don't
// generate node ID higher than 1<<20 so this doesn't matter. But
// basically just ignore the first 32 bits.
return uint32(testID) == uint32(currentID)
}
func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error {
if len(nodes) > 0 {
if err := d.NodeStore.Set(ctx, nodes); err != nil {
return err
}
}
client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...)
if err != nil {
return err
}
defer client.Close()
current, err := client.Cluster(ctx)
if err != nil {
return err
}
nodeID, err := getClusterID(false, d.DataDir)
if err != nil {
return errors.Wrap(err, "get cluster ID")
}
for _, testNode := range current {
if testNode.Address == d.NodeInfo.Address {
if !nodeIDsEqual(testNode.ID, nodeID) {
if err := d.node.Close(); err != nil {
return errors.Wrap(err, "node close for id reset")
}
if err := writeClusterID(testNode.ID, d.DataDir); err != nil {
return errors.Wrap(err, "restart node to reset ID")
}
logrus.Fatalf("resetting node ID from %d to %d, please restart", nodeID, testNode.ID)
}
return nil
} else if nodeIDsEqual(testNode.ID, nodeID) {
deleteClusterID(d.DataDir)
logrus.Fatalf("node ID %d is in use, please restart", nodeID)
}
}
if found, err := cleanDir(d.DataDir, true); err != nil {
return err
} else if found {
if err := d.node.Close(); err != nil {
return errors.Wrap(err, "node close for cleaning")
}
_, _ = cleanDir(d.DataDir, false)
return fmt.Errorf("cleaned DB directory, now restart and join")
}
logrus.Infof("Joining dqlite cluster as address=%s, id=%d", d.NodeInfo.Address, d.NodeInfo.ID)
return client.Add(ctx, d.NodeInfo)
}
func cleanDir(dataDir string, check bool) (bool, error) {
dbDir := GetDBDir(dataDir)
backupDir := filepath.Join(dbDir, fmt.Sprintf(".backup-%d", time.Now().Unix()))
files, err := ioutil.ReadDir(dbDir)
if err != nil {
return false, errors.Wrap(err, "cleaning dqlite DB dir")
}
for _, file := range files {
if file.IsDir() || strings.HasPrefix(file.Name(), ".") || ignoreFile[file.Name()] {
continue
}
if check {
return true, nil
}
if err := os.MkdirAll(backupDir, 0700); err != nil {
return false, errors.Wrapf(err, "creating backup dir %s", backupDir)
}
oldName := filepath.Join(dbDir, file.Name())
newName := filepath.Join(backupDir, file.Name())
logrus.Infof("Backing up %s => %s", oldName, newName)
if err := os.Rename(oldName, newName); err != nil {
return false, errors.Wrapf(err, "backup %s", oldName)
}
}
return false, nil
}

27
pkg/dqlite/log.go
View File

@ -1,27 +0,0 @@
package dqlite
import (
"strings"
"github.com/canonical/go-dqlite/client"
"github.com/sirupsen/logrus"
)
func log() client.LogFunc {
return func(level client.LogLevel, s string, i ...interface{}) {
switch level {
case client.LogDebug:
logrus.Debugf(s, i...)
case client.LogError:
logrus.Errorf(s, i...)
case client.LogInfo:
if strings.HasPrefix(s, "connected") {
logrus.Debugf(s, i...)
} else {
logrus.Infof(s, i...)
}
case client.LogWarn:
logrus.Warnf(s, i...)
}
}
}

56
pkg/dqlite/pipe/http.go
View File

@ -1,56 +0,0 @@
package pipe
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"github.com/pkg/errors"
)
func ToHTTP(ctx context.Context, url string, tlsConfig *tls.Config) (net.Conn, error) {
request, err := http.NewRequest(http.MethodPost, url, nil)
if err != nil {
return nil, err
}
request = request.WithContext(ctx)
netDial := &net.Dialer{}
if deadline, ok := ctx.Deadline(); ok {
netDial.Deadline = deadline
}
conn, err := tls.DialWithDialer(netDial, "tcp", request.URL.Host, tlsConfig)
if err != nil {
return nil, errors.Wrap(err, "tls dial")
}
err = request.Write(conn)
if err != nil {
return nil, errors.Wrap(err, "request write")
}
response, err := http.ReadResponse(bufio.NewReader(conn), request)
if err != nil {
return nil, errors.Wrap(err, "read request")
}
if response.StatusCode != http.StatusSwitchingProtocols {
return nil, fmt.Errorf("expected 101 response, got: %d", response.StatusCode)
}
listener, err := net.Listen("unix", "")
if err != nil {
return nil, errors.Wrap(err, "Failed to create unix listener")
}
defer listener.Close()
if err := Unix(conn, listener.Addr().String()); err != nil {
return nil, err
}
return listener.Accept()
}

46
pkg/dqlite/pipe/pipe.go
View File

@ -1,46 +0,0 @@
package pipe
import (
"io"
"net"
"github.com/lxc/lxd/shared/eagain"
"github.com/sirupsen/logrus"
)
func UnixPiper(srcs <-chan net.Conn, bindAddress string) {
for src := range srcs {
go Unix(src, bindAddress)
}
}
func Unix(src net.Conn, target string) error {
dst, err := net.Dial("unix", target)
if err != nil {
src.Close()
return err
}
Connect(src, dst)
return nil
}
func Connect(src net.Conn, dst net.Conn) {
go func() {
_, err := io.Copy(eagain.Writer{Writer: dst}, eagain.Reader{Reader: src})
if err != nil && err != io.EOF {
logrus.Debugf("copy pipe src->dst closed: %v", err)
}
src.Close()
dst.Close()
}()
go func() {
_, err := io.Copy(eagain.Writer{Writer: src}, eagain.Reader{Reader: dst})
if err != nil {
logrus.Debugf("copy pipe dst->src closed: %v", err)
}
src.Close()
dst.Close()
}()
}

53
pkg/dqlite/proxy.go
View File

@ -1,53 +0,0 @@
package dqlite
import (
"context"
"net"
"net/http"
"github.com/pkg/errors"
"github.com/rancher/k3s/pkg/dqlite/pipe"
)
var (
upgradeResponse = []byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: dqlite\r\n\r\n")
)
type proxy struct {
conns chan net.Conn
}
func newProxy(ctx context.Context, bindAddress string) http.Handler {
p := &proxy{
conns: make(chan net.Conn, 100),
}
go func() {
<-ctx.Done()
close(p.conns)
}()
go pipe.UnixPiper(p.conns, bindAddress)
return p
}
func (h *proxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
hijacker, ok := rw.(http.Hijacker)
if !ok {
http.Error(rw, "failed to hijack", http.StatusInternalServerError)
return
}
conn, _, err := hijacker.Hijack()
if err != nil {
err := errors.Wrap(err, "Hijack connection")
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
if n, err := conn.Write(upgradeResponse); err != nil || n != len(upgradeResponse) {
conn.Close()
return
}
h.conns <- conn
}

15
pkg/dqlite/reset.go
View File

@ -1,15 +0,0 @@
package dqlite
import (
"context"
"github.com/canonical/go-dqlite/client"
"github.com/sirupsen/logrus"
)
func (d *DQLite) Reset(ctx context.Context) error {
logrus.Infof("Resetting cluster to single master")
return d.node.Recover([]client.NodeInfo{
d.NodeInfo,
})
}

80
pkg/dqlite/router.go
View File

@ -1,80 +0,0 @@
package dqlite
import (
"context"
"crypto/x509"
"encoding/json"
"net/http"
"github.com/canonical/go-dqlite"
"github.com/canonical/go-dqlite/client"
"github.com/gorilla/mux"
)
func router(ctx context.Context, next http.Handler, nodeInfo dqlite.NodeInfo, clientCA *x509.Certificate, clientCN string, bindAddress string) http.Handler {
mux := mux.NewRouter()
mux.Handle("/db/connect", newChecker(newProxy(ctx, bindAddress), clientCA, clientCN))
mux.Handle("/db/info", infoHandler(ctx, nodeInfo, bindAddress))
mux.NotFoundHandler = next
return mux
}
func infoHandler(ctx context.Context, nodeInfo dqlite.NodeInfo, bindAddress string) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
client, err := client.New(ctx, bindAddress, client.WithLogFunc(log()))
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
defer client.Close()
info, err := client.Cluster(ctx)
if err != nil {
http.Error(rw, err.Error(), http.StatusInternalServerError)
return
}
rw.Header().Set("Content-Type", "application/json")
json.NewEncoder(rw).Encode(info)
})
}
type checker struct {
next http.Handler
verify x509.VerifyOptions
cn string
}
func newChecker(next http.Handler, ca *x509.Certificate, cn string) http.Handler {
pool := x509.NewCertPool()
pool.AddCert(ca)
return &checker{
next: next,
verify: x509.VerifyOptions{
Roots: pool,
KeyUsages: []x509.ExtKeyUsage{
x509.ExtKeyUsageClientAuth,
},
DNSName: cn,
},
cn: cn,
}
}
func (c *checker) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
if !c.check(req) {
http.Error(rw, "unauthorized", http.StatusUnauthorized)
return
}
c.next.ServeHTTP(rw, req)
}
func (c *checker) check(r *http.Request) bool {
for _, cert := range r.TLS.PeerCertificates {
_, err := cert.Verify(c.verify)
if err == nil {
return cert.Subject.CommonName == c.cn
}
}
return false
}

245
pkg/dqlite/server.go
View File

@ -1,245 +0,0 @@
package dqlite
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"math/rand"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/canonical/go-dqlite"
"github.com/canonical/go-dqlite/client"
"github.com/pkg/errors"
controllerclient "github.com/rancher/k3s/pkg/dqlite/controller/client"
"github.com/rancher/k3s/pkg/dqlite/dialer"
dqlitedriver "github.com/rancher/kine/pkg/drivers/dqlite"
v1 "github.com/rancher/wrangler-api/pkg/generated/controllers/core/v1"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/util/net"
)
const (
PeersFile = "peers.db"
NodeIDFile = "node-id"
)
var (
ignoreFile = map[string]bool{
PeersFile: true,
NodeIDFile: true,
}
)
type Certs struct {
ServerTrust *x509.Certificate
ClientTrust *x509.Certificate
ClientCert tls.Certificate
}
type DQLite struct {
ClientCA string
ClientCAKey string
ClientCert string
ClientCertKey string
ServerCA string
ServerCAKey string
AdvertiseIP string
AdvertisePort int
DataDir string
NodeStore client.NodeStore
NodeInfo client.NodeInfo
node *dqlite.Node
StorageEndpoint string
NodeControllerGetter NodeControllerGetter
clientOpts []client.Option
}
type NodeControllerGetter func() v1.NodeController
func New(dataDir, advertiseIP string, advertisePort int, getter NodeControllerGetter) *DQLite {
return &DQLite{
AdvertiseIP: advertiseIP,
AdvertisePort: advertisePort,
DataDir: dataDir,
NodeControllerGetter: getter,
}
}
func (d *DQLite) Start(ctx context.Context, initCluster, resetCluster bool, certs *Certs, next http.Handler) (http.Handler, error) {
bindAddress := d.getBindAddress()
clientTLSConfig, err := getClientTLSConfig(certs.ClientCert, certs.ServerTrust)
if err != nil {
return nil, err
}
advertise, err := getAdvertiseAddress(d.AdvertiseIP, d.AdvertisePort)
if err != nil {
return nil, errors.Wrap(err, "get advertise address")
}
dial, err := getDialer(advertise, bindAddress, clientTLSConfig)
if err != nil {
return nil, err
}
dqlitedriver.Dialer = dial
dqlitedriver.Logger = log()
d.clientOpts = append(d.clientOpts, client.WithDialFunc(dial), client.WithLogFunc(log()))
nodeInfo, node, err := getNode(d.DataDir, advertise, bindAddress, initCluster, dial)
if err != nil {
return nil, err
}
d.NodeInfo = nodeInfo
d.node = node
go func() {
<-ctx.Done()
node.Close()
}()
if err := d.nodeStore(ctx, initCluster); err != nil {
return nil, err
}
go d.startController(ctx)
if !resetCluster {
if err := node.Start(); err != nil {
return nil, err
}
}
return router(ctx, next, nodeInfo, certs.ClientTrust, "kube-apiserver", bindAddress), nil
}
func (d *DQLite) startController(ctx context.Context) {
for {
if nc := d.NodeControllerGetter(); nc != nil {
if os.Getenv("NODE_NAME") == "" {
logrus.Errorf("--disable-agent is not compatible with dqlite")
} else {
break
}
}
time.Sleep(time.Second)
}
controllerclient.Register(ctx, os.Getenv("NODE_NAME"), d.NodeInfo, d.NodeStore, d.NodeControllerGetter(), d.clientOpts)
}
func (d *DQLite) nodeStore(ctx context.Context, initCluster bool) error {
peerDB := filepath.Join(GetDBDir(d.DataDir), PeersFile)
ns, err := client.DefaultNodeStore(peerDB)
if err != nil {
return err
}
d.NodeStore = ns
d.StorageEndpoint = fmt.Sprintf("dqlite://?peer-file=%s", peerDB)
if initCluster {
if err := dqlitedriver.AddPeers(ctx, d.NodeStore, d.NodeInfo); err != nil {
return err
}
}
return nil
}
func getAdvertiseAddress(advertiseIP string, advertisePort int) (string, error) {
ip := advertiseIP
if ip == "" {
ipAddr, err := net.ChooseHostInterface()
if err != nil {
return "", err
}
ip = ipAddr.String()
}
return fmt.Sprintf("%s:%d", ip, advertisePort), nil
}
func getClientTLSConfig(cert tls.Certificate, ca *x509.Certificate) (*tls.Config, error) {
tlsConfig := &tls.Config{
RootCAs: x509.NewCertPool(),
Certificates: []tls.Certificate{
cert,
},
ServerName: "kubernetes",
}
tlsConfig.RootCAs.AddCert(ca)
return tlsConfig, nil
}
func getDialer(advertiseAddress, bindAddress string, tlsConfig *tls.Config) (client.DialFunc, error) {
return dialer.NewHTTPDialer(advertiseAddress, bindAddress, tlsConfig)
}
func GetDBDir(dataDir string) string {
return filepath.Join(dataDir, "db", "state.dqlite")
}
func getNode(dataDir string, advertiseAddress, bindAddress string, initCluster bool, dial client.DialFunc) (dqlite.NodeInfo, *dqlite.Node, error) {
id, err := getClusterID(initCluster, dataDir)
if err != nil {
return dqlite.NodeInfo{}, nil, errors.Wrap(err, "reading cluster id")
}
dbDir := GetDBDir(dataDir)
node, err := dqlite.New(id, advertiseAddress, dbDir,
dqlite.WithBindAddress(bindAddress),
dqlite.WithDialFunc(dial),
dqlite.WithNetworkLatency(20*time.Millisecond))
return dqlite.NodeInfo{
ID: id,
Address: advertiseAddress,
}, node, err
}
func writeClusterID(id uint64, dataDir string) error {
idFile := filepath.Join(GetDBDir(dataDir), NodeIDFile)
if err := os.MkdirAll(filepath.Dir(idFile), 0700); err != nil {
return err
}
return ioutil.WriteFile(idFile, []byte(strconv.FormatUint(id, 10)), 0644)
}
func deleteClusterID(dataDir string) {
os.Remove(filepath.Join(GetDBDir(dataDir), NodeIDFile))
}
func getClusterID(initCluster bool, dataDir string) (uint64, error) {
idFile := filepath.Join(GetDBDir(dataDir), NodeIDFile)
content, err := ioutil.ReadFile(idFile)
if os.IsNotExist(err) {
content = nil
} else if err != nil {
return 0, err
}
idStr := strings.TrimSpace(string(content))
if idStr == "" {
id := uint64(rand.Intn(1 << 20))
if initCluster {
id = 1
}
return id, writeClusterID(id, dataDir)
}
return strconv.ParseUint(idStr, 10, 64)
}
func (d *DQLite) getBindAddress() string {
// only anonymous works???
return "@" + filepath.Join(GetDBDir(d.DataDir), "dqlite.sock")
}

9
scripts/build
View File

@ -39,15 +39,10 @@ STATIC="
-extldflags '-static'
"
if [ "$DQLITE" = "true" ]; then
DQLITE_TAGS="dqlite"
DQLITE_STATIC_SQLITE="-luv -lraft -lco"
fi
STATIC_SQLITE="
-extldflags '-static -lm -ldl -lz -lpthread $DQLITE_STATIC_SQLITE'
-extldflags '-static -lm -ldl -lz -lpthread'
"
TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless $DQLITE_TAGS"
TAGS="ctrd apparmor seccomp no_btrfs netcgo osusergo providerless"
RUNC_TAGS="apparmor seccomp"
RUNC_STATIC="static"

5
scripts/package-cli
View File

@ -56,9 +56,6 @@ LDFLAGS="
-w -s
"
STATIC="-extldflags '-static'"
if [ "$DQLITE" = 'true' ]; then
DQLITE_TAGS='dqlite'
fi
CGO_ENABLED=0 "${GO}" build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
CGO_ENABLED=0 "${GO}" build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
./scripts/build-upload ${CMD_NAME} ${COMMIT}

4
scripts/provision/generic/alpine310/vagrant
View File

@ -3,11 +3,9 @@ set -ve
apk add -q -f curl libc6-compat tzdata
download_go
download_dqlite
# ---
cat <<EOF >/etc/profile.d/build.sh
export SELINUX=true
export DQLITE=true
export STATIC_BUILD=true
EOF
. /etc/profile.d/build.sh
@ -28,4 +26,4 @@ EOF
else
echo "Using host docker server v$(cat /tmp/docker-server-version)"
fi
)
)

1
scripts/provision/generic/centos7/vagrant
View File

@ -5,7 +5,6 @@ download_go
# ---
cat <<EOF >/etc/profile.d/build.sh
export SELINUX=true
# export DQLITE=true
# export STATIC_BUILD=true
EOF
. /etc/profile.d/build.sh

1
scripts/provision/generic/ubuntu1804/vagrant
View File

@ -5,7 +5,6 @@ download_go
# ---
cat <<EOF >/etc/profile.d/build.sh
export SELINUX=true
# export DQLITE=true
# export STATIC_BUILD=true
EOF
. /etc/profile.d/build.sh

12
scripts/provision/vagrant
View File

@ -79,18 +79,6 @@ download_go() {
curl -sL https://storage.googleapis.com/golang/go${goversion}.linux-${ARCH}.tar.gz | tar -xzf - -C /usr/local
}
# --- Utility function to download dqlite
download_dqlite() {
dqliteURL="https://github.com/$(grep dqlite-build Dockerfile.dapper | sed -e 's/^.*--from=\([^ ]*\).*$/\1/' -e 's|:|/releases/download/|')/dqlite-$ARCH.tgz"
if [ -z "$dqliteURL" ]; then
echo 'Cannot find dqlite URL to fetch'
return 1
fi
mkdir -p /usr/src/
echo "Downloading DQLITE from $dqliteURL"
curl -sL $dqliteURL -o /usr/src/dqlite.tgz
}
# --- Run vagrant provision script if available
if [ ! -f "${PROVISION}" ]; then
echo "WARNING: Unable to execute provision script \"${PROVISION}\""

1
scripts/test
View File

@ -27,4 +27,3 @@ E2E_OUTPUT=$artifacts test-run-sonobuoy
test-run-sonobuoy mysql
test-run-sonobuoy postgres
# test-run-sonobuoy dqlite