diff --git a/.drone.yml b/.drone.yml index 9ae8d11813..c3959e42e1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -11,7 +11,7 @@ pipeline: stage-binaries: image: rancher/dapper:1.11.2 commands: - - cp -f ./bin/rio-incluster ./package/rio + - cp -f ./bin/k3s-incluster ./package/k3s when: branch: master event: tag @@ -19,7 +19,7 @@ pipeline: publish-image: image: plugins/docker dockerfile: package/Dockerfile - repo: rancher/rio + repo: rancher/k3s context: package/ tag: ${DRONE_TAG} secrets: [docker_username, docker_password] diff --git a/.gitignore b/.gitignore index 2f485433e5..e541bee4ca 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,8 @@ /image/agent /image/go_build_agent /image/main.squashfs -/package/rio +/package/k3s +/pkg/data/zz_generated_bindata.go __pycache__ /tests/.pytest_cache/ /tests/.tox/ diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 4d92357872..2d3422c67c 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -1,9 +1,8 @@ FROM golang:1.11-alpine3.8 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \ - python3 py3-pip python3-dev openssl-dev libffi-dev + python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make RUN pip3 install 'tox==3.6.0' -RUN npm install -g 'bats@1.1.0' RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static RUN go get -d golang.org/x/lint/golint && \ git -C /go/src/golang.org/x/lint/golint checkout -b current 06c8688daad7faa9da5a0c2f163a3d14aac986ca && \ @@ -16,12 +15,13 @@ RUN go get -d github.com/alecthomas/gometalinter && \ rm -rf /go/src /go/pkg ENV DAPPER_RUN_ARGS --privileged -ENV DAPPER_ENV REPO TAG DRONE_TAG -ENV DAPPER_SOURCE /go/src/github.com/rancher/rio/ +ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME +ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/ ENV DAPPER_OUTPUT ./bin ./dist ENV DAPPER_DOCKER_SOCKET true ENV HOME ${DAPPER_SOURCE} ENV CROSS true +ENV STATIC_BUILD true WORKDIR ${DAPPER_SOURCE} ENTRYPOINT ["./scripts/entry"] diff --git a/agent/main.go b/agent/main.go deleted file mode 100644 index fbfdbf60a1..0000000000 --- a/agent/main.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "context" - - "github.com/rancher/norman/signal" - "github.com/rancher/rio/agent/config" - "github.com/rancher/rio/agent/containerd" - "github.com/rancher/rio/agent/flannel" - "github.com/rancher/rio/agent/proxy" - "github.com/rancher/rio/agent/syssetup" - "github.com/rancher/rio/agent/tunnel" - "github.com/rancher/rio/pkg/daemons/agent" - "github.com/sirupsen/logrus" -) - -func main() { - if err := run(); err != nil { - logrus.Fatal(err) - } -} - -func run() error { - ctx := signal.SigTermCancelContext(context.Background()) - - nodeConfig := config.Get() - - if nodeConfig.Docker { - nodeConfig.AgentConfig.RuntimeSocket = "" - } else { - containerd.Run(ctx, nodeConfig) - } - - if err := syssetup.Configure(); err != nil { - return err - } - - if err := tunnel.Setup(nodeConfig); err != nil { - return err - } - - if err := proxy.Run(nodeConfig); err != nil { - return err - } - - if err := agent.Agent(&nodeConfig.AgentConfig); err != nil { - return err - } - - if !nodeConfig.NoFlannel { - if err := flannel.Run(ctx, nodeConfig); err != nil { - return err - } - } - - <-ctx.Done() - return ctx.Err() -} diff --git a/cli/cmd/agent/agent.go b/cli/cmd/agent/agent.go deleted file mode 100644 index 6ba2cf522e..0000000000 --- a/cli/cmd/agent/agent.go +++ /dev/null @@ -1,19 +0,0 @@ -package agent - -import "github.com/urfave/cli" - -type Agent struct { - T_Token string `desc:"Token to use for authentication" env:"K3S_TOKEN"` - S_Server string `desc:"Server to connect to" env:"K3S_URL"` - D_DataDir string `desc:"Folder to hold state" default:"/var/lib/rancher/k3s"` - L_Log string `desc:"log to file"` - AgentShared -} - -type AgentShared struct { - I_NodeIp string `desc:"IP address to advertise for node"` -} - -func (a *Agent) Customize(command *cli.Command) { - command.Category = "CLUSTER RUNTIME" -} diff --git a/cli/cmd/agent/agent_k3s.go b/cli/cmd/agent/agent_k3s.go deleted file mode 100644 index 12ae4f54ff..0000000000 --- a/cli/cmd/agent/agent_k3s.go +++ /dev/null @@ -1,81 +0,0 @@ -// +build k8s - -package agent - -import ( - "fmt" - "io" - "os" - "path/filepath" - "time" - - "github.com/natefinch/lumberjack" - "github.com/rancher/norman/pkg/clientaccess" - "github.com/rancher/norman/pkg/resolvehome" - "github.com/rancher/rio/pkg/enterchroot" - "github.com/sirupsen/logrus" - "github.com/urfave/cli" -) - -func (a *Agent) Run(ctx *cli.Context) error { - if os.Getuid() != 0 { - return fmt.Errorf("agent must be ran as root") - } - - if len(a.T_Token) == 0 { - return fmt.Errorf("--token is required") - } - - if len(a.S_Server) == 0 { - return fmt.Errorf("--server is required") - } - - dataDir, err := resolvehome.Resolve(a.D_DataDir) - if err != nil { - return err - } - - return RunAgent(a.S_Server, a.T_Token, dataDir, a.L_Log, a.I_NodeIp) -} - -func RunAgent(server, token, dataDir, logFile, ipAddress string) error { - dataDir = filepath.Join(dataDir, "agent") - - for { - tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", server, token) - if err != nil { - logrus.Error(err) - time.Sleep(2 * time.Second) - continue - } - os.Remove(tmpFile) - break - } - - os.Setenv("K3S_URL", server) - os.Setenv("K3S_TOKEN", token) - os.Setenv("K3S_DATA_DIR", dataDir) - os.Setenv("K3S_NODE_IP", ipAddress) - - os.MkdirAll(dataDir, 0700) - - stdout := io.Writer(os.Stdout) - stderr := io.Writer(os.Stderr) - - if logFile == "" { - stdout = os.Stdout - stderr = os.Stderr - } else { - l := &lumberjack.Logger{ - Filename: logFile, - MaxSize: 50, - MaxBackups: 3, - MaxAge: 28, - Compress: true, - } - stdout = l - stderr = l - } - - return enterchroot.Mount(filepath.Join(dataDir, "root"), stdout, stderr, os.Args[1:]) -} diff --git a/cli/cmd/agent/agent_none.go b/cli/cmd/agent/agent_none.go deleted file mode 100644 index 8465ab302e..0000000000 --- a/cli/cmd/agent/agent_none.go +++ /dev/null @@ -1,17 +0,0 @@ -// +build !k8s - -package agent - -import ( - "fmt" - - "github.com/urfave/cli" -) - -func (a *Agent) Run(ctx *cli.Context) error { - return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"") -} - -func RunAgent(server, token, dataDir, logFile string) error { - return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"") -} diff --git a/cli/cmd/kubectl/kubectl.go b/cli/cmd/kubectl/kubectl.go deleted file mode 100644 index 2067bb6125..0000000000 --- a/cli/cmd/kubectl/kubectl.go +++ /dev/null @@ -1,24 +0,0 @@ -package kubectl - -import ( - "os" - - "github.com/rancher/rio/pkg/kubectl" - "github.com/urfave/cli" -) - -func NewKubectlCommand() cli.Command { - return cli.Command{ - Name: "kubectl", - Usage: "Run kubectl", - SkipFlagParsing: true, - SkipArgReorder: true, - Action: run, - } -} - -func run(ctx *cli.Context) error { - os.Args = append([]string{"kubectl"}, ctx.Args()...) - kubectl.Main() - return nil -} diff --git a/cli/cmd/server/server.go b/cli/cmd/server/server.go deleted file mode 100644 index 3532d46574..0000000000 --- a/cli/cmd/server/server.go +++ /dev/null @@ -1,128 +0,0 @@ -package server - -import ( - "context" - "flag" - "fmt" - "os" - "path/filepath" - - "github.com/docker/docker/pkg/reexec" - "github.com/natefinch/lumberjack" - "github.com/rancher/norman/signal" - "github.com/rancher/rio/pkg/server" - "github.com/sirupsen/logrus" - "github.com/urfave/cli" - "k8s.io/apimachinery/pkg/util/net" -) - -var ( - appName = filepath.Base(os.Args[0]) - - config server.Config - log string -) - -var ServerCommand = cli.Command{ - Name: "server", - Usage: "Run management server", - UsageText: appName + " server [OPTIONS]", - Action: Run, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "https-listen-port", - Usage: "HTTPS listen port", - Value: 6443, - Destination: &config.TLSConfig.HTTPSPort, - }, - cli.IntFlag{ - Name: "http-listen-port", - Usage: "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)", - Value: 0, - Destination: &config.TLSConfig.HTTPPort, - }, - cli.StringFlag{ - Name: "data-dir", - Usage: "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root", - Destination: &config.ControlConfig.DataDir, - }, - //cli.StringFlag{ - // Name: "advertise-address", - // Usage: "Address of the server to put in the generated kubeconfig", - // Destination: &config.AdvertiseIP, - //}, - cli.BoolFlag{ - Name: "disable-agent", - Usage: "Do not run a local agent and register a local kubelet", - Destination: &config.DisableAgent, - }, - cli.StringFlag{ - Name: "log", - Usage: "Log to file", - Destination: &log, - }, - }, -} - -func setupLogging(app *cli.Context) { - if !app.GlobalBool("debug") { - flag.Set("stderrthreshold", "3") - flag.Set("alsologtostderr", "false") - flag.Set("logtostderr", "false") - } -} - -func runWithLogging(app *cli.Context) error { - l := &lumberjack.Logger{ - Filename: log, - MaxSize: 50, - MaxBackups: 3, - MaxAge: 28, - Compress: true, - } - - args := append([]string{"k3s"}, os.Args[1:]...) - cmd := reexec.Command(args...) - cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "_RIO_REEXEC_=true") - cmd.Stderr = l - cmd.Stdout = l - cmd.Stdin = os.Stdin - return cmd.Run() -} - -func Run(app *cli.Context) error { - if log != "" && os.Getenv("_RIO_REEXEC_") == "" { - return runWithLogging(app) - } - - setupLogging(app) - - if !config.DisableAgent && os.Getuid() != 0 { - return fmt.Errorf("must run as root unless --disable-agent is specified") - } - - if config.ControlConfig.NodeConfig.AgentConfig.NodeIP == "" { - ip, err := net.ChooseHostInterface() - if err == nil { - config.ControlConfig.NodeConfig.AgentConfig.NodeIP = ip.String() - } - } - - logrus.Info("Starting k3s ", app.App.Version) - ctx := signal.SigTermCancelContext(context.Background()) - if err := server.StartServer(ctx, &config); err != nil { - return err - } - - if config.DisableAgent { - <-ctx.Done() - return nil - } - - return nil - //logFile := filepath.Join(serverConfig.DataDir, "agent/agent.log") - //url := fmt.Sprintf("https://localhost:%d", httpsListenPort) - //logrus.Infof("Agent starting, logging to %s", logFile) - //return agent.RunAgent(url, server2.FormatToken(serverConfig.Runtime.NodeToken), serverConfig.DataDir, logFile, "") -} diff --git a/cli/pkg/builder/builder.go b/cli/pkg/builder/builder.go deleted file mode 100644 index fcce557f2e..0000000000 --- a/cli/pkg/builder/builder.go +++ /dev/null @@ -1,151 +0,0 @@ -package builder - -import ( - "fmt" - "reflect" - "regexp" - "strconv" - "strings" - "unsafe" - - "github.com/urfave/cli" -) - -var ( - caseRegexp = regexp.MustCompile("([a-z])([A-Z])") -) - -type clirunnable interface { - Run(app *cli.Context) error -} - -type customizer interface { - Customize(cmd *cli.Command) -} - -type fieldInfo struct { - FieldType reflect.StructField - FieldValue reflect.Value -} - -func fields(obj interface{}) []fieldInfo { - ptrValue := reflect.ValueOf(obj) - objValue := ptrValue.Elem() - - var result []fieldInfo - - for i := 0; i < objValue.NumField(); i++ { - fieldType := objValue.Type().Field(i) - if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct { - result = append(result, fields(objValue.Field(i).Addr().Interface())...) - } else if !fieldType.Anonymous { - result = append(result, fieldInfo{ - FieldValue: objValue.Field(i), - FieldType: objValue.Type().Field(i), - }) - } - } - - return result -} - -func Command(obj interface{}, usage, usageText, description string) cli.Command { - slices := map[string]reflect.Value{} - maps := map[string]reflect.Value{} - ptrValue := reflect.ValueOf(obj) - objValue := ptrValue.Elem() - - c := cli.Command{ - Name: strings.ToLower(strings.Replace(objValue.Type().Name(), "Command", "", 1)), - Usage: usage, - UsageText: usageText, - Description: description, - UseShortOptionHandling: true, - SkipArgReorder: true, - } - - for _, info := range fields(obj) { - defMessage := "" - fieldType := info.FieldType - v := info.FieldValue - - switch fieldType.Type.Kind() { - case reflect.Int: - flag := cli.IntFlag{ - Name: name(fieldType.Name), - Usage: fieldType.Tag.Get("desc"), - EnvVar: fieldType.Tag.Get("env"), - Destination: (*int)(unsafe.Pointer(v.Addr().Pointer())), - } - defValue := fieldType.Tag.Get("default") - if defValue != "" { - n, err := strconv.Atoi(defValue) - if err != nil { - panic("bad default " + defValue + " on field " + fieldType.Name) - } - flag.Value = n - } - c.Flags = append(c.Flags, flag) - case reflect.String: - flag := cli.StringFlag{ - Name: name(fieldType.Name), - Usage: fieldType.Tag.Get("desc"), - Value: fieldType.Tag.Get("default"), - EnvVar: fieldType.Tag.Get("env"), - Destination: (*string)(unsafe.Pointer(v.Addr().Pointer())), - } - c.Flags = append(c.Flags, flag) - case reflect.Slice: - slices[name(fieldType.Name)] = v - defMessage = " " - fallthrough - case reflect.Map: - if defMessage == "" { - maps[name(fieldType.Name)] = v - defMessage = " " - } - flag := cli.StringSliceFlag{ - Name: name(fieldType.Name), - Usage: fieldType.Tag.Get("desc") + defMessage, - EnvVar: fieldType.Tag.Get("env"), - Value: &cli.StringSlice{}, - } - c.Flags = append(c.Flags, flag) - case reflect.Bool: - flag := cli.BoolFlag{ - Name: name(fieldType.Name), - Usage: fieldType.Tag.Get("desc"), - EnvVar: fieldType.Tag.Get("env"), - Destination: (*bool)(unsafe.Pointer(v.Addr().Pointer())), - } - c.Flags = append(c.Flags, flag) - default: - panic("Unknown kind on field " + fieldType.Name + " on " + objValue.Type().Name()) - } - } - - if run, ok := obj.(clirunnable); ok { - c.Action = run.Run - } else { - panic(fmt.Sprintf("failed to find Action function for %T", obj)) - } - - cust, ok := obj.(customizer) - if ok { - cust.Customize(&c) - } - - return c -} - -func name(name string) string { - parts := strings.Split(name, "_") - i := len(parts) - 1 - name = caseRegexp.ReplaceAllString(parts[i], "$1-$2") - name = strings.ToLower(name) - result := append([]string{name}, parts[0:i]...) - for i := 0; i < len(result); i++ { - result[i] = strings.ToLower(result[i]) - } - return strings.Join(result, ",") -} diff --git a/cmd/agent/main.go b/cmd/agent/main.go new file mode 100644 index 0000000000..9d9049bcb1 --- /dev/null +++ b/cmd/agent/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "os" + + "github.com/rancher/k3s/pkg/cli/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +func main() { + app := cmds.NewApp() + app.Commands = []cli.Command{ + cmds.NewAgentCommand(agent.Run), + } + + err := app.Run(os.Args) + if err != nil { + logrus.Fatal(err) + } +} diff --git a/cmd/cni/main.go b/cmd/cni/main.go new file mode 100644 index 0000000000..3b7abd73a6 --- /dev/null +++ b/cmd/cni/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + "path/filepath" + + hostlocal "github.com/containernetworking/plugins/plugins/ipam/host-local" + "github.com/containernetworking/plugins/plugins/main/bridge" + "github.com/containernetworking/plugins/plugins/main/loopback" + "github.com/containernetworking/plugins/plugins/meta/flannel" + "github.com/containernetworking/plugins/plugins/meta/portmap" + "github.com/docker/docker/pkg/reexec" +) + +func main() { + os.Args[0] = filepath.Base(os.Args[0]) + reexec.Register("host-local", hostlocal.Main) + reexec.Register("bridge", bridge.Main) + reexec.Register("flannel", flannel.Main) + reexec.Register("loopback", loopback.Main) + reexec.Register("portmap", portmap.Main) + reexec.Init() +} diff --git a/cmd/containerd/main.go b/cmd/containerd/main.go new file mode 100644 index 0000000000..42e08b1e7e --- /dev/null +++ b/cmd/containerd/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "github.com/rancher/k3s/pkg/containerd" + "k8s.io/klog" +) + +func main() { + klog.InitFlags(nil) + containerd.Main() +} diff --git a/cmd/k3s/main.go b/cmd/k3s/main.go new file mode 100644 index 0000000000..7eb159fc8e --- /dev/null +++ b/cmd/k3s/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/data" + "github.com/rancher/k3s/pkg/datadir" + "github.com/rancher/k3s/pkg/untar" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +func main() { + app := cmds.NewApp() + app.Commands = []cli.Command{ + cmds.NewServerCommand(wrap("k3s-server", os.Args)), + cmds.NewAgentCommand(wrap("k3s-agent", os.Args)), + cmds.NewKubectlCommand(kubectl), + } + + err := app.Run(os.Args) + if err != nil { + logrus.Fatal(err) + } +} + +func kubectl(ctx *cli.Context) error { + return stageAndRun(ctx, "kubectl", append([]string{"kubectl"}, ctx.Args()...)) +} + +func wrap(cmd string, args []string) func(ctx *cli.Context) error { + return func(ctx *cli.Context) error { + return stageAndRun(ctx, cmd, args) + } +} + +func stageAndRun(cli *cli.Context, cmd string, args []string) error { + dataDir, err := datadir.Resolve(cli.String("data-dir")) + if err != nil { + return err + } + + asset, dir := getAssetAndDir(dataDir) + + if err := extract(asset, dir); err != nil { + return errors.Wrap(err, "extracting data") + } + + if err := os.Setenv("PATH", filepath.Join(dir, "bin")+":"+os.Getenv("PATH")); err != nil { + return err + } + + cmd, err = exec.LookPath(cmd) + if err != nil { + return err + } + + return syscall.Exec(cmd, args, os.Environ()) +} + +func getAssetAndDir(dataDir string) (string, string) { + asset := data.AssetNames()[0] + dir := filepath.Join(dataDir, "data", strings.SplitN(filepath.Base(asset), ".", 2)[0]) + return asset, dir +} + +func extract(asset, dir string) error { + logrus.Infof("Asset dir %s", dir) + + if _, err := os.Stat(dir); err == nil { + return nil + } + + logrus.Infof("Staging to dir %s", dir) + + content, err := data.Asset(asset) + if err != nil { + return err + } + buf := bytes.NewBuffer(content) + + tempDest := dir + "-tmp" + defer os.RemoveAll(tempDest) + + os.RemoveAll(tempDest) + + if err := untar.Untar(buf, tempDest); err != nil { + return err + } + + return os.Rename(tempDest, dir) +} diff --git a/cmd/kubectl/main.go b/cmd/kubectl/main.go new file mode 100644 index 0000000000..b85cd68b74 --- /dev/null +++ b/cmd/kubectl/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/rancher/k3s/pkg/kubectl" + +func main() { + kubectl.Main() +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000000..1694e821fe --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "os" + "path/filepath" + + "github.com/docker/docker/pkg/reexec" + "github.com/rancher/k3s/pkg/cli/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/cli/kubectl" + "github.com/rancher/k3s/pkg/cli/server" + "github.com/rancher/k3s/pkg/containerd" + kubectl2 "github.com/rancher/k3s/pkg/kubectl" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +func init() { + reexec.Register("containerd", containerd.Main) + reexec.Register("kubectl", kubectl2.Main) +} + +func main() { + cmd := os.Args[0] + os.Args[0] = filepath.Base(os.Args[0]) + if reexec.Init() { + return + } + os.Args[0] = cmd + + app := cmds.NewApp() + app.Commands = []cli.Command{ + cmds.NewServerCommand(server.Run), + cmds.NewAgentCommand(agent.Run), + cmds.NewKubectlCommand(kubectl.Run), + } + + err := app.Run(os.Args) + if err != nil { + logrus.Fatal(err) + } +} diff --git a/image/Dockerfile b/image/Dockerfile deleted file mode 100644 index dbcd9204ed..0000000000 --- a/image/Dockerfile +++ /dev/null @@ -1,179 +0,0 @@ -### BASE ### -FROM alpine:3.8 as base -RUN apk -U add findutils iptables ipset bash ca-certificates jq iproute2 nfs-utils coreutils libseccomp conntrack-tools -# RUN apk -U add eudev tinyssh e2fsprogs mdadm rsync nfs-utils parted - -FROM base as k3s-build -COPY --from=base /bin /usr/src/image/bin/ -COPY --from=base /lib /usr/src/image/lib/ -COPY --from=base /sbin /usr/src/image/sbin/ -COPY --from=base /etc/ssl/certs/ca-certificates.crt /usr/src/image/etc/ssl/certs/ca-certificates.crt -COPY --from=base /etc/terminfo /usr/src/image/etc/terminfo -COPY --from=base /usr /usr/src/image/usr/ - -WORKDIR /usr/src/image - -RUN rm -rf usr/bin/iconv \ - usr/bin/scanelf \ - usr/bin/ssl_client \ - usr/bin/pkgconf \ - usr/bin/getent \ - usr/bin/locate \ - usr/bin/updatedb \ - usr/bin/c_rehash \ - usr/bin/getconf \ - usr/etc \ - usr/include \ - usr/lib/bash \ - usr/lib/krb5 \ - usr/lib/pkgconfig \ - usr/lib/tc \ - usr/libexec \ - usr/local \ - usr/sbin/nfsiostat \ - usr/sbin/rpc.gssd \ - usr/sbin/nfsidmap \ - usr/sbin/blkmapd \ - usr/sbin/conntrackd \ - usr/sbin/nfct \ - usr/sbin/nfsstat \ - usr/sbin/mountstats \ - usr/sbin/setcap \ - usr/sbin/exportfs \ - usr/sbin/update-ca-certificates \ - usr/sbin/capsh \ - usr/sbin/getcap \ - usr/sbin/rpcdebug \ - usr/sbin/start-statd \ - usr/sbin/getpcaps \ - usr/sbin/sm-notify \ - usr/share/aclocal \ - usr/share/apk \ - usr/share/ca-certificates \ - usr/share/man \ - usr/share/misc && \ - find usr/share/terminfo -type f -exec rm {} \; && \ - ln -s xterm-color usr/share/terminfo/x/xterm-256color && \ - rmdir usr/share/terminfo/* || true -RUN rm -rf bin/sh \ - lib/apk \ - lib/mdev \ - sbin/ss \ - sbin/routel \ - sbin/*-compat* \ - sbin/genl \ - sbin/lnstat \ - sbin/ifstat \ - sbin/mkmntdirs \ - sbin/nfsdcltrack \ - sbin/rtacct \ - sbin/nstat \ - sbin/routef \ - sbin/apk \ - sbin/tc \ - sbin/ifcfg \ - sbin/setup-udev \ - sbin/rtpr \ - sbin/osd_login \ - sbin/bridge \ - sbin/rtmon && \ - ln -s bash bin/sh && \ - mkdir -p lib/modules - -RUN mv sbin/* bin/ && \ - rmdir sbin && \ - ln -s bin sbin - -RUN mkdir lib2 && \ - mv usr/lib/* lib2/ && \ - mv lib2/* lib/ && \ - mv usr/bin/* bin/ && \ - mv usr/sbin/* bin/ && \ - mv usr/share . && \ - rm -rf usr lib2 && \ - for i in $(ls -l bin | grep usr/bin/coreutils | awk '{print $(NF-2)}'); do \ - rm bin/$i && ln -s coreutils bin/$i \ - ;done && \ - find -L bin -type l -exec rm {} \; -print - -RUN apk add upx && \ - upx $(find bin -type f -executable \! -name coreutils) || true - -RUN echo '#### LAYOUT #####' && \ - find -type d && \ - echo '#### BIN #####' && \ - find bin -type f -executable && \ - du -x -s -h - -RUN tar cf ../rootfs.tar * && \ - ls -la ../rootfs.tar - -CMD ["sh"] - -### BUILD IMAGE ### -FROM golang:1.11-alpine AS gobuild -RUN apk -U add git gcc linux-headers musl-dev make libseccomp libseccomp-dev bash -RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh - -### CNI ### -FROM gobuild AS cni -RUN mkdir -p $GOPATH/src/github.com/containernetworking && \ - cd $GOPATH/src/github.com/containernetworking && \ - git clone https://github.com/ibuildthecloud/plugins.git && \ - cd plugins && \ - git checkout 9810b7d5137b171c4e07ce59bb18be9feccec557 -RUN go build -buildmode=pie -ldflags -s -o /usr/bin/cni github.com/containernetworking/plugins - -### RUNC ### -FROM gobuild AS runc -RUN go get -d github.com/opencontainers/runc && \ - git -C $GOPATH/src/github.com/opencontainers/runc checkout -b build v1.0.0-rc5 -WORKDIR $GOPATH/src/github.com/opencontainers/runc -RUN make runc && \ - cp runc /usr/bin/ - -### CONTAINERD ### -FROM gobuild AS containerd -RUN go get -d github.com/containerd/containerd && \ - git -C $GOPATH/src/github.com/containerd/containerd checkout -b build v1.1.4 -WORKDIR $GOPATH/src/github.com/containerd/containerd -RUN sed -i -e '/aufs/d' -e '/zfs/d' cmd/containerd/builtins_linux.go -RUN make BUILDTAGS="apparmor seccomp no_btrfs netgo osusergo" bin/containerd bin/containerd-shim && \ - cp bin/containerd bin/containerd-shim /usr/bin/ - -### AGENT ### -FROM gobuild AS agent -ADD /build/vendor.tar $GOPATH/src/github.com/rancher/rio/ -COPY /agent $GOPATH/src/github.com/rancher/rio/agent -WORKDIR $GOPATH/src/github.com/rancher/rio -RUN go build -buildmode=pie -tags k3s -ldflags -s -o /usr/bin/agent ./agent - -### ASSEMBLE IMAGE ### -FROM gobuild - -RUN apk add -U squashfs-tools - -COPY --from=k3s-build /usr/src/rootfs.tar /usr/src/rootfs.tar -RUN mkdir /usr/src/image && \ - tar xf /usr/src/rootfs.tar -C /usr/src/image - -COPY --from=runc /usr/bin/runc /usr/src/image/bin/runc -RUN strip --strip-unneeded /usr/src/image/bin/runc - -COPY --from=agent /usr/bin/agent /usr/src/image/bin/ -RUN strip --strip-unneeded /usr/src/image/bin/agent - -COPY --from=containerd /usr/bin/containerd-shim /usr/bin/containerd /usr/src/image/bin/ -#COPY containerd /usr/src/image/bin -RUN strip --strip-unneeded /usr/src/image/bin/containerd /usr/src/image/bin/containerd-shim - -RUN mkdir -p /usr/src/image/share/cni/bin -COPY --from=cni /usr/bin/cni /usr/src/image/share/cni/bin -RUN cd /usr/src/image/share/cni/bin && \ - for i in ./bridge ./flannel ./host-local ./loopback ./portmap; do \ - ln -s cni $i \ - ;done - -COPY image/init /usr/src/image/init - -RUN mksquashfs /usr/src/image main.squashfs diff --git a/image/build b/image/build deleted file mode 100755 index df568fb951..0000000000 --- a/image/build +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -cd $(dirname $0) -rm -rf main.squashfs - -pushd .. -mkdir -p build -echo Copying vendor -tar chf build/vendor.tar \ - --exclude vendor/github.com/coreos/etcd/cmd \ - --exclude vendor/github.com/jteeuwen/go-bindata/testdata \ - --exclude vendor/github.com/karrick/godirwalk/testdata \ - vendor -echo Running agent docker build -docker build -t bb -f image/Dockerfile . -popd - -docker run --rm bb cat main.squashfs > main.squashfs diff --git a/image/init b/image/init deleted file mode 100755 index 2877a9cf49..0000000000 --- a/image/init +++ /dev/null @@ -1,147 +0,0 @@ -#!/usr/bin/sh -set -e - -if [ "$ENTER_DEBUG" = true ]; then - set -x -fi - -layout() -{ - mount --make-rshared / - - mkdir -p /proc - mount -t proc -o nodev,nosuid,noexec,relatime none /proc - - for i in cache empty lib local local log opt spool tmp; do - mkdir -p /var/$i - done - - for i in run dev home mnt media opt root lib/modules lib/firmware var/lib/docker; do - if [ -d /.root/$i ]; then - mkdir -p /$i - mount --rbind /.root/$i /$i - fi - done - - if [ -L /.root/var/run ]; then - ln -sf /run /var/run - else - mkdir -p /var/run - mount --rbind /.root/var/run /var/run - fi - - mkdir -p $K3S_DATA_DIR - mount --rbind /.root/$K3S_DATA_DIR /$K3S_DATA_DIR - - mkdir -p /run/k3s/containerd -} - -sysfs() -{ - mkdir -p /sys - mount -t sysfs none /sys - - mount -t securityfs -o noexec,nosuid,nodev none /sys/kernel/security 2>/dev/null|| true - mount -t configfs -o noexec,nosuid,nodev none /sys/kernel/config 2>/dev/null || true - mount -t fusectl -o noexec,nosuid,nodev none /sys/fs/fuse/connections 2>/dev/null || true - mount -t binfmt_misc -o noexec,nosuid,nodev none /proc/sys/fs/binfmt_misc 2>/dev/null || true -} - -cgroups() -{ - mount -t tmpfs -o mode=755,size=10m none /sys/fs/cgroup - cat /proc/cgroups > /tmp/cgroups - - for i in $(seq 0 20); do - t="" - l="$(cat /tmp/cgroups | grep '1$' | awk '{print $1 " " $2}' | grep -w $i | awk '{print $1}')" - for j in $l; do - if [ -z "$t" ]; then - t=$j - else - t="${t},$j" - fi - done - - if [ -z "$t" ]; then - continue - fi - - mkdir -p /sys/fs/cgroup/${t} - mount -t cgroup -o ${t},noexec,nosuid,nodev none /sys/fs/cgroup/${t} - mkdir -p /sys/fs/cgroup/${t}/k3s - - for j in $l; do - if [ $j != $t ]; then - ln -s $t /sys/fs/cgroup/$j - fi - done - done - - # good ole systemd - mkdir -p /sys/fs/cgroup/systemd - mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd - mkdir -p /sys/fs/cgroup/systemd/k3s - - rm /tmp/cgroups -} - -mketc() -{ - mkdir -p /etc - cp -rf usr/etc/* /etc/ - for i in /.root/usr/lib/os-release /.root/etc/os-release; do - if [ -e $i ]; then - cp -f $i /etc/os-release - fi - done - - if [ -e /.root/etc/machine-id ]; then - cp -f /.root/etc/machine-id /etc/machine-id - fi - - hostname > /etc/hostname - - cat > /etc/hosts << EOF -127.0.0.1 localhost $NODE_NAME $(hostname) -::1 localhost ip6-localhost ip6-loopback -ff02::1 ip6-allnodes -ff02::2 ip6-allrouters -EOF - - cat > /etc/resolv.conf << EOF -nameserver 1.1.1.1 -EOF -} - -nodename() -{ - if [ ! -e $K3S_DATA_DIR/id ]; then - echo $RANDOM > $K3S_DATA_DIR/id - fi - export NODE_NAME="$(hostname | cut -f1 -d.)-$(<$K3S_DATA_DIR/id)" -} - -layout -nodename -mketc -sysfs -cgroups - -umount -l .root -rmdir .root - -if [ "$1" = "--" ]; then - shift 1 - exec "$@" -fi - -exec env -i -- \ - HOME=/root \ - NODE_NAME=$NODE_NAME \ - PATH=/sbin:/bin \ - K3S_DATA_DIR=$K3S_DATA_DIR \ - K3S_NODE_IP=$K3S_NODE_IP \ - K3S_TOKEN=$K3S_TOKEN \ - K3S_URL=$K3S_URL \ - agent diff --git a/main.go b/main.go index ea26f18c4a..f423e1fbff 100644 --- a/main.go +++ b/main.go @@ -4,68 +4,25 @@ package main import ( - "fmt" "os" - "path/filepath" - "github.com/docker/docker/pkg/reexec" - "github.com/rancher/rio/cli/cmd/agent" - "github.com/rancher/rio/cli/cmd/kubectl" - "github.com/rancher/rio/cli/cmd/server" - "github.com/rancher/rio/cli/pkg/builder" - "github.com/rancher/rio/version" + "github.com/rancher/k3s/pkg/cli/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/cli/kubectl" + "github.com/rancher/k3s/pkg/cli/server" "github.com/sirupsen/logrus" "github.com/urfave/cli" - - _ "github.com/rancher/rio/pkg/kubectl" -) - -var ( - appName = filepath.Base(os.Args[0]) - debug bool ) func main() { - old := os.Args[0] - os.Args[0] = filepath.Base(os.Args[0]) - if reexec.Init() { - return - } - os.Args[0] = old - - app := cli.NewApp() - app.Name = appName - app.Usage = "Kubernetes, but small and simple" - app.Version = version.Version - cli.VersionPrinter = func(c *cli.Context) { - fmt.Printf("%s version %s\n", app.Name, app.Version) - } - app.Flags = []cli.Flag{ - cli.BoolFlag{ - Name: "debug", - Usage: "Turn on debug logs", - Destination: &debug, - }, - } - + app := cmds.NewApp() app.Commands = []cli.Command{ - server.ServerCommand, - builder.Command(&agent.Agent{}, - "Run node agent", - appName+" agent [OPTIONS]", - ""), - - kubectl.NewKubectlCommand(), - } - app.Before = func(ctx *cli.Context) error { - if debug { - logrus.SetLevel(logrus.DebugLevel) - } - return nil + cmds.NewServerCommand(server.Run), + cmds.NewAgentCommand(agent.Run), + cmds.NewKubectlCommand(kubectl.Run), } - err := app.Run(os.Args) - if err != nil { + if err := app.Run(os.Args); err != nil { logrus.Fatal(err) } } diff --git a/package/Dockerfile b/package/Dockerfile index 2beda04e6f..2a932fbd72 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,5 +1,16 @@ -FROM alpine:3.8 -RUN apk -U --no-cache add ca-certificates +FROM alpine:3.8 as base +RUN apk add -U ca-certificates +ADD data.tar.gz /image +RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \ + cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt +RUN cd image/bin && \ + rm -f k3s && \ + ln -s k3s-server k3s -COPY rio / -WORKDIR /var/lib/rancher/rio +FROM scratch +COPY --from=base /image / +VOLUME /var/lib/rancher/k3s +VOLUME /var/lib/cni +VOLUME /var/log +ENTRYPOINT ["/bin/k3s"] +CMD ["agent"] diff --git a/agent/config/config.go b/pkg/agent/config/config.go similarity index 57% rename from agent/config/config.go rename to pkg/agent/config/config.go index 0708daae8d..4c2b4899bc 100644 --- a/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -1,79 +1,47 @@ package config import ( - "crypto/md5" + "context" "crypto/tls" - "encoding/hex" "encoding/pem" "fmt" "io/ioutil" "net/url" "os" + "os/exec" "path/filepath" - "strings" "time" + "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1" + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/norman/pkg/clientaccess" - "github.com/rancher/rio/pkg/daemons/config" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" - net2 "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/net" "k8s.io/client-go/util/cert" ) -type envInfo struct { - ServerURL string - Token string - DataDir string - NodeIP string - NodeName string -} - -func Get() *config.Node { +func Get(ctx context.Context, agent cmds.Agent) *config.Node { for { - agentConfig, err := get() + agentConfig, err := get(&agent) if err != nil { logrus.Error(err) - time.Sleep(5 * time.Second) - continue + select { + case <-time.After(5 * time.Second): + continue + case <-ctx.Done(): + logrus.Fatalf("Interrupted") + } } return agentConfig } } -func getEnvInfo() (*envInfo, error) { - u := os.Getenv("K3S_URL") - if u == "" { - return nil, fmt.Errorf("K3S_URL env var is required") - } - - _, err := url.Parse(u) - if err != nil { - return nil, fmt.Errorf("K3S_URL [%s] is invalid: %v", u, err) - } - - t := os.Getenv("K3S_TOKEN") - if t == "" { - return nil, fmt.Errorf("K3S_TOKEN env var is required") - } - - dataDir := os.Getenv("K3S_DATA_DIR") - if dataDir == "" { - return nil, fmt.Errorf("K3S_DATA_DIR is required") - } - - return &envInfo{ - ServerURL: u, - Token: t, - DataDir: dataDir, - NodeIP: os.Getenv("K3S_NODE_IP"), - NodeName: os.Getenv("NODE_NAME"), - }, nil -} - func getNodeCert(info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := clientaccess.Get("/v1-k3s/node.cert", info) + nodeCert, err := clientaccess.Get("/v1-k3s/node.crt", info) if err != nil { return nil, err } @@ -104,10 +72,10 @@ func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) { return clientCA, nil } -func getHostnameAndIP(info envInfo) (string, string, error) { +func getHostnameAndIP(info cmds.Agent) (string, string, error) { ip := info.NodeIP if ip == "" { - hostIP, err := net2.ChooseHostInterface() + hostIP, err := net.ChooseHostInterface() if err != nil { return "", "", err } @@ -120,10 +88,7 @@ func getHostnameAndIP(info envInfo) (string, string, error) { if err != nil { return "", "", err } - hostname = strings.Split(hostname, ".")[0] - - d := md5.Sum([]byte(ip)) - name = hostname + "-" + hex.EncodeToString(d[:])[:8] + name = hostname } return name, ip, nil @@ -133,7 +98,7 @@ func localAddress(controlConfig *config.Control) string { return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort) } -func writeKubeConfig(envInfo *envInfo, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) { +func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) { os.MkdirAll(envInfo.DataDir, 0700) kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml") @@ -146,10 +111,9 @@ func writeKubeConfig(envInfo *envInfo, info clientaccess.Info, controlConfig *co return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath) } -func get() (*config.Node, error) { - envInfo, err := getEnvInfo() - if err != nil { - return nil, err +func get(envInfo *cmds.Agent) (*config.Node, error) { + if envInfo.Debug { + logrus.SetLevel(logrus.DebugLevel) } serverURLParsed, err := url.Parse(envInfo.ServerURL) @@ -187,20 +151,45 @@ func get() (*config.Node, error) { return nil, err } - nodeConfig := &controlConfig.NodeConfig + hostLocal, err := exec.LookPath("host-local") + if err != nil { + return nil, errors.Wrapf(err, "failed to find host-local") + } + + nodeConfig := &config.Node{ + Docker: envInfo.Docker, + NoFlannel: envInfo.NoFlannel, + } nodeConfig.LocalAddress = localAddress(controlConfig) - nodeConfig.AgentConfig.NodeIP = defString(nodeConfig.AgentConfig.NodeIP, nodeIP) - nodeConfig.AgentConfig.NodeName = defString(nodeConfig.AgentConfig.NodeName, nodeName) - nodeConfig.AgentConfig.CNIBinDir = defString(nodeConfig.AgentConfig.CNIBinDir, "/usr/share/cni") + nodeConfig.AgentConfig.NodeIP = nodeIP + nodeConfig.AgentConfig.NodeName = nodeName + nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS nodeConfig.AgentConfig.CACertPath = clientCA - nodeConfig.AgentConfig.ListenAddress = defString(nodeConfig.AgentConfig.ListenAddress, "127.0.0.1") + nodeConfig.AgentConfig.ListenAddress = "127.0.0.1" nodeConfig.AgentConfig.KubeConfig = kubeConfig + nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet") nodeConfig.CACerts = info.CACerts + nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml") + nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd") + nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "containerd") + nodeConfig.Containerd.State = "/run/k3s/containerd" + nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock") nodeConfig.ServerAddress = serverURLParsed.Host nodeConfig.Certificate = nodeCert - if !nodeConfig.Docker { - nodeConfig.AgentConfig.RuntimeSocket = "/run/k3s/containerd.sock" + if !nodeConfig.NoFlannel { + nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json") + nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) + nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "etc/cni/net.d") } + if !nodeConfig.Docker { + nodeConfig.AgentConfig.RuntimeSocket = "unix://" + nodeConfig.Containerd.Address + } + if controlConfig.ClusterIPRange != nil { + nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange + } + + os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName) + v1beta1.KubeletSocket = filepath.Join(envInfo.DataDir, "kubelet/device-plugins/kubelet.sock") return nodeConfig, nil } diff --git a/agent/containerd/containerd.go b/pkg/agent/containerd/containerd.go similarity index 52% rename from agent/containerd/containerd.go rename to pkg/agent/containerd/containerd.go index 414216bec4..156eb4fe50 100644 --- a/agent/containerd/containerd.go +++ b/pkg/agent/containerd/containerd.go @@ -9,8 +9,8 @@ import ( "syscall" "time" - "github.com/rancher/rio/agent/config" - util2 "github.com/rancher/rio/agent/util" + util2 "github.com/rancher/k3s/pkg/agent/util" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" "google.golang.org/grpc" runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2" @@ -18,34 +18,50 @@ import ( ) const ( - address = "/run/k3s/containerd.sock" maxMsgSize = 1024 * 1024 * 16 - configToml = `[plugins.cri] + configToml = ` +[plugins.opt] + path = "%OPT%" +[plugins.cri] stream_server_address = "%NODE%" stream_server_port = "10010" +` + configCNIToml = ` [plugins.cri.cni] - bin_dir = "/usr/share/cni/bin" - conf_dir = "/etc/cni/net.d" + bin_dir = "%CNIBIN%" + conf_dir = "%CNICFG%" ` ) -func Run(ctx context.Context, config *config.NodeConfig) error { +func Run(ctx context.Context, cfg *config.Node) error { args := []string{ "containerd", - "-a", address, - "--state", "/run/k3s/containerd", + "-c", cfg.Containerd.Config, + "-a", cfg.Containerd.Address, + "--state", cfg.Containerd.State, + "--root", cfg.Containerd.Root, } - if err := util2.WriteFile("/etc/containerd/config.toml", - strings.Replace(configToml, "%NODE%", config.AgentConfig.NodeName, -1)); err != nil { + template := configToml + if !cfg.NoFlannel { + template += configCNIToml + } + + template = strings.Replace(template, "%OPT%", cfg.Containerd.Opt, -1) + template = strings.Replace(template, "%CNIBIN%", cfg.AgentConfig.CNIBinDir, -1) + template = strings.Replace(template, "%CNICFG%", cfg.AgentConfig.CNIConfDir, -1) + template = strings.Replace(template, "%NODE%", cfg.AgentConfig.NodeName, -1) + + if err := util2.WriteFile(cfg.Containerd.Config, template); err != nil { return err } if logrus.GetLevel() >= logrus.DebugLevel { - args = append(args, "--verbose") + args = append(args, "-l", "debug") } go func() { + logrus.Infof("Running containerd %s", config.ArgString(args[1:])) cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -59,7 +75,7 @@ func Run(ctx context.Context, config *config.NodeConfig) error { }() for { - addr, dailer, err := util.GetAddressAndDialer("unix://" + address) + addr, dailer, err := util.GetAddressAndDialer("unix://" + cfg.Containerd.Address) if err != nil { time.Sleep(1 * time.Second) continue @@ -81,8 +97,12 @@ func Run(ctx context.Context, config *config.NodeConfig) error { break } conn.Close() - logrus.Infof("Waiting for containerd startup") - time.Sleep(1 * time.Second) + logrus.Infof("Waiting for containerd startup: %v", err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } } return nil diff --git a/agent/flannel/flannel.go b/pkg/agent/flannel/flannel.go similarity index 95% rename from agent/flannel/flannel.go rename to pkg/agent/flannel/flannel.go index 3a31dbfea5..60d1bc65f2 100644 --- a/agent/flannel/flannel.go +++ b/pkg/agent/flannel/flannel.go @@ -1,5 +1,5 @@ -// Copyright 2015 flannel authors // +// Copyright 2015 flannel authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -36,13 +36,13 @@ const ( subnetFile = "/run/flannel/subnet.env" ) -func flannel(ctx context.Context, kubeConfigFile string) error { +func flannel(ctx context.Context, flannelConf, kubeConfigFile string) error { extIface, err := LookupExtIface() if err != nil { return err } - sm, err := kube.NewSubnetManager("", kubeConfigFile) + sm, err := kube.NewSubnetManager("", flannelConf, kubeConfigFile, "flannel.alpha.coreos.com") if err != nil { return err } diff --git a/agent/flannel/setup.go b/pkg/agent/flannel/setup.go similarity index 58% rename from agent/flannel/setup.go rename to pkg/agent/flannel/setup.go index c3ffeba6d7..35ff54952c 100644 --- a/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -2,12 +2,12 @@ package flannel import ( "context" + "path/filepath" "strings" "time" - "github.com/rancher/rio/agent/util" - - "github.com/rancher/rio/agent/config" + "github.com/rancher/k3s/pkg/agent/util" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -44,7 +44,15 @@ const ( ` ) -func Run(ctx context.Context, config *config.NodeConfig) error { +func Prepare(ctx context.Context, config *config.Node) error { + if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil { + return err + } + + return createFlannelConf(config) +} + +func Run(ctx context.Context, config *config.Node) error { nodeName := config.AgentConfig.NodeName restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) @@ -70,22 +78,26 @@ func Run(ctx context.Context, config *config.NodeConfig) error { time.Sleep(2 * time.Second) } - if err := createCNIConf(); err != nil { - return err + go func() { + err := flannel(ctx, config.FlannelConf, config.AgentConfig.KubeConfig) + logrus.Fatalf("fannel exited: %v", err) + }() + + return err +} + +func createCNIConf(dir string) error { + if dir == "" { + return nil } + p := filepath.Join(dir, "10-flannel.conflist") + return util.WriteFile(p, cniConf) +} - if err := createFlannelConf(config); err != nil { - return err +func createFlannelConf(config *config.Node) error { + if config.FlannelConf == "" { + return nil } - - return flannel(ctx, config.AgentConfig.KubeConfig) -} - -func createCNIConf() error { - return util.WriteFile("/etc/cni/net.d/10-flannel.conflist", cniConf) -} - -func createFlannelConf(config *config.NodeConfig) error { - return util.WriteFile("/etc/kube-flannel/net-conf.json", - strings.Replace(netJson, "%CIDR", config.AgentConfig.ClusterCIDR.String(), -1)) + return util.WriteFile(config.FlannelConf, + strings.Replace(netJson, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1)) } diff --git a/agent/proxy/proxy.go b/pkg/agent/proxy/proxy.go similarity index 87% rename from agent/proxy/proxy.go rename to pkg/agent/proxy/proxy.go index c26003c468..b7a1b17a31 100644 --- a/agent/proxy/proxy.go +++ b/pkg/agent/proxy/proxy.go @@ -5,12 +5,12 @@ import ( "net/http" "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/norman/pkg/proxy" - "github.com/rancher/rio/agent/config" "github.com/sirupsen/logrus" ) -func Run(config *config.NodeConfig) error { +func Run(config *config.Node) error { proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true) if err != nil { return err diff --git a/pkg/agent/run.go b/pkg/agent/run.go new file mode 100644 index 0000000000..8b38cc8608 --- /dev/null +++ b/pkg/agent/run.go @@ -0,0 +1,85 @@ +package agent + +import ( + "context" + "os" + "path/filepath" + "time" + + "github.com/sirupsen/logrus" + + "github.com/rancher/k3s/pkg/agent/config" + "github.com/rancher/k3s/pkg/agent/containerd" + "github.com/rancher/k3s/pkg/agent/flannel" + "github.com/rancher/k3s/pkg/agent/proxy" + "github.com/rancher/k3s/pkg/agent/syssetup" + "github.com/rancher/k3s/pkg/agent/tunnel" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/daemons/agent" + "github.com/rancher/norman/pkg/clientaccess" +) + +func run(ctx context.Context, cfg cmds.Agent) error { + nodeConfig := config.Get(ctx, cfg) + + if !nodeConfig.NoFlannel { + if err := flannel.Prepare(ctx, nodeConfig); err != nil { + return err + } + } + + if nodeConfig.Docker { + nodeConfig.AgentConfig.RuntimeSocket = "" + } else { + if err := containerd.Run(ctx, nodeConfig); err != nil { + return err + } + } + + if err := syssetup.Configure(); err != nil { + return err + } + + if err := tunnel.Setup(nodeConfig); err != nil { + return err + } + + if err := proxy.Run(nodeConfig); err != nil { + return err + } + + if err := agent.Agent(&nodeConfig.AgentConfig); err != nil { + return err + } + + if !nodeConfig.NoFlannel { + if err := flannel.Run(ctx, nodeConfig); err != nil { + return err + } + } + + <-ctx.Done() + return ctx.Err() +} + +func Run(ctx context.Context, cfg cmds.Agent) error { + cfg.DataDir = filepath.Join(cfg.DataDir, "agent") + + for { + tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token) + if err != nil { + logrus.Error(err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + continue + } + os.Remove(tmpFile) + break + } + + os.MkdirAll(cfg.DataDir, 0700) + return run(ctx, cfg) +} diff --git a/agent/syssetup/setup.go b/pkg/agent/syssetup/setup.go similarity index 100% rename from agent/syssetup/setup.go rename to pkg/agent/syssetup/setup.go diff --git a/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go similarity index 90% rename from agent/tunnel/tunnel.go rename to pkg/agent/tunnel/tunnel.go index b9cf674513..2faf32af97 100644 --- a/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -12,8 +12,8 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/norman/pkg/remotedialer" - "github.com/rancher/rio/agent/config" "github.com/sirupsen/logrus" "k8s.io/client-go/tools/clientcmd" ) @@ -25,7 +25,7 @@ var ( } ) -func Setup(config *config.NodeConfig) error { +func Setup(config *config.Node) error { restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) if err != nil { return err @@ -36,7 +36,7 @@ func Setup(config *config.NodeConfig) error { return err } - wsURL := fmt.Sprintf("wss://%s/v1/connect", config.ServerAddress) + wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", config.ServerAddress) headers := map[string][]string{ "X-K3s-NodeName": {config.AgentConfig.NodeName}, } diff --git a/agent/util/file.go b/pkg/agent/util/file.go similarity index 87% rename from agent/util/file.go rename to pkg/agent/util/file.go index 1e20e05c8a..0cace178c7 100644 --- a/agent/util/file.go +++ b/pkg/agent/util/file.go @@ -9,7 +9,7 @@ import ( ) func WriteFile(name string, content string) error { - os.Mkdir(filepath.Dir(name), 0755) + os.MkdirAll(filepath.Dir(name), 0755) err := ioutil.WriteFile(name, []byte(content), 0644) if err != nil { return errors.Wrapf(err, "writing %s", name) diff --git a/pkg/cli/agent/agent.go b/pkg/cli/agent/agent.go new file mode 100644 index 0000000000..c148e374cd --- /dev/null +++ b/pkg/cli/agent/agent.go @@ -0,0 +1,38 @@ +package agent + +import ( + "context" + "fmt" + "os" + + "github.com/rancher/k3s/pkg/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/norman/pkg/resolvehome" + "github.com/rancher/norman/signal" + "github.com/urfave/cli" +) + +func Run(ctx *cli.Context) error { + if os.Getuid() != 0 { + return fmt.Errorf("agent must be ran as root") + } + + if cmds.AgentConfig.Token == "" { + return fmt.Errorf("--token is required") + } + + if cmds.AgentConfig.ServerURL == "" { + return fmt.Errorf("--server is required") + } + + dataDir, err := resolvehome.Resolve(cmds.AgentConfig.DataDir) + if err != nil { + return err + } + + cfg := cmds.AgentConfig + cfg.DataDir = dataDir + + contextCtx := signal.SigTermCancelContext(context.Background()) + return agent.Run(contextCtx, cfg) +} diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go new file mode 100644 index 0000000000..53e74cffed --- /dev/null +++ b/pkg/cli/cmds/agent.go @@ -0,0 +1,81 @@ +package cmds + +import ( + "os" + "path/filepath" + + "github.com/urfave/cli" +) + +type Agent struct { + Token string + ServerURL string + DataDir string + NodeIP string + NodeName string + Docker bool + NoFlannel bool + Debug bool + AgentShared +} + +type AgentShared struct { + NodeIP string +} + +var ( + appName = filepath.Base(os.Args[0]) + AgentConfig Agent + NodeIPFlag = cli.StringFlag{ + Name: "node-ip,i", + Usage: "IP address to advertise for node", + Destination: &AgentConfig.NodeIP, + } + NodeNameFlag = cli.StringFlag{ + Name: "node-name", + Usage: "Node name", + EnvVar: "K3S_NODE_NAME", + Destination: &AgentConfig.NodeName, + } +) + +func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command { + return cli.Command{ + Name: "agent", + Usage: "Run node agent", + UsageText: appName + " agent [OPTIONS]", + Action: action, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "token,t", + Usage: "Token to use for authentication", + EnvVar: "K3S_TOKEN", + Destination: &AgentConfig.Token, + }, + cli.StringFlag{ + Name: "server,s", + Usage: "Server to connect to", + EnvVar: "K3S_URL", + Destination: &AgentConfig.ServerURL, + }, + cli.StringFlag{ + Name: "data-dir,d", + Usage: "Folder to hold state", + Destination: &AgentConfig.DataDir, + Value: "/var/lib/rancher/k3s", + }, + cli.BoolFlag{ + Name: "docker", + Usage: "Use docker instead of containerd", + Destination: &AgentConfig.Docker, + }, + cli.BoolFlag{ + Name: "no-flannel", + Usage: "Disable embedded flannel", + Destination: &AgentConfig.NoFlannel, + }, + NodeNameFlag, + NodeIPFlag, + }, + } +} diff --git a/pkg/cli/cmds/kubectl.go b/pkg/cli/cmds/kubectl.go new file mode 100644 index 0000000000..72bdd92371 --- /dev/null +++ b/pkg/cli/cmds/kubectl.go @@ -0,0 +1,15 @@ +package cmds + +import ( + "github.com/urfave/cli" +) + +func NewKubectlCommand(action func(*cli.Context) error) cli.Command { + return cli.Command{ + Name: "kubectl", + Usage: "Run kubectl", + SkipFlagParsing: true, + SkipArgReorder: true, + Action: action, + } +} diff --git a/pkg/cli/cmds/root.go b/pkg/cli/cmds/root.go new file mode 100644 index 0000000000..1f2923f587 --- /dev/null +++ b/pkg/cli/cmds/root.go @@ -0,0 +1,39 @@ +package cmds + +import ( + "fmt" + + "github.com/rancher/k3s/pkg/version" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var ( + debug bool +) + +func NewApp() *cli.App { + app := cli.NewApp() + app.Name = appName + app.Usage = "Kubernetes, but small and simple" + app.Version = version.Version + cli.VersionPrinter = func(c *cli.Context) { + fmt.Printf("%s version %s\n", app.Name, app.Version) + } + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + Usage: "Turn on debug logs", + Destination: &debug, + }, + } + + app.Before = func(ctx *cli.Context) error { + if debug { + logrus.SetLevel(logrus.DebugLevel) + } + return nil + } + + return app +} diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go new file mode 100644 index 0000000000..3836954fb7 --- /dev/null +++ b/pkg/cli/cmds/server.go @@ -0,0 +1,62 @@ +package cmds + +import ( + "github.com/urfave/cli" +) + +type Server struct { + Log string + ClusterCIDR string + ServiceCIDR string + HTTPSPort int + HTTPPort int + DataDir string + DisableAgent bool +} + +var ServerConfig Server + +func NewServerCommand(action func(*cli.Context) error) cli.Command { + return cli.Command{ + Name: "server", + Usage: "Run management server", + UsageText: appName + " server [OPTIONS]", + Action: action, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "https-listen-port", + Usage: "HTTPS listen port", + Value: 6443, + Destination: &ServerConfig.HTTPSPort, + }, + cli.IntFlag{ + Name: "http-listen-port", + Usage: "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)", + Value: 0, + Destination: &ServerConfig.HTTPPort, + }, + cli.StringFlag{ + Name: "data-dir,d", + Usage: "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root", + Destination: &ServerConfig.DataDir, + }, + cli.BoolFlag{ + Name: "disable-agent", + Usage: "Do not run a local agent and register a local kubelet", + Destination: &ServerConfig.DisableAgent, + }, + cli.StringFlag{ + Name: "log,l", + Usage: "Log to file", + Destination: &ServerConfig.Log, + }, + cli.StringFlag{ + Name: "cluster-cidr", + Usage: "Network CIDR to use for pod IPs", + Destination: &ServerConfig.ClusterCIDR, + }, + NodeIPFlag, + NodeNameFlag, + }, + } +} diff --git a/pkg/cli/kubectl/kubectl.go b/pkg/cli/kubectl/kubectl.go new file mode 100644 index 0000000000..4c047a4c53 --- /dev/null +++ b/pkg/cli/kubectl/kubectl.go @@ -0,0 +1,11 @@ +package kubectl + +import ( + "github.com/rancher/k3s/pkg/kubectl" + "github.com/urfave/cli" +) + +func Run(ctx *cli.Context) error { + kubectl.Main() + return nil +} diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go new file mode 100644 index 0000000000..2b317a8d36 --- /dev/null +++ b/pkg/cli/server/server.go @@ -0,0 +1,90 @@ +package server + +import ( + "context" + "flag" + "fmt" + "os" + "path/filepath" + + "github.com/docker/docker/pkg/reexec" + "github.com/natefinch/lumberjack" + "github.com/rancher/k3s/pkg/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/server" + "github.com/rancher/norman/signal" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +func setupLogging(app *cli.Context) { + if !app.GlobalBool("debug") { + flag.Set("stderrthreshold", "3") + flag.Set("alsologtostderr", "false") + flag.Set("logtostderr", "false") + } +} + +func runWithLogging(app *cli.Context, cfg *cmds.Server) error { + l := &lumberjack.Logger{ + Filename: cfg.Log, + MaxSize: 50, + MaxBackups: 3, + MaxAge: 28, + Compress: true, + } + + args := append([]string{"k3s"}, os.Args[1:]...) + cmd := reexec.Command(args...) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "_RIO_REEXEC_=true") + cmd.Stderr = l + cmd.Stdout = l + cmd.Stdin = os.Stdin + return cmd.Run() +} + +func Run(app *cli.Context) error { + return run(app, &cmds.ServerConfig) +} + +func run(app *cli.Context, cfg *cmds.Server) error { + if cfg.Log != "" && os.Getenv("_RIO_REEXEC_") == "" { + return runWithLogging(app, cfg) + } + + setupLogging(app) + + if !cfg.DisableAgent && os.Getuid() != 0 { + return fmt.Errorf("must run as root unless --disable-agent is specified") + } + + serverConfig := server.Config{} + serverConfig.ControlConfig.DataDir = cfg.DataDir + serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort + serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort + + logrus.Info("Starting k3s ", app.App.Version) + ctx := signal.SigTermCancelContext(context.Background()) + certs, err := server.StartServer(ctx, &serverConfig) + if err != nil { + return err + } + + if cfg.DisableAgent { + <-ctx.Done() + return nil + } + + logFile := filepath.Join(serverConfig.ControlConfig.DataDir, "agent/agent.log") + url := fmt.Sprintf("https://localhost:%d", serverConfig.TLSConfig.HTTPSPort) + logrus.Infof("Agent starting, logging to %s", logFile) + token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs) + + agentConfig := cmds.AgentConfig + agentConfig.DataDir = serverConfig.ControlConfig.DataDir + agentConfig.ServerURL = url + agentConfig.Token = token + + return agent.Run(ctx, agentConfig) +} diff --git a/pkg/containerd/builtins.go b/pkg/containerd/builtins.go new file mode 100644 index 0000000000..169db2ea44 --- /dev/null +++ b/pkg/containerd/builtins.go @@ -0,0 +1,39 @@ +// +build ctrd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +// register containerd builtins here +import ( + _ "github.com/containerd/containerd/diff/walking/plugin" + _ "github.com/containerd/containerd/gc/scheduler" + _ "github.com/containerd/containerd/runtime/restart/monitor" + _ "github.com/containerd/containerd/services/containers" + _ "github.com/containerd/containerd/services/content" + _ "github.com/containerd/containerd/services/diff" + _ "github.com/containerd/containerd/services/events" + _ "github.com/containerd/containerd/services/healthcheck" + _ "github.com/containerd/containerd/services/images" + _ "github.com/containerd/containerd/services/introspection" + _ "github.com/containerd/containerd/services/leases" + _ "github.com/containerd/containerd/services/namespaces" + _ "github.com/containerd/containerd/services/opt" + _ "github.com/containerd/containerd/services/snapshots" + _ "github.com/containerd/containerd/services/tasks" + _ "github.com/containerd/containerd/services/version" +) diff --git a/pkg/containerd/builtins_cri_linux.go b/pkg/containerd/builtins_cri_linux.go new file mode 100644 index 0000000000..a35477f8c2 --- /dev/null +++ b/pkg/containerd/builtins_cri_linux.go @@ -0,0 +1,21 @@ +// +build ctrd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import _ "github.com/containerd/cri" diff --git a/pkg/containerd/builtins_linux.go b/pkg/containerd/builtins_linux.go new file mode 100644 index 0000000000..2b28979b15 --- /dev/null +++ b/pkg/containerd/builtins_linux.go @@ -0,0 +1,28 @@ +// +build ctrd + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + _ "github.com/containerd/containerd/metrics/cgroups" + _ "github.com/containerd/containerd/runtime/v1/linux" + _ "github.com/containerd/containerd/runtime/v2" + _ "github.com/containerd/containerd/runtime/v2/runc/options" + _ "github.com/containerd/containerd/snapshots/native" + _ "github.com/containerd/containerd/snapshots/overlay" +) diff --git a/pkg/containerd/main.go b/pkg/containerd/main.go new file mode 100644 index 0000000000..75ccc188fa --- /dev/null +++ b/pkg/containerd/main.go @@ -0,0 +1,21 @@ +// +build ctrd + +package containerd + +import ( + "fmt" + "os" + + "github.com/containerd/containerd/cmd/containerd/command" + "github.com/containerd/containerd/pkg/seed" +) + +func Main() { + //klog.InitFlags(nil) + seed.WithTimeAndRand() + app := command.App() + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "containerd: %s\n", err) + os.Exit(1) + } +} diff --git a/pkg/containerd/none.go b/pkg/containerd/none.go new file mode 100644 index 0000000000..44112ee5e7 --- /dev/null +++ b/pkg/containerd/none.go @@ -0,0 +1,6 @@ +// +build !ctrd + +package containerd + +func Main() { +} diff --git a/pkg/controller/tls/handler.go b/pkg/controller/tls/handler.go deleted file mode 100644 index 7dfa0d20f2..0000000000 --- a/pkg/controller/tls/handler.go +++ /dev/null @@ -1,12 +0,0 @@ -package tls - -import ( - "context" - - v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1" -) - -func Register(ctx context.Context) error { - clients := v1.ClientsFrom(ctx) - clients.ListenerConfig.OnChange(ctx) -} diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 1dff5881d9..1171eb6e3a 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -3,10 +3,12 @@ package agent import ( "context" "math/rand" + "path/filepath" "time" - "github.com/rancher/rio/pkg/daemons/config" + "k8s.io/apimachinery/pkg/util/net" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" "k8s.io/apiserver/pkg/util/logs" app2 "k8s.io/kubernetes/cmd/kube-proxy/app" @@ -18,23 +20,12 @@ import ( func Agent(config *config.Agent) error { rand.Seed(time.Now().UTC().UnixNano()) - prepare(config) - kubelet(config) kubeProxy(config) return nil } -func prepare(config *config.Agent) { - if config.CNIBinDir == "" { - config.CNIBinDir = "/opt/cni/bin" - } - if config.CNIConfDir == "" { - config.CNIConfDir = "/etc/cni/net.d" - } -} - func kubeProxy(config *config.Agent) { args := []string{ "--proxy-mode", "iptables", @@ -52,7 +43,7 @@ func kubeProxy(config *config.Agent) { }() } -func kubelet(config *config.Agent) { +func kubelet(cfg *config.Agent) { command := app.NewKubeletCommand(context.Background().Done()) logs.InitLogs() defer logs.FlushLogs() @@ -62,37 +53,51 @@ func kubelet(config *config.Agent) { "--read-only-port", "0", "--allow-privileged=true", "--cluster-domain", "cluster.local", - "--kubeconfig", config.KubeConfig, + "--kubeconfig", cfg.KubeConfig, "--eviction-hard", "imagefs.available<5%,nodefs.available<5%", "--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%", - "--feature-gates=MountPropagation=true", - "--node-ip", config.NodeIP, + "--node-ip", cfg.NodeIP, "--fail-swap-on=false", - "--cgroup-root", "/k3s", + //"--cgroup-root", "/k3s", "--cgroup-driver", "cgroupfs", - "--cni-conf-dir", config.CNIConfDir, - "--cni-bin-dir", config.CNIBinDir, } - if len(config.ClusterDNS) > 0 { - args = append(args, "--cluster-dns", config.ClusterDNS.String()) + if cfg.RootDir != "" { + args = append(args, "--root-dir", cfg.RootDir) + args = append(args, "--cert-dir", filepath.Join(cfg.RootDir, "pki")) + args = append(args, "--seccomp-profile-root", filepath.Join(cfg.RootDir, "seccomp")) } - if config.RuntimeSocket != "" { - args = append(args, "--container-runtime-endpoint", config.RuntimeSocket) + if cfg.CNIConfDir != "" { + args = append(args, "--cni-conf-dir", cfg.CNIConfDir) } - if config.ListenAddress != "" { - args = append(args, "--address", config.ListenAddress) + if cfg.CNIBinDir != "" { + args = append(args, "--cni-bin-dir", cfg.CNIBinDir) } - if config.CACertPath != "" { - args = append(args, "--anonymous-auth=false", "--client-ca-file", config.CACertPath) + if len(cfg.ClusterDNS) > 0 { + args = append(args, "--cluster-dns", cfg.ClusterDNS.String()) } - if config.NodeName != "" { - args = append(args, "--hostname-override", config.NodeName) + if cfg.RuntimeSocket != "" { + args = append(args, "--container-runtime", "remote") + args = append(args, "--container-runtime-endpoint", cfg.RuntimeSocket) } - args = append(args, config.ExtraKubeletArgs...) + if cfg.ListenAddress != "" { + args = append(args, "--address", cfg.ListenAddress) + } + if cfg.CACertPath != "" { + args = append(args, "--anonymous-auth=false", "--client-ca-file", cfg.CACertPath) + } + if cfg.NodeName != "" { + args = append(args, "--hostname-override", cfg.NodeName) + } + defaultIP, err := net.ChooseHostInterface() + if err != nil || defaultIP.String() != cfg.NodeIP { + args = append(args, "--node-ip", cfg.NodeIP) + } + args = append(args, cfg.ExtraKubeletArgs...) command.SetArgs(args) go func() { + logrus.Infof("Running kubelet %s", config.ArgString(args)) logrus.Fatalf("kubelet exited: %v", command.Execute()) }() } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 9520a6534f..6f5b1922c0 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "net" "net/http" + "strings" "k8s.io/apiserver/pkg/authentication/authenticator" ) @@ -11,18 +12,28 @@ import ( type Node struct { Docker bool NoFlannel bool - NoCoreDNS bool + FlannelConf string LocalAddress string + Containerd Containerd AgentConfig Agent CACerts []byte ServerAddress string Certificate *tls.Certificate } +type Containerd struct { + Address string + Root string + State string + Config string + Opt string +} + type Agent struct { NodeName string ClusterCIDR net.IPNet ClusterDNS net.IP + RootDir string KubeConfig string NodeIP string RuntimeSocket string @@ -39,6 +50,8 @@ type Control struct { ListenPort int ClusterIPRange *net.IPNet ServiceIPRange *net.IPNet + ClusterDNS net.IP + NoCoreDNS bool DataDir string ETCDEndpoints []string ETCDKeyFile string @@ -48,7 +61,7 @@ type Control struct { ExtraAPIArgs []string ExtraControllerArgs []string ExtraSchedulerAPIArgs []string - NodeConfig Node + //NodeConfig Node Runtime *ControlRuntime `json:"-"` } @@ -72,3 +85,16 @@ type ControlRuntime struct { Tunnel http.Handler Authenticator authenticator.Request } + +type ArgString []string + +func (a ArgString) String() string { + b := strings.Builder{} + for _, s := range a { + if b.Len() > 0 { + b.WriteString(" ") + } + b.WriteString(s) + } + return b.String() +} diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 9d7d790b25..d224be8ca6 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/rancher/rio/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/config" _ "github.com/mattn/go-sqlite3" // sqlite "github.com/sirupsen/logrus" @@ -93,81 +93,68 @@ func Server(ctx context.Context, cfg *config.Control) error { return nil } -func controllerManager(config *config.Control, runtime *config.ControlRuntime) { +func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { args := []string{ "--kubeconfig", runtime.KubeConfigSystem, "--leader-elect=true", "--service-account-private-key-file", runtime.ServiceKey, "--allocate-node-cidrs", - "--cluster-cidr", config.ClusterIPRange.String(), + "--cluster-cidr", cfg.ClusterIPRange.String(), "--root-ca-file", runtime.TokenCA, "--port", "0", "--secure-port", "0", } - args = append(args, config.ExtraControllerArgs...) + args = append(args, cfg.ExtraControllerArgs...) command := cmapp.NewControllerManagerCommand() command.SetArgs(args) go func() { - logrus.Infof("Running kube-controller-manager %s", argString(args)) + logrus.Infof("Running kube-controller-manager %s", config.ArgString(args)) logrus.Fatalf("controller-manager exited: %v", command.Execute()) }() } -type argString []string - -func (a argString) String() string { - b := strings.Builder{} - for _, s := range a { - if b.Len() > 0 { - b.WriteString(" ") - } - b.WriteString(s) - } - return b.String() -} - -func scheduler(config *config.Control, runtime *config.ControlRuntime) { +func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { args := []string{ "--kubeconfig", runtime.KubeConfigSystem, "--port", "0", "--secure-port", "0", } - args = append(args, config.ExtraSchedulerAPIArgs...) + args = append(args, cfg.ExtraSchedulerAPIArgs...) command := sapp.NewSchedulerCommand() command.SetArgs(args) go func() { - logrus.Infof("Running kube-scheduler %s", argString(args)) + logrus.Infof("Running kube-scheduler %s", config.ArgString(args)) logrus.Fatalf("scheduler exited: %v", command.Execute()) }() } -func apiServer(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) { +func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) { var args []string - if len(config.ETCDEndpoints) > 0 { + if len(cfg.ETCDEndpoints) > 0 { args = append(args, "--storage-backend", "etcd3") - args = append(args, "--etcd-servers", strings.Join(config.ETCDEndpoints, ",")) - if config.ETCDKeyFile != "" { - args = append(args, "--etcd-keyfile", config.ETCDKeyFile) + args = append(args, "--etcd-servers", strings.Join(cfg.ETCDEndpoints, ",")) + if cfg.ETCDKeyFile != "" { + args = append(args, "--etcd-keyfile", cfg.ETCDKeyFile) } - if config.ETCDCAFile != "" { - args = append(args, "--etcd-cafile", config.ETCDCAFile) + if cfg.ETCDCAFile != "" { + args = append(args, "--etcd-cafile", cfg.ETCDCAFile) } - if config.ETCDCertFile != "" { - args = append(args, "--etcd-certfile", config.ETCDCertFile) + if cfg.ETCDCertFile != "" { + args = append(args, "--etcd-certfile", cfg.ETCDCertFile) } } args = append(args, "--allow-privileged=true") args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ",")) args = append(args, "--service-account-signing-key-file", runtime.ServiceKey) - args = append(args, "--service-cluster-ip-range", config.ServiceIPRange.String()) - args = append(args, "--advertise-port", strconv.Itoa(config.AdvertisePort)) + args = append(args, "--service-cluster-ip-range", cfg.ServiceIPRange.String()) + args = append(args, "--advertise-port", strconv.Itoa(cfg.AdvertisePort)) args = append(args, "--advertise-address", localhostIP.String()) args = append(args, "--insecure-port", "0") - args = append(args, "--secure-port", strconv.Itoa(config.ListenPort)) + args = append(args, "--secure-port", strconv.Itoa(cfg.ListenPort)) args = append(args, "--bind-address", localhostIP.String()) args = append(args, "--tls-cert-file", runtime.TLSCert) args = append(args, "--tls-private-key-file", runtime.TLSKey) @@ -175,15 +162,15 @@ func apiServer(ctx context.Context, config *config.Control, runtime *config.Cont args = append(args, "--service-account-issuer", "k3s") args = append(args, "--api-audiences", "unknown") args = append(args, "--basic-auth-file", runtime.PasswdFile) - //args = append(args, "--kubelet-client-certificate", runtime.NodeCert) - //args = append(args, "--kubelet-client-key", runtime.NodeKey) + args = append(args, "--kubelet-client-certificate", runtime.NodeCert) + args = append(args, "--kubelet-client-key", runtime.NodeKey) - args = append(args, config.ExtraAPIArgs...) + args = append(args, cfg.ExtraAPIArgs...) command := app.NewAPIServerCommand(ctx.Done()) command.SetArgs(args) go func() { - logrus.Infof("Running kube-apiserver %s", argString(args)) + logrus.Infof("Running kube-apiserver %s", config.ArgString(args)) logrus.Fatalf("apiserver exited: %v", command.Execute()) }() @@ -203,6 +190,10 @@ func defaults(config *config.Control) { config.ServiceIPRange = serviceIPNet } + if len(config.ClusterDNS) == 0 { + config.ClusterDNS = net.ParseIP("10.43.0.10") + } + if config.AdvertisePort == 0 { config.AdvertisePort = 6445 } diff --git a/pkg/datadir/datadir.go b/pkg/datadir/datadir.go new file mode 100644 index 0000000000..863b65a57a --- /dev/null +++ b/pkg/datadir/datadir.go @@ -0,0 +1,25 @@ +package datadir + +import ( + "os" + + "github.com/pkg/errors" + "github.com/rancher/norman/pkg/resolvehome" +) + +func Resolve(dataDir string) (string, error) { + if dataDir == "" { + if os.Getuid() == 0 { + dataDir = "/var/lib/rancher/k3s" + } else { + dataDir = "${HOME}/.rancher/k3s" + } + } + + dataDir, err := resolvehome.Resolve(dataDir) + if err != nil { + return "", errors.Wrapf(err, "resolving %s", dataDir) + } + + return dataDir, nil +} diff --git a/pkg/enterchroot/enter.go b/pkg/enterchroot/enter.go deleted file mode 100644 index 301405c097..0000000000 --- a/pkg/enterchroot/enter.go +++ /dev/null @@ -1,254 +0,0 @@ -package enterchroot - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/docker/docker/pkg/mount" - "github.com/docker/docker/pkg/reexec" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - losetup "gopkg.in/freddierice/go-losetup.v1" -) - -const ( - magic = "_SQMAGIC_" -) - -var ( - symlinks = []string{"lib", "bin", "sbin", "lib64"} -) - -func init() { - reexec.Register("enter-root", enter) -} - -func enter() { - if os.Getenv("ENTER_DEBUG") == "true" { - logrus.SetLevel(logrus.DebugLevel) - } - - logrus.Debug("Running bootstrap") - err := run(os.Getenv("ENTER_DATA")) - if err != nil { - logrus.Fatal(err) - } -} - -func Mount(dataDir string, stdout, stderr io.Writer, args []string) error { - if logrus.GetLevel() >= logrus.DebugLevel { - os.Setenv("ENTER_DEBUG", "true") - } - - root, offset, err := findRoot() - if err != nil { - return err - } - - os.Setenv("ENTER_DATA", dataDir) - os.Setenv("ENTER_ROOT", root) - - logrus.Debugf("Using data [%s] root [%s]", dataDir, root) - - stat, err := os.Stat(root) - if err != nil { - return fmt.Errorf("failed to find %s: %v", root, err) - } - - if !stat.IsDir() { - logrus.Debugf("Attaching file [%s] offset [%d]", root, offset) - dev, err := losetup.Attach(root, offset, true) - if err != nil { - return errors.Wrap(err, "creating loopback device") - } - defer dev.Detach() - os.Setenv("ENTER_DEVICE", dev.Path()) - - go func() { - // Assume that after 3 seconds loop back device has been mounted - time.Sleep(3 * time.Second) - info, err := dev.GetInfo() - if err != nil { - return - } - - info.Flags |= losetup.FlagsAutoClear - err = dev.SetInfo(info) - if err != nil { - return - } - }() - } - - logrus.Debugf("Running enter-root %v", args) - cmd := &exec.Cmd{ - Path: reexec.Self(), - Args: append([]string{"enter-root"}, args...), - SysProcAttr: &syscall.SysProcAttr{ - //Cloneflags: syscall.CLONE_NEWPID | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC, - Unshareflags: syscall.CLONE_NEWNS, - Pdeathsig: syscall.SIGKILL, - }, - Stdout: stdout, - Stdin: os.Stdin, - Stderr: stderr, - Env: os.Environ(), - } - return cmd.Run() -} - -func findRoot() (string, uint64, error) { - root := os.Getenv("ENTER_ROOT") - if root != "" { - return root, 0, nil - } - - for _, suffix := range []string{".root", ".squashfs"} { - test := os.Args[0] + suffix - if _, err := os.Stat(test); err == nil { - return test, 0, nil - } - } - - return inFile() -} - -func inFile() (string, uint64, error) { - f, err := os.Open(reexec.Self()) - if err != nil { - return "", 0, err - } - defer f.Close() - - buf := make([]byte, 8192) - test := []byte(strings.ToLower(magic)) - testLength := len(test) - offset := uint64(0) - found := 0 - - for { - n, err := f.Read(buf) - if err == io.EOF && n == 0 { - break - } else if err != nil { - return "", 0, err - } - - for _, b := range buf[:n] { - if b == test[found] { - found++ - if found == testLength { - return reexec.Self(), offset + 1, nil - } - } else { - found = 0 - } - offset++ - } - } - - return "", 0, fmt.Errorf("failed to find image in file %s", os.Args[0]) -} - -func run(data string) error { - os.MkdirAll(data, 0755) - - if err := mount.Mount("tmpfs", data, "tmpfs", ""); err != nil { - return errors.Wrapf(err, "remounting data %s", data) - } - - root := os.Getenv("ENTER_ROOT") - device := os.Getenv("ENTER_DEVICE") - - logrus.Debugf("Using root %s %s", root, device) - - usr := filepath.Join(data, "usr") - dotRoot := filepath.Join(data, ".root") - - for _, d := range []string{usr, dotRoot} { - if err := os.MkdirAll(d, 0755); err != nil { - return fmt.Errorf("failed to make dir %s: %v", data, err) - } - } - - if device == "" { - logrus.Debugf("Bind mounting %s to %s", root, usr) - if err := mount.Mount(root, usr, "none", "bind"); err != nil { - return fmt.Errorf("failed to bind mount") - } - } else { - logrus.Debugf("Mounting squashfs %s to %s", device, usr) - squashErr := checkSquashfs() - if err := mount.Mount(device, usr, "squashfs", "ro"); err != nil { - err = errors.Wrap(err, "mounting squashfs") - if squashErr != nil { - err = errors.Wrap(err, squashErr.Error()) - } - return err - } - } - - if err := os.Chdir(data); err != nil { - return err - } - - for _, p := range symlinks { - if _, err := os.Lstat(p); os.IsNotExist(err) { - if err := os.Symlink(filepath.Join("usr", p), p); err != nil { - return errors.Wrapf(err, "failed to symlink %s", p) - } - } - } - - logrus.Debugf("pivoting to . .root") - if err := syscall.PivotRoot(".", ".root"); err != nil { - return errors.Wrap(err, "pivot_root failed") - } - - if err := mount.ForceMount("", ".", "none", "rprivate"); err != nil { - return errors.Wrapf(err, "making . private %s", data) - } - - if err := syscall.Chroot("/"); err != nil { - return err - } - - if err := os.Chdir("/"); err != nil { - return err - } - - if _, err := os.Stat("/usr/init"); err != nil { - return errors.Wrap(err, "failed to find /usr/init") - } - - return syscall.Exec("/usr/init", os.Args, os.Environ()) -} - -func checkSquashfs() error { - if !inProcFS() { - exec.Command("modprobe", "squashfs").Run() - } - - if !inProcFS() { - return errors.New("This kernel does not support squashfs, please enable. " + - "On Fedora you may need to run \"dnf install kernel-modules-$(uname -r)\"") - } - - return nil -} - -func inProcFS() bool { - bytes, err := ioutil.ReadFile("/proc/filesystems") - if err != nil { - logrus.Errorf("Failed to read /proc/filesystems: %v", err) - return false - } - return strings.Contains(string(bytes), "squashfs") -} diff --git a/pkg/kubectl/main.go b/pkg/kubectl/main.go index c73aad1198..239bb5ca79 100644 --- a/pkg/kubectl/main.go +++ b/pkg/kubectl/main.go @@ -7,18 +7,13 @@ import ( "os" "time" - "github.com/docker/docker/pkg/reexec" - "github.com/rancher/rio/pkg/server" + "github.com/rancher/k3s/pkg/server" "github.com/spf13/pflag" utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/pkg/kubectl/cmd" ) -func init() { - reexec.Register("kubectl", Main) -} - func Main() { kubenv := os.Getenv("KUBECONFIG") if kubenv == "" { diff --git a/pkg/server/auth.go b/pkg/server/auth.go index f0d20aed46..e0cb0a0d6d 100644 --- a/pkg/server/auth.go +++ b/pkg/server/auth.go @@ -4,7 +4,7 @@ import ( "net/http" "github.com/gorilla/mux" - "github.com/rancher/rio/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/config" "github.com/sirupsen/logrus" "k8s.io/apiserver/pkg/endpoints/request" ) diff --git a/pkg/server/router.go b/pkg/server/router.go index 83eb9b36b1..74cbe78326 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -3,11 +3,9 @@ package server import ( "net/http" - "github.com/rancher/rio/pkg/daemons/config" - - "k8s.io/apimachinery/pkg/util/json" - "github.com/gorilla/mux" + "github.com/rancher/k3s/pkg/daemons/config" + "k8s.io/apimachinery/pkg/util/json" ) type CACertsGetter func() (string, error) diff --git a/pkg/server/server.go b/pkg/server/server.go index 8cf2f52dea..1623a646c8 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -6,21 +6,22 @@ import ( "encoding/hex" "fmt" "io/ioutil" + net2 "net" "os" "path/filepath" "strings" "time" "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control" + "github.com/rancher/k3s/pkg/tls" + v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" "github.com/rancher/norman" "github.com/rancher/norman/pkg/clientaccess" "github.com/rancher/norman/pkg/dynamiclistener" "github.com/rancher/norman/pkg/resolvehome" "github.com/rancher/norman/types" - "github.com/rancher/rio/pkg/daemons/config" - "github.com/rancher/rio/pkg/daemons/control" - "github.com/rancher/rio/pkg/tls" - v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/net" ) @@ -38,25 +39,29 @@ func resolveDataDir(dataDir string) (string, error) { return resolvehome.Resolve(dataDir) } -func StartServer(ctx context.Context, config *Config) error { +func StartServer(ctx context.Context, config *Config) (string, error) { if err := setupDataDirAndChdir(&config.ControlConfig); err != nil { - return err + return "", err } if err := control.Server(ctx, &config.ControlConfig); err != nil { - return errors.Wrap(err, "starting kubernetes") + return "", errors.Wrap(err, "starting kubernetes") } certs, err := startNorman(ctx, &config.TLSConfig, &config.ControlConfig) if err != nil { - return errors.Wrap(err, "starting tls server") + return "", errors.Wrap(err, "starting tls server") } - printTokens(certs, config.ControlConfig.NodeConfig.AgentConfig.NodeIP, &config.TLSConfig, &config.ControlConfig) + ip, err := net.ChooseHostInterface() + if err != nil { + ip = net2.ParseIP("127.0.0.1") + } + printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig) writeKubeConfig(certs, &config.TLSConfig, &config.ControlConfig) - return nil + return certs, nil } func startNorman(ctx context.Context, tlsConfig *dynamiclistener.UserConfig, config *config.Control) (string, error) { @@ -203,7 +208,7 @@ func printToken(httpsPort int, advertiseIP, prefix, file, cmd string) { ip = hostIP.String() } - logrus.Infof("%s rio %s -s https://%s:%d -t %s", prefix, cmd, ip, httpsPort, token) + logrus.Infof("%s k3s %s -s https://%s:%d -t %s", prefix, cmd, ip, httpsPort, token) } func FormatToken(token string, certs string) string { diff --git a/pkg/server/types.go b/pkg/server/types.go index 27216cd2c9..63f0dc56d8 100644 --- a/pkg/server/types.go +++ b/pkg/server/types.go @@ -1,8 +1,8 @@ package server import ( + "github.com/rancher/k3s/pkg/daemons/config" "github.com/rancher/norman/pkg/dynamiclistener" - "github.com/rancher/rio/pkg/daemons/config" ) type Config struct { diff --git a/pkg/tls/storage.go b/pkg/tls/storage.go index 4afd73acbd..95cad94f07 100644 --- a/pkg/tls/storage.go +++ b/pkg/tls/storage.go @@ -3,8 +3,8 @@ package tls import ( "context" + v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" "github.com/rancher/norman/pkg/dynamiclistener" - v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/untar/untar.go b/pkg/untar/untar.go new file mode 100644 index 0000000000..3b344bd0f2 --- /dev/null +++ b/pkg/untar/untar.go @@ -0,0 +1,132 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package untar untars a tarball to disk. +package untar + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/sirupsen/logrus" +) + +// TODO(bradfitz): this was copied from x/build/cmd/buildlet/buildlet.go +// but there were some buildlet-specific bits in there, so the code is +// forked for now. Unfork and add some opts arguments here, so the +// buildlet can use this code somehow. + +// Untar reads the gzip-compressed tar file from r and writes it into dir. +func Untar(r io.Reader, dir string) error { + return untar(r, dir) +} + +func untar(r io.Reader, dir string) (err error) { + t0 := time.Now() + nFiles := 0 + madeDir := map[string]bool{} + defer func() { + td := time.Since(t0) + if err != nil { + logrus.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err) + } + }() + zr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("requires gzip-compressed body: %v", err) + } + tr := tar.NewReader(zr) + loggedChtimesError := false + for { + f, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + logrus.Printf("tar reading error: %v", err) + return fmt.Errorf("tar error: %v", err) + } + if !validRelPath(f.Name) { + return fmt.Errorf("tar contained invalid name error %q", f.Name) + } + rel := filepath.FromSlash(f.Name) + abs := filepath.Join(dir, rel) + + fi := f.FileInfo() + mode := fi.Mode() + switch { + case mode.IsRegular(): + // Make the directory. This is redundant because it should + // already be made by a directory entry in the tar + // beforehand. Thus, don't check for errors; the next + // write will fail with the same error. + dir := filepath.Dir(abs) + if !madeDir[dir] { + if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil { + return err + } + madeDir[dir] = true + } + wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) + if err != nil { + return err + } + n, err := io.Copy(wf, tr) + if closeErr := wf.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("error writing to %s: %v", abs, err) + } + if n != f.Size { + return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) + } + modTime := f.ModTime + if modTime.After(t0) { + // Clamp modtimes at system time. See + // golang.org/issue/19062 when clock on + // buildlet was behind the gitmirror server + // doing the git-archive. + modTime = t0 + } + if !modTime.IsZero() { + if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError { + // benign error. Gerrit doesn't even set the + // modtime in these, and we don't end up relying + // on it anywhere (the gomote push command relies + // on digests only), so this is a little pointless + // for now. + logrus.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err) + loggedChtimesError = true // once is enough + } + } + nFiles++ + case mode.IsDir(): + if err := os.MkdirAll(abs, 0755); err != nil { + return err + } + madeDir[abs] = true + case f.Linkname != "": + if err := os.Symlink(f.Linkname, abs); err != nil { + return err + } + default: + return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode) + } + } + return nil +} + +func validRelPath(p string) bool { + if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { + return false + } + return true +} diff --git a/version/version.go b/pkg/version/version.go similarity index 100% rename from version/version.go rename to pkg/version/version.go diff --git a/scripts/build b/scripts/build index 0773cc1335..9fe5509c6a 100755 --- a/scripts/build +++ b/scripts/build @@ -1,31 +1,41 @@ #!/bin/bash -set -e +set -e -x source $(dirname $0)/version cd $(dirname $0)/.. -LDFLAGS="-X github.com/rancher/rio/version.Version=$VERSION -w -s" -STATIC="-extldflags -static" +LDFLAGS="-X github.com/rancher/k3s/version.Version=$VERSION -w -s" +STATIC="-extldflags '-static'" STATIC_SQLITE="-extldflags '-static -lm -ldl -lz -lpthread'" +TAGS="ctrd apparmor seccomp no_btrfs netgo osusergo" -cross() -{ - echo Building windows CLI - GOOS=windows GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-windows ./cli/main.go - echo Building mac CLI - GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-darwin ./cli/main.go -} +if [ "$STATIC_BUILD" != "true" ]; then + STATIC="" + STATIC_SQLITE="" +else + TAGS="static_build libsqlite3 $TAGS" +fi mkdir -p bin -if [ -n "$CROSS" ]; then - cross -fi - -echo Building incluster -GOOS=linux GOARCH=amd64 -CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o bin/rio-incluster - -echo Building cli full -CGO_ENABLED=1 go build -tags "static_build libsqlite3 k8s ctr no_etcd netgo osusergo" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/rio-full ./cli/main.go +rm -f bin/k3s-agent bin/hyperkube bin/containerd bin/cni ./bin/runc bin/containerd-shim bin/k3s-server bin/kubectl +# echo Building agent +# CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go +echo Building server +CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/containerd ./cmd/server/main.go +ln -s containerd ./bin/k3s-agent +ln -s containerd ./bin/k3s-server +ln -s containerd ./bin/kubectl +echo Building hyperkube +CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/ +# echo Building containerd +# CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/ +echo Building cni +CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/cni ./cmd/cni/main.go +echo Building runc +make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static +cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc +echo Building containerd-shim +make -C ./vendor/github.com/containerd/containerd bin/containerd-shim +cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim ./bin/containerd-shim diff --git a/scripts/ci b/scripts/ci index 9b9498e843..74dcc397db 100755 --- a/scripts/ci +++ b/scripts/ci @@ -6,4 +6,3 @@ cd $(dirname $0) ./validate ./build ./package -./test diff --git a/scripts/dev-agent.sh b/scripts/dev-agent.sh index 3543c494f2..0e9be1be96 100755 --- a/scripts/dev-agent.sh +++ b/scripts/dev-agent.sh @@ -1,14 +1,18 @@ #!/bin/bash set -e -cd $(dirname $0)/../bin +cd $(dirname $0)/.. # Prime sudo -sudo echo Compiling CLI -go build -tags "k8s no_etcd" -o rio-agent ../cli/main.go +sudo echo Compiling -echo Building image and agent -../image/build +if [ ! -e bin/containerd ]; then + ./scripts/build + ./scripts/package +else + rm -f ./bin/k3s-agent + go build -tags "apparmor seccomp" -o ./bin/k3s-agent ./cmd/agent/main.go +fi -echo Running -exec sudo ENTER_ROOT=../image/main.squashfs ./rio-agent --debug agent -s https://localhost:7443 -t $(<${HOME}/.rancher/rio/server/node-token) +echo Starting agent +sudo env "PATH=$(pwd)/bin:$PATH" ./bin/k3s-agent agent -s https://localhost:6443 -t $(<${HOME}/.rancher/k3s/server/node-token) "$@" diff --git a/scripts/dev-docker-agent.sh b/scripts/dev-docker-agent.sh new file mode 100755 index 0000000000..c9d8d47c46 --- /dev/null +++ b/scripts/dev-docker-agent.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +cd $(dirname $0)/.. +IP=$(ip addr show dev docker0 | grep -w inet | awk '{print $2}' | cut -f1 -d/) +docker run \ + --read-only \ + --tmpfs /run \ + --tmpfs /var/run \ + --tmpfs /tmp \ + -v /lib/modules:/lib/modules:ro \ + -v /lib/firmware:/lib/firmware:ro \ + -v /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro \ + -v $(pwd)/bin:/usr/bin \ + -v /var/log \ + -v /var/lib/rancher/k3s \ + -v /var/lib/cni \ + -v /usr/lib/x86_64-linux-gnu/libsqlite3.so.0:/usr/lib/x86_64-linux-gnu/libsqlite3.so.0:ro \ + --privileged \ + ubuntu:18.04 /usr/bin/k3s-agent agent -t $(<~/.rancher/k3s/server/node-token) -s https://${IP}:6443 diff --git a/scripts/dev-k8s-only-server.sh b/scripts/dev-k8s-only-server.sh deleted file mode 100755 index 255bf477b3..0000000000 --- a/scripts/dev-k8s-only-server.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -cd $(dirname $0)/../bin - -echo Running -go run -tags "k8s no_etcd" ../cli/main.go --debug server --disable-controllers --disable-agent diff --git a/scripts/dev-login.sh b/scripts/dev-login.sh deleted file mode 100755 index df29895588..0000000000 --- a/scripts/dev-login.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -cd $(dirname $0)/../bin - -rio login -s https://localhost:5443 -t $(<${HOME}/.rancher/rio/server/client-token) diff --git a/scripts/dev-server.sh b/scripts/dev-server.sh index bf31922961..19c41304f8 100755 --- a/scripts/dev-server.sh +++ b/scripts/dev-server.sh @@ -4,4 +4,4 @@ set -e cd $(dirname $0)/../bin echo Running -go run -tags k3s ../cli/main.go --debug server --disable-agent +go run -tags "apparmor" ../cmd/server/main.go --debug server --disable-agent diff --git a/scripts/package b/scripts/package index 86d02160d0..9e57953ae6 100755 --- a/scripts/package +++ b/scripts/package @@ -3,6 +3,9 @@ set -e cd $(dirname $0) +if [ ! -e ../bin/k3s-server ]; then + ./build +fi + ./package-cli ./package-image -./package-tar diff --git a/scripts/package-cli b/scripts/package-cli index 2bbcedb9ad..79f727e540 100755 --- a/scripts/package-cli +++ b/scripts/package-cli @@ -1,11 +1,42 @@ #!/bin/bash -set -e +set -e -x + +ROOT_VERSION=v0.0.1 source $(dirname $0)/version -cd $(dirname $0)/../image -./build +cd $(dirname $0)/.. -cp ../bin/rio-full ../bin/rio -echo -n "_sqmagic_" >> ../bin/rio -cat main.squashfs >> ../bin/rio +curl --compressed -sfL https://github.com/ibuildthecloud/k3s-root/releases/download/${ROOT_VERSION}/k3s-root-${ARCH}.tar | tar xf - + +rm -rf bin/kubectl bin/k3s-agent bin/k3s build/data +ln -s containerd bin/k3s-agent +ln -s containerd bin/k3s-server +ln -s containerd bin/kubectl +for i in bridge flannel host-local loopback portmap; do + if [ -e ./bin/$i ]; then + rm -f ./bin/$i + fi + ln -s cni ./bin/$i +done + + +mkdir -p build/data + +tar cvzf ./build/data.tar.gz --exclude ./bin/hyperkube ./bin +HASH=$(sha256sum ./build/data.tar.gz | awk '{print $1}') + +cp ./build/data.tar.gz ./build/data/${HASH}.tgz + +if [ ${ARCH} = amd64 ]; then + cp -f ./bin/hyperkube dist/hyperkube + CMD_NAME=dist/k3s +else + cp -f ./bin/hyperkube dist/hyperkube-${ARCH} + CMD_NAME=dist/k3s-${ARCH} +fi + +go generate +LDFLAGS="-X github.com/rancher/k3s/version.Version=$VERSION -w -s" +STATIC="-extldflags '-static'" +CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go diff --git a/scripts/package-image b/scripts/package-image index 8fbe222059..bb28b9a12d 100755 --- a/scripts/package-image +++ b/scripts/package-image @@ -7,10 +7,11 @@ cd $(dirname $0)/../package TAG=${TAG:-${VERSION}${SUFFIX}} REPO=${REPO:-rancher} +IMAGE_NAME=${IMAGE_NAME:-k3s} -cp ../bin/rio ./rio +cp -f ../build/data.tar.gz . -IMAGE=${REPO}/rio:${TAG} +IMAGE=${REPO}/${IMAGE_NAME}:${TAG} docker build -t ${IMAGE} . mkdir -p ../dist echo ${IMAGE} > ../dist/images diff --git a/scripts/package-tar b/scripts/package-tar deleted file mode 100755 index e59666598f..0000000000 --- a/scripts/package-tar +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -cd $(dirname $0)/.. - -. ./scripts/version - -mkdir -p dist/artifacts - -tar cvzf dist/artifacts/rio-${VERSION}-linux-amd64.tar.gz -h bin/rio --xform='s!^bin!rio-'${VERSION}'-linux-amd64!' -tar cvzf dist/artifacts/rio-${VERSION}-darwin.tar.gz bin/rio-darwin --xform='s!.*!rio-'${VERSION}'-darwin/rio!' - -W=rio-${VERSION}-windows -mkdir -p $W -trap "rm -rf $W" EXIT - -cp -f bin/rio-windows $W/rio.exe -zip dist/artifacts/rio-${VERSION}-windows.zip $W/rio.exe diff --git a/scripts/test b/scripts/test deleted file mode 100755 index f0e1177b7c..0000000000 --- a/scripts/test +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -e - -cd $(dirname $0)/.. - -echo Running tests - -mkdir -p ./build - -mkdir -p /var/lib/rancher/rio/agent -mount -t tmpfs none /var/lib/rancher/rio/agent - -./bin/rio server --disable-agent | grep -v level=info > /var/lib/rancher/rio/agent/agent.log 2>&1 & - -for i in {1..120}; do - if [ ! -e /var/lib/rancher/rio/server/node-token ]; then - sleep .5 - continue - fi - - curl -sf http://localhost:7080/healthz >/dev/null && break - sleep .5 -done -curl -sf http://localhost:7080/healthz >/dev/null - -rm -rf ./image/root -unsquashfs -d ./image/root ./image/main.squashfs -ENTER_ROOT=$(pwd)/image/root ./bin/rio --debug agent -s https://localhost:7443 -t $(>/var/lib/rancher/rio/agent/agent.log 2>&1 & - -export PATH=$(pwd)/bin:$PATH - -rio login -s https://localhost:7443 -t /var/lib/rancher/rio/server/client-token - -echo Waiting for istio/istio-gateway -rio --project=rio-system wait istio/istio-gateway -rio --project=rio-system ps -rio --project=rio-system ps -c - -chmod +x ./tests/init-nfs.bash -./tests/init-nfs.bash -export RUN_NFS_TEST=true - -cd ./tests -tox -- -n $(nproc) -cd .. - -bats -r ./tests || { - tail -n 100 /var/lib/rancher/rio/agent/agent.log - exit 1 -} diff --git a/scripts/validate b/scripts/validate index e8549215f3..de4865ab9e 100755 --- a/scripts/validate +++ b/scripts/validate @@ -5,18 +5,14 @@ cd $(dirname $0)/.. echo Running validation -if grep -r '^[[:space:]]*\[\[.*]][[:space:]]*$' tests; then - echo "Add \"|| false\" to bats tests in any place you use [[ ... ]]" - exit 1 -fi - PACKAGES="$(go list ./...)" -echo Running: go vet -go vet ${PACKAGES} +#echo Running: go vet +#go vet ${PACKAGES} echo Running: gometalinter for i in ${PACKAGES}; do if [ -n "$(gometalinter $i | \ + grep -v 'pkg/data/zz_generated_bindata.go' | \ grep -v 'should have comment.*or be unexported' | \ grep -v 'cli/cmd.*don.t use underscores in Go name' | \ grep -v 'cli/cmd.*should be DNS' | \ @@ -26,4 +22,6 @@ for i in ${PACKAGES}; do done test -z "$failed" echo Running: go fmt -test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)" +test -z "$(go fmt ${PACKAGES} | \ + grep -v 'pkg/data/zz_generated_bindata.go' | \ + tee /dev/stderr)" diff --git a/scripts/version b/scripts/version index 87d180fc0c..6798d78ccb 100755 --- a/scripts/version +++ b/scripts/version @@ -13,6 +13,5 @@ else VERSION="${COMMIT}${DIRTY}" fi -if [ -z "$ARCH" ]; then - ARCH=amd64 -fi +ARCH=$(go env GOARCH) +SUFFIX="-${ARCH}" diff --git a/types/codegen/main.go b/types/codegen/main.go index 30ef66aa92..13489cc4c1 100644 --- a/types/codegen/main.go +++ b/types/codegen/main.go @@ -1,16 +1,33 @@ package main import ( + bindata "github.com/jteeuwen/go-bindata" + v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1" "github.com/rancher/norman/generator" - v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1" "github.com/sirupsen/logrus" ) var ( - basePackage = "github.com/rancher/rio/types" + basePackage = "github.com/rancher/k3s/types" ) func main() { + bc := &bindata.Config{ + Input: []bindata.InputConfig{ + { + Path: "build/data", + Recursive: true, + }, + }, + Package: "data", + NoCompress: true, + NoMemCopy: true, + NoMetadata: true, + Output: "pkg/data/zz_generated_bindata.go", + } + if err := bindata.Translate(bc); err != nil { + logrus.Fatal(err) + } if err := generator.DefaultGenerate(v1.Schemas, basePackage, false, nil); err != nil { logrus.Fatal(err) } diff --git a/vendor.conf b/vendor.conf index 45c4a70c67..0b3993bdc3 100644 --- a/vendor.conf +++ b/vendor.conf @@ -1,11 +1,18 @@ -package=github.com/rancher/rio +package=github.com/rancher/k3s package=github.com/jteeuwen/go-bindata package=github.com/jteeuwen/go-bindata/go-bindata +package=github.com/containerd/containerd/cmd/containerd-shim +package=k8s.io/kubernetes/cmd/hyperkube +package=github.com/opencontainers/runc +package=github.com/coreos/go-systemd/activation +package=github.com/opencontainers/runc/libcontainer/nsenter +package=github.com/opencontainers/runc/libcontainer/specconv +package=github.com/opencontainers/runc/contrib/cmd/recvtty -k8s.io/kubernetes a1d7d1b140f43b6503311ffc1dd80017c553bf8e file:///home/darren/src/kuberlite/.git transitive=true,staging=true +k8s.io/kubernetes v1.13.1-k3s2 https://github.com/ibuildthecloud/k3s.git transitive=true,staging=true -github.com/rancher/norman 5726ebfba191eef161af9638e85bf4fcaa58e008 https://github.com/ibuildthecloud/norman.git -github.com/coreos/flannel 39af3d7e46f2efa156644e247bdcf3b5bc5f1394 +github.com/rancher/norman 29915f8336c0a242560a9fef1d11bbaf04660915 https://github.com/ibuildthecloud/norman.git +github.com/coreos/flannel 3d7cff78e2ca4cade87c6c7d44adf27fe3de2709 https://github.com/ibuildthecloud/flannel.git github.com/natefinch/lumberjack aee4629129445bbdfb69aa565537dcfa16544311 github.com/gorilla/mux v1.6.2 github.com/gorilla/websocket v1.2.0 @@ -15,3 +22,93 @@ golang.org/x/crypto a49355c7e3f8fe157a85be2f77e6e269a0f89602 gopkg.in/freddierice/go-losetup.v1 fc9adea44124401d8bfef3a97eaf61b5d44cc2c6 github.com/urfave/cli 8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c + +# flannel +github.com/golang/glog 23def4e6c14b4da8ac2ed8007337bc5eb5007998 +github.com/coreos/go-iptables 47f22b0dd3355c0ba570ba12b0b8a36bf214c04b + +# cni +github.com/containernetworking/plugins 9810b7d5137b171c4e07ce59bb18be9feccec557 https://github.com/ibuildthecloud/plugins.git +github.com/j-keck/arping 2cf9dc699c5640a7e2c81403a44127bf28033600 +github.com/alexflint/go-filemutex 72bdc8eae2aef913234599b837f5dda445ca9bd9 + +# runc +github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 + +# containerd +github.com/containerd/containerd v1.2.1 +github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3 +github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23 +github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2 +github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 +github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c +github.com/containerd/btrfs 2e1aa0ddf94f91fa282b6ed87c23bf0d64911244 +github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 +github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 +github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 +github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 +github.com/docker/go-units v0.3.1 +github.com/godbus/dbus c7fdd8b5cd55e87b4e1f4e372cdb1db61dd6c66f +#github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823 +#github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c +#github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563 +#github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd +github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9 +github.com/matttproud/golang_protobuf_extensions v1.0.0 +github.com/gogo/protobuf v1.0.0 +github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef +github.com/golang/protobuf v1.1.0 +#github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d +github.com/opencontainers/runc 96ec2177ae841256168fcf76954f7177af9446eb +github.com/sirupsen/logrus v1.0.3 +github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c +golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac +google.golang.org/grpc v1.12.0 +github.com/pkg/errors v0.8.0 +github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 +#golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2 https://github.com/golang/sys +github.com/opencontainers/image-spec v1.0.1 +golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895 +github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 +github.com/Microsoft/go-winio v0.4.11 +github.com/Microsoft/hcsshim v0.8.3 +google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 +golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 +github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a +github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16 +gotest.tools v2.1.0 +github.com/google/go-cmp v0.1.0 +go.etcd.io/bbolt v1.3.1-etcd.8 + +# cri dependencies +github.com/containerd/cri 0ca1e3c2b73b5c38e72f29bb76338d0078b23d6c # release/1.2 branch +github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90 +github.com/blang/semver v3.1.0 +github.com/containernetworking/cni v0.6.0 +#github.com/containernetworking/plugins v0.7.0 +github.com/davecgh/go-spew v1.1.0 +github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621 +#github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 +github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528 +github.com/emicklei/go-restful v2.2.1 +github.com/ghodss/yaml v1.0.0 +#github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed +github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c +github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55 +github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f +github.com/json-iterator/go 1.1.5 +github.com/modern-go/reflect2 1.0.1 +github.com/modern-go/concurrent 1.0.3 +github.com/opencontainers/runtime-tools v0.6.0 +github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a +github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0 +github.com/tchap/go-patricia v2.2.6 +github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6 +github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b +github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874 +golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067 +golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4 +golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631 +gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 +gopkg.in/yaml.v2 v2.2.1