Continued refactoring

.drone.yml

@@ -11,7 +11,7 @@ pipeline:
   stage-binaries:
     image: rancher/dapper:1.11.2
     commands:
-    - cp -f ./bin/rio-incluster ./package/rio
+    - cp -f ./bin/k3s-incluster ./package/k3s
     when:
       branch: master
       event: tag
@@ -19,7 +19,7 @@ pipeline:
   publish-image:
     image: plugins/docker
     dockerfile: package/Dockerfile
-    repo: rancher/rio
+    repo: rancher/k3s
     context: package/
     tag: ${DRONE_TAG}
     secrets: [docker_username, docker_password]

.gitignore

@@ -12,7 +12,8 @@
 /image/agent
 /image/go_build_agent
 /image/main.squashfs
-/package/rio
+/package/k3s
+/pkg/data/zz_generated_bindata.go
 __pycache__
 /tests/.pytest_cache/
 /tests/.tox/

Dockerfile.dapper

@@ -1,9 +1,8 @@
 FROM golang:1.11-alpine3.8
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
-    python3 py3-pip python3-dev openssl-dev libffi-dev
+    python3 py3-pip python3-dev openssl-dev libffi-dev libseccomp libseccomp-dev make
 RUN pip3 install 'tox==3.6.0'
-RUN npm install -g 'bats@1.1.0'
 RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static
 RUN go get -d golang.org/x/lint/golint && \
     git -C /go/src/golang.org/x/lint/golint checkout -b current 06c8688daad7faa9da5a0c2f163a3d14aac986ca && \
@@ -16,12 +15,13 @@ RUN go get -d github.com/alecthomas/gometalinter && \
     rm -rf /go/src /go/pkg
 
 ENV DAPPER_RUN_ARGS --privileged
-ENV DAPPER_ENV REPO TAG DRONE_TAG
+ENV DAPPER_ENV REPO TAG DRONE_TAG IMAGE_NAME
-ENV DAPPER_SOURCE /go/src/github.com/rancher/rio/
+ENV DAPPER_SOURCE /go/src/github.com/rancher/k3s/
 ENV DAPPER_OUTPUT ./bin ./dist
 ENV DAPPER_DOCKER_SOCKET true
 ENV HOME ${DAPPER_SOURCE}
 ENV CROSS true
+ENV STATIC_BUILD true
 WORKDIR ${DAPPER_SOURCE}
 
 ENTRYPOINT ["./scripts/entry"]

agent/main.go

@@ -1,58 +0,0 @@
-package main
-
-import (
-	"context"
-
-	"github.com/rancher/norman/signal"
-	"github.com/rancher/rio/agent/config"
-	"github.com/rancher/rio/agent/containerd"
-	"github.com/rancher/rio/agent/flannel"
-	"github.com/rancher/rio/agent/proxy"
-	"github.com/rancher/rio/agent/syssetup"
-	"github.com/rancher/rio/agent/tunnel"
-	"github.com/rancher/rio/pkg/daemons/agent"
-	"github.com/sirupsen/logrus"
-)
-
-func main() {
-	if err := run(); err != nil {
-		logrus.Fatal(err)
-	}
-}
-
-func run() error {
-	ctx := signal.SigTermCancelContext(context.Background())
-
-	nodeConfig := config.Get()
-
-	if nodeConfig.Docker {
-		nodeConfig.AgentConfig.RuntimeSocket = ""
-	} else {
-		containerd.Run(ctx, nodeConfig)
-	}
-
-	if err := syssetup.Configure(); err != nil {
-		return err
-	}
-
-	if err := tunnel.Setup(nodeConfig); err != nil {
-		return err
-	}
-
-	if err := proxy.Run(nodeConfig); err != nil {
-		return err
-	}
-
-	if err := agent.Agent(&nodeConfig.AgentConfig); err != nil {
-		return err
-	}
-
-	if !nodeConfig.NoFlannel {
-		if err := flannel.Run(ctx, nodeConfig); err != nil {
-			return err
-		}
-	}
-
-	<-ctx.Done()
-	return ctx.Err()
-}

cli/cmd/agent/agent.go

@@ -1,19 +0,0 @@
-package agent
-
-import "github.com/urfave/cli"
-
-type Agent struct {
-	T_Token   string `desc:"Token to use for authentication" env:"K3S_TOKEN"`
-	S_Server  string `desc:"Server to connect to" env:"K3S_URL"`
-	D_DataDir string `desc:"Folder to hold state" default:"/var/lib/rancher/k3s"`
-	L_Log     string `desc:"log to file"`
-	AgentShared
-}
-
-type AgentShared struct {
-	I_NodeIp string `desc:"IP address to advertise for node"`
-}
-
-func (a *Agent) Customize(command *cli.Command) {
-	command.Category = "CLUSTER RUNTIME"
-}

cli/cmd/agent/agent_k3s.go

@@ -1,81 +0,0 @@
-// +build k8s
-
-package agent
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/natefinch/lumberjack"
-	"github.com/rancher/norman/pkg/clientaccess"
-	"github.com/rancher/norman/pkg/resolvehome"
-	"github.com/rancher/rio/pkg/enterchroot"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-)
-
-func (a *Agent) Run(ctx *cli.Context) error {
-	if os.Getuid() != 0 {
-		return fmt.Errorf("agent must be ran as root")
-	}
-
-	if len(a.T_Token) == 0 {
-		return fmt.Errorf("--token is required")
-	}
-
-	if len(a.S_Server) == 0 {
-		return fmt.Errorf("--server is required")
-	}
-
-	dataDir, err := resolvehome.Resolve(a.D_DataDir)
-	if err != nil {
-		return err
-	}
-
-	return RunAgent(a.S_Server, a.T_Token, dataDir, a.L_Log, a.I_NodeIp)
-}
-
-func RunAgent(server, token, dataDir, logFile, ipAddress string) error {
-	dataDir = filepath.Join(dataDir, "agent")
-
-	for {
-		tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", server, token)
-		if err != nil {
-			logrus.Error(err)
-			time.Sleep(2 * time.Second)
-			continue
-		}
-		os.Remove(tmpFile)
-		break
-	}
-
-	os.Setenv("K3S_URL", server)
-	os.Setenv("K3S_TOKEN", token)
-	os.Setenv("K3S_DATA_DIR", dataDir)
-	os.Setenv("K3S_NODE_IP", ipAddress)
-
-	os.MkdirAll(dataDir, 0700)
-
-	stdout := io.Writer(os.Stdout)
-	stderr := io.Writer(os.Stderr)
-
-	if logFile == "" {
-		stdout = os.Stdout
-		stderr = os.Stderr
-	} else {
-		l := &lumberjack.Logger{
-			Filename:   logFile,
-			MaxSize:    50,
-			MaxBackups: 3,
-			MaxAge:     28,
-			Compress:   true,
-		}
-		stdout = l
-		stderr = l
-	}
-
-	return enterchroot.Mount(filepath.Join(dataDir, "root"), stdout, stderr, os.Args[1:])
-}

cli/cmd/agent/agent_none.go

@@ -1,17 +0,0 @@
-// +build !k8s
-
-package agent
-
-import (
-	"fmt"
-
-	"github.com/urfave/cli"
-)
-
-func (a *Agent) Run(ctx *cli.Context) error {
-	return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"")
-}
-
-func RunAgent(server, token, dataDir, logFile string) error {
-	return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"")
-}

cli/cmd/kubectl/kubectl.go

@@ -1,24 +0,0 @@
-package kubectl
-
-import (
-	"os"
-
-	"github.com/rancher/rio/pkg/kubectl"
-	"github.com/urfave/cli"
-)
-
-func NewKubectlCommand() cli.Command {
-	return cli.Command{
-		Name:            "kubectl",
-		Usage:           "Run kubectl",
-		SkipFlagParsing: true,
-		SkipArgReorder:  true,
-		Action:          run,
-	}
-}
-
-func run(ctx *cli.Context) error {
-	os.Args = append([]string{"kubectl"}, ctx.Args()...)
-	kubectl.Main()
-	return nil
-}

cli/cmd/server/server.go

@@ -1,128 +0,0 @@
-package server
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"os"
-	"path/filepath"
-
-	"github.com/docker/docker/pkg/reexec"
-	"github.com/natefinch/lumberjack"
-	"github.com/rancher/norman/signal"
-	"github.com/rancher/rio/pkg/server"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
-	"k8s.io/apimachinery/pkg/util/net"
-)
-
-var (
-	appName = filepath.Base(os.Args[0])
-
-	config server.Config
-	log    string
-)
-
-var ServerCommand = cli.Command{
-	Name:      "server",
-	Usage:     "Run management server",
-	UsageText: appName + " server [OPTIONS]",
-	Action:    Run,
-	Flags: []cli.Flag{
-		cli.IntFlag{
-			Name:        "https-listen-port",
-			Usage:       "HTTPS listen port",
-			Value:       6443,
-			Destination: &config.TLSConfig.HTTPSPort,
-		},
-		cli.IntFlag{
-			Name:        "http-listen-port",
-			Usage:       "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)",
-			Value:       0,
-			Destination: &config.TLSConfig.HTTPPort,
-		},
-		cli.StringFlag{
-			Name:        "data-dir",
-			Usage:       "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
-			Destination: &config.ControlConfig.DataDir,
-		},
-		//cli.StringFlag{
-		//	Name:        "advertise-address",
-		//	Usage:       "Address of the server to put in the generated kubeconfig",
-		//	Destination: &config.AdvertiseIP,
-		//},
-		cli.BoolFlag{
-			Name:        "disable-agent",
-			Usage:       "Do not run a local agent and register a local kubelet",
-			Destination: &config.DisableAgent,
-		},
-		cli.StringFlag{
-			Name:        "log",
-			Usage:       "Log to file",
-			Destination: &log,
-		},
-	},
-}
-
-func setupLogging(app *cli.Context) {
-	if !app.GlobalBool("debug") {
-		flag.Set("stderrthreshold", "3")
-		flag.Set("alsologtostderr", "false")
-		flag.Set("logtostderr", "false")
-	}
-}
-
-func runWithLogging(app *cli.Context) error {
-	l := &lumberjack.Logger{
-		Filename:   log,
-		MaxSize:    50,
-		MaxBackups: 3,
-		MaxAge:     28,
-		Compress:   true,
-	}
-
-	args := append([]string{"k3s"}, os.Args[1:]...)
-	cmd := reexec.Command(args...)
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, "_RIO_REEXEC_=true")
-	cmd.Stderr = l
-	cmd.Stdout = l
-	cmd.Stdin = os.Stdin
-	return cmd.Run()
-}
-
-func Run(app *cli.Context) error {
-	if log != "" && os.Getenv("_RIO_REEXEC_") == "" {
-		return runWithLogging(app)
-	}
-
-	setupLogging(app)
-
-	if !config.DisableAgent && os.Getuid() != 0 {
-		return fmt.Errorf("must run as root unless --disable-agent is specified")
-	}
-
-	if config.ControlConfig.NodeConfig.AgentConfig.NodeIP == "" {
-		ip, err := net.ChooseHostInterface()
-		if err == nil {
-			config.ControlConfig.NodeConfig.AgentConfig.NodeIP = ip.String()
-		}
-	}
-
-	logrus.Info("Starting k3s ", app.App.Version)
-	ctx := signal.SigTermCancelContext(context.Background())
-	if err := server.StartServer(ctx, &config); err != nil {
-		return err
-	}
-
-	if config.DisableAgent {
-		<-ctx.Done()
-		return nil
-	}
-
-	return nil
-	//logFile := filepath.Join(serverConfig.DataDir, "agent/agent.log")
-	//url := fmt.Sprintf("https://localhost:%d", httpsListenPort)
-	//logrus.Infof("Agent starting, logging to %s", logFile)
-	//return agent.RunAgent(url, server2.FormatToken(serverConfig.Runtime.NodeToken), serverConfig.DataDir, logFile, "")
-}

cli/pkg/builder/builder.go

@@ -1,151 +0,0 @@
-package builder
-
-import (
-	"fmt"
-	"reflect"
-	"regexp"
-	"strconv"
-	"strings"
-	"unsafe"
-
-	"github.com/urfave/cli"
-)
-
-var (
-	caseRegexp = regexp.MustCompile("([a-z])([A-Z])")
-)
-
-type clirunnable interface {
-	Run(app *cli.Context) error
-}
-
-type customizer interface {
-	Customize(cmd *cli.Command)
-}
-
-type fieldInfo struct {
-	FieldType  reflect.StructField
-	FieldValue reflect.Value
-}
-
-func fields(obj interface{}) []fieldInfo {
-	ptrValue := reflect.ValueOf(obj)
-	objValue := ptrValue.Elem()
-
-	var result []fieldInfo
-
-	for i := 0; i < objValue.NumField(); i++ {
-		fieldType := objValue.Type().Field(i)
-		if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct {
-			result = append(result, fields(objValue.Field(i).Addr().Interface())...)
-		} else if !fieldType.Anonymous {
-			result = append(result, fieldInfo{
-				FieldValue: objValue.Field(i),
-				FieldType:  objValue.Type().Field(i),
-			})
-		}
-	}
-
-	return result
-}
-
-func Command(obj interface{}, usage, usageText, description string) cli.Command {
-	slices := map[string]reflect.Value{}
-	maps := map[string]reflect.Value{}
-	ptrValue := reflect.ValueOf(obj)
-	objValue := ptrValue.Elem()
-
-	c := cli.Command{
-		Name:                   strings.ToLower(strings.Replace(objValue.Type().Name(), "Command", "", 1)),
-		Usage:                  usage,
-		UsageText:              usageText,
-		Description:            description,
-		UseShortOptionHandling: true,
-		SkipArgReorder:         true,
-	}
-
-	for _, info := range fields(obj) {
-		defMessage := ""
-		fieldType := info.FieldType
-		v := info.FieldValue
-
-		switch fieldType.Type.Kind() {
-		case reflect.Int:
-			flag := cli.IntFlag{
-				Name:        name(fieldType.Name),
-				Usage:       fieldType.Tag.Get("desc"),
-				EnvVar:      fieldType.Tag.Get("env"),
-				Destination: (*int)(unsafe.Pointer(v.Addr().Pointer())),
-			}
-			defValue := fieldType.Tag.Get("default")
-			if defValue != "" {
-				n, err := strconv.Atoi(defValue)
-				if err != nil {
-					panic("bad default " + defValue + " on field " + fieldType.Name)
-				}
-				flag.Value = n
-			}
-			c.Flags = append(c.Flags, flag)
-		case reflect.String:
-			flag := cli.StringFlag{
-				Name:        name(fieldType.Name),
-				Usage:       fieldType.Tag.Get("desc"),
-				Value:       fieldType.Tag.Get("default"),
-				EnvVar:      fieldType.Tag.Get("env"),
-				Destination: (*string)(unsafe.Pointer(v.Addr().Pointer())),
-			}
-			c.Flags = append(c.Flags, flag)
-		case reflect.Slice:
-			slices[name(fieldType.Name)] = v
-			defMessage = " "
-			fallthrough
-		case reflect.Map:
-			if defMessage == "" {
-				maps[name(fieldType.Name)] = v
-				defMessage = " "
-			}
-			flag := cli.StringSliceFlag{
-				Name:   name(fieldType.Name),
-				Usage:  fieldType.Tag.Get("desc") + defMessage,
-				EnvVar: fieldType.Tag.Get("env"),
-				Value:  &cli.StringSlice{},
-			}
-			c.Flags = append(c.Flags, flag)
-		case reflect.Bool:
-			flag := cli.BoolFlag{
-				Name:        name(fieldType.Name),
-				Usage:       fieldType.Tag.Get("desc"),
-				EnvVar:      fieldType.Tag.Get("env"),
-				Destination: (*bool)(unsafe.Pointer(v.Addr().Pointer())),
-			}
-			c.Flags = append(c.Flags, flag)
-		default:
-			panic("Unknown kind on field " + fieldType.Name + " on " + objValue.Type().Name())
-		}
-	}
-
-	if run, ok := obj.(clirunnable); ok {
-		c.Action = run.Run
-	} else {
-		panic(fmt.Sprintf("failed to find Action function for %T", obj))
-	}
-
-	cust, ok := obj.(customizer)
-	if ok {
-		cust.Customize(&c)
-	}
-
-	return c
-}
-
-func name(name string) string {
-	parts := strings.Split(name, "_")
-	i := len(parts) - 1
-	name = caseRegexp.ReplaceAllString(parts[i], "$1-$2")
-	name = strings.ToLower(name)
-	result := append([]string{name}, parts[0:i]...)
-	for i := 0; i < len(result); i++ {
-		result[i] = strings.ToLower(result[i])
-	}
-	return strings.Join(result, ",")
-}

cmd/agent/main.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"os"
+
+	"github.com/rancher/k3s/pkg/cli/agent"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewAgentCommand(agent.Run),
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}

cmd/cni/main.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	hostlocal "github.com/containernetworking/plugins/plugins/ipam/host-local"
+	"github.com/containernetworking/plugins/plugins/main/bridge"
+	"github.com/containernetworking/plugins/plugins/main/loopback"
+	"github.com/containernetworking/plugins/plugins/meta/flannel"
+	"github.com/containernetworking/plugins/plugins/meta/portmap"
+	"github.com/docker/docker/pkg/reexec"
+)
+
+func main() {
+	os.Args[0] = filepath.Base(os.Args[0])
+	reexec.Register("host-local", hostlocal.Main)
+	reexec.Register("bridge", bridge.Main)
+	reexec.Register("flannel", flannel.Main)
+	reexec.Register("loopback", loopback.Main)
+	reexec.Register("portmap", portmap.Main)
+	reexec.Init()
+}

cmd/containerd/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+	"github.com/rancher/k3s/pkg/containerd"
+	"k8s.io/klog"
+)
+
+func main() {
+	klog.InitFlags(nil)
+	containerd.Main()
+}

cmd/k3s/main.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/data"
+	"github.com/rancher/k3s/pkg/datadir"
+	"github.com/rancher/k3s/pkg/untar"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func main() {
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewServerCommand(wrap("k3s-server", os.Args)),
+		cmds.NewAgentCommand(wrap("k3s-agent", os.Args)),
+		cmds.NewKubectlCommand(kubectl),
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func kubectl(ctx *cli.Context) error {
+	return stageAndRun(ctx, "kubectl", append([]string{"kubectl"}, ctx.Args()...))
+}
+
+func wrap(cmd string, args []string) func(ctx *cli.Context) error {
+	return func(ctx *cli.Context) error {
+		return stageAndRun(ctx, cmd, args)
+	}
+}
+
+func stageAndRun(cli *cli.Context, cmd string, args []string) error {
+	dataDir, err := datadir.Resolve(cli.String("data-dir"))
+	if err != nil {
+		return err
+	}
+
+	asset, dir := getAssetAndDir(dataDir)
+
+	if err := extract(asset, dir); err != nil {
+		return errors.Wrap(err, "extracting data")
+	}
+
+	if err := os.Setenv("PATH", filepath.Join(dir, "bin")+":"+os.Getenv("PATH")); err != nil {
+		return err
+	}
+
+	cmd, err = exec.LookPath(cmd)
+	if err != nil {
+		return err
+	}
+
+	return syscall.Exec(cmd, args, os.Environ())
+}
+
+func getAssetAndDir(dataDir string) (string, string) {
+	asset := data.AssetNames()[0]
+	dir := filepath.Join(dataDir, "data", strings.SplitN(filepath.Base(asset), ".", 2)[0])
+	return asset, dir
+}
+
+func extract(asset, dir string) error {
+	logrus.Infof("Asset dir %s", dir)
+
+	if _, err := os.Stat(dir); err == nil {
+		return nil
+	}
+
+	logrus.Infof("Staging to dir %s", dir)
+
+	content, err := data.Asset(asset)
+	if err != nil {
+		return err
+	}
+	buf := bytes.NewBuffer(content)
+
+	tempDest := dir + "-tmp"
+	defer os.RemoveAll(tempDest)
+
+	os.RemoveAll(tempDest)
+
+	if err := untar.Untar(buf, tempDest); err != nil {
+		return err
+	}
+
+	return os.Rename(tempDest, dir)
+}

cmd/kubectl/main.go

@@ -0,0 +1,7 @@
+package main
+
+import "github.com/rancher/k3s/pkg/kubectl"
+
+func main() {
+	kubectl.Main()
+}

cmd/server/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/rancher/k3s/pkg/cli/agent"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/cli/kubectl"
+	"github.com/rancher/k3s/pkg/cli/server"
+	"github.com/rancher/k3s/pkg/containerd"
+	kubectl2 "github.com/rancher/k3s/pkg/kubectl"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func init() {
+	reexec.Register("containerd", containerd.Main)
+	reexec.Register("kubectl", kubectl2.Main)
+}
+
+func main() {
+	cmd := os.Args[0]
+	os.Args[0] = filepath.Base(os.Args[0])
+	if reexec.Init() {
+		return
+	}
+	os.Args[0] = cmd
+
+	app := cmds.NewApp()
+	app.Commands = []cli.Command{
+		cmds.NewServerCommand(server.Run),
+		cmds.NewAgentCommand(agent.Run),
+		cmds.NewKubectlCommand(kubectl.Run),
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}

image/Dockerfile

@@ -1,179 +0,0 @@
-### BASE ###
-FROM alpine:3.8 as base
-RUN apk -U add findutils iptables ipset bash ca-certificates jq iproute2 nfs-utils coreutils libseccomp conntrack-tools
-# RUN apk -U add eudev tinyssh e2fsprogs mdadm rsync nfs-utils parted
-
-FROM base as k3s-build
-COPY --from=base /bin /usr/src/image/bin/
-COPY --from=base /lib /usr/src/image/lib/
-COPY --from=base /sbin /usr/src/image/sbin/
-COPY --from=base /etc/ssl/certs/ca-certificates.crt /usr/src/image/etc/ssl/certs/ca-certificates.crt
-COPY --from=base /etc/terminfo /usr/src/image/etc/terminfo
-COPY --from=base /usr /usr/src/image/usr/
-
-WORKDIR /usr/src/image
-
-RUN rm -rf  usr/bin/iconv \
-            usr/bin/scanelf \
-            usr/bin/ssl_client \
-            usr/bin/pkgconf \
-            usr/bin/getent \
-            usr/bin/locate \
-            usr/bin/updatedb \
-            usr/bin/c_rehash \
-            usr/bin/getconf \
-            usr/etc \
-            usr/include \
-            usr/lib/bash \
-            usr/lib/krb5 \
-            usr/lib/pkgconfig \
-            usr/lib/tc \
-            usr/libexec \
-            usr/local \
-            usr/sbin/nfsiostat \
-            usr/sbin/rpc.gssd \
-            usr/sbin/nfsidmap \
-            usr/sbin/blkmapd \
-            usr/sbin/conntrackd \
-            usr/sbin/nfct \
-            usr/sbin/nfsstat \
-            usr/sbin/mountstats \
-            usr/sbin/setcap \
-            usr/sbin/exportfs \
-            usr/sbin/update-ca-certificates \
-            usr/sbin/capsh \
-            usr/sbin/getcap \
-            usr/sbin/rpcdebug \
-            usr/sbin/start-statd \
-            usr/sbin/getpcaps \
-            usr/sbin/sm-notify \
-            usr/share/aclocal \
-            usr/share/apk \
-            usr/share/ca-certificates \
-            usr/share/man \
-            usr/share/misc && \
-            find usr/share/terminfo -type f -exec rm {} \; && \
-            ln -s xterm-color usr/share/terminfo/x/xterm-256color && \
-            rmdir usr/share/terminfo/* || true
-RUN rm -rf  bin/sh \
-            lib/apk \
-            lib/mdev \
-            sbin/ss \
-            sbin/routel \
-            sbin/*-compat* \
-            sbin/genl \
-            sbin/lnstat \
-            sbin/ifstat \
-            sbin/mkmntdirs \
-            sbin/nfsdcltrack \
-            sbin/rtacct \
-            sbin/nstat \
-            sbin/routef \
-            sbin/apk \
-            sbin/tc \
-            sbin/ifcfg \
-            sbin/setup-udev \
-            sbin/rtpr \
-            sbin/osd_login \
-            sbin/bridge \
-            sbin/rtmon && \
-    ln -s bash bin/sh && \
-    mkdir -p lib/modules
-
-RUN mv sbin/* bin/ && \
-    rmdir sbin && \
-    ln -s bin sbin
-
-RUN mkdir lib2 && \
-    mv usr/lib/* lib2/ && \
-    mv lib2/* lib/ && \
-    mv usr/bin/* bin/ && \
-    mv usr/sbin/* bin/ && \
-    mv usr/share . && \
-    rm -rf usr lib2 && \
-    for i in $(ls -l bin | grep usr/bin/coreutils | awk '{print $(NF-2)}'); do \
-        rm bin/$i && ln -s coreutils bin/$i \
-    ;done && \
-    find -L bin -type l -exec rm {} \; -print
-
-RUN apk add upx && \
-    upx $(find bin -type f -executable \! -name coreutils) || true
-
-RUN echo '#### LAYOUT #####' && \
-    find -type d && \
-    echo '#### BIN #####' && \
-    find bin -type f -executable && \
-    du -x -s -h
-
-RUN tar cf ../rootfs.tar * && \
-    ls -la ../rootfs.tar
-
-CMD ["sh"]
-
-### BUILD IMAGE ###
-FROM golang:1.11-alpine AS gobuild
-RUN apk -U add git gcc linux-headers musl-dev make libseccomp libseccomp-dev bash
-RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
-
-### CNI ###
-FROM gobuild AS cni
-RUN mkdir -p $GOPATH/src/github.com/containernetworking && \
-    cd $GOPATH/src/github.com/containernetworking && \
-    git clone https://github.com/ibuildthecloud/plugins.git && \
-    cd plugins && \
-    git checkout 9810b7d5137b171c4e07ce59bb18be9feccec557
-RUN go build -buildmode=pie -ldflags -s -o /usr/bin/cni github.com/containernetworking/plugins
-
-### RUNC ###
-FROM gobuild AS runc
-RUN go get -d github.com/opencontainers/runc && \
-    git -C $GOPATH/src/github.com/opencontainers/runc checkout -b build v1.0.0-rc5
-WORKDIR $GOPATH/src/github.com/opencontainers/runc
-RUN make runc && \
-    cp runc /usr/bin/
-
-### CONTAINERD ###
-FROM gobuild AS containerd
-RUN go get -d github.com/containerd/containerd && \
-    git -C $GOPATH/src/github.com/containerd/containerd checkout -b build v1.1.4
-WORKDIR $GOPATH/src/github.com/containerd/containerd
-RUN sed -i -e '/aufs/d' -e '/zfs/d' cmd/containerd/builtins_linux.go
-RUN make BUILDTAGS="apparmor seccomp no_btrfs netgo osusergo" bin/containerd bin/containerd-shim && \
-    cp bin/containerd bin/containerd-shim /usr/bin/
-
-### AGENT ###
-FROM gobuild AS agent
-ADD /build/vendor.tar $GOPATH/src/github.com/rancher/rio/
-COPY /agent $GOPATH/src/github.com/rancher/rio/agent
-WORKDIR $GOPATH/src/github.com/rancher/rio
-RUN go build -buildmode=pie -tags k3s -ldflags -s -o /usr/bin/agent ./agent
-
-### ASSEMBLE IMAGE ###
-FROM gobuild
-
-RUN apk add -U squashfs-tools
-
-COPY --from=k3s-build /usr/src/rootfs.tar /usr/src/rootfs.tar
-RUN mkdir /usr/src/image && \
-    tar xf /usr/src/rootfs.tar -C /usr/src/image
-
-COPY --from=runc /usr/bin/runc /usr/src/image/bin/runc
-RUN strip --strip-unneeded /usr/src/image/bin/runc
-
-COPY --from=agent /usr/bin/agent /usr/src/image/bin/
-RUN strip --strip-unneeded /usr/src/image/bin/agent
-
-COPY --from=containerd /usr/bin/containerd-shim /usr/bin/containerd /usr/src/image/bin/
-#COPY containerd /usr/src/image/bin
-RUN strip --strip-unneeded /usr/src/image/bin/containerd /usr/src/image/bin/containerd-shim
-
-RUN mkdir -p /usr/src/image/share/cni/bin
-COPY --from=cni /usr/bin/cni /usr/src/image/share/cni/bin
-RUN cd /usr/src/image/share/cni/bin && \
-    for i in ./bridge ./flannel ./host-local ./loopback ./portmap; do \
-        ln -s cni $i \
-    ;done
-
-COPY image/init /usr/src/image/init
-
-RUN mksquashfs /usr/src/image main.squashfs

image/build

@@ -1,19 +0,0 @@
-#!/bin/bash
-set -e
-
-cd $(dirname $0)
-rm -rf main.squashfs
-
-pushd ..
-mkdir -p build
-echo Copying vendor
-tar chf build/vendor.tar \
-    --exclude vendor/github.com/coreos/etcd/cmd \
-    --exclude vendor/github.com/jteeuwen/go-bindata/testdata \
-    --exclude vendor/github.com/karrick/godirwalk/testdata \
-    vendor
-echo Running agent docker build
-docker build -t bb -f image/Dockerfile .
-popd
-
-docker run --rm bb cat main.squashfs > main.squashfs

image/init

@@ -1,147 +0,0 @@
-#!/usr/bin/sh
-set -e
-
-if [ "$ENTER_DEBUG" = true ]; then
-    set -x
-fi
-
-layout()
-{
-    mount --make-rshared /
-
-    mkdir -p /proc
-    mount -t proc -o nodev,nosuid,noexec,relatime none /proc
-
-    for i in cache empty lib local local log opt spool tmp; do
-        mkdir -p /var/$i
-    done
-
-    for i in run dev home mnt media opt root lib/modules lib/firmware var/lib/docker; do
-        if [ -d /.root/$i ]; then
-            mkdir -p /$i
-            mount --rbind /.root/$i /$i
-        fi
-    done
-
-    if [ -L /.root/var/run ]; then
-        ln -sf /run /var/run
-    else
-        mkdir -p /var/run
-        mount --rbind /.root/var/run /var/run
-    fi
-
-    mkdir -p $K3S_DATA_DIR
-    mount --rbind /.root/$K3S_DATA_DIR /$K3S_DATA_DIR
-
-    mkdir -p /run/k3s/containerd
-}
-
-sysfs()
-{
-    mkdir -p /sys
-    mount -t sysfs none /sys
-
-    mount -t securityfs -o noexec,nosuid,nodev none /sys/kernel/security 2>/dev/null|| true
-    mount -t configfs -o noexec,nosuid,nodev none /sys/kernel/config 2>/dev/null || true
-    mount -t fusectl -o noexec,nosuid,nodev none /sys/fs/fuse/connections 2>/dev/null || true
-    mount -t binfmt_misc -o noexec,nosuid,nodev none /proc/sys/fs/binfmt_misc 2>/dev/null || true
-}
-
-cgroups()
-{
-    mount -t tmpfs -o mode=755,size=10m none /sys/fs/cgroup
-    cat /proc/cgroups > /tmp/cgroups
-
-    for i in $(seq 0 20); do
-        t=""
-        l="$(cat /tmp/cgroups | grep '1$' | awk '{print $1 " " $2}' | grep -w $i | awk '{print $1}')"
-        for j in $l; do
-            if [ -z "$t" ]; then
-                t=$j
-            else
-                t="${t},$j"
-            fi
-        done
-
-        if [ -z "$t" ]; then
-            continue
-        fi
-
-        mkdir -p /sys/fs/cgroup/${t}
-        mount -t cgroup -o ${t},noexec,nosuid,nodev none /sys/fs/cgroup/${t}
-        mkdir -p /sys/fs/cgroup/${t}/k3s
-
-        for j in $l; do
-            if [ $j != $t ]; then
-                ln -s $t /sys/fs/cgroup/$j
-            fi
-        done
-    done
-
-    # good ole systemd
-    mkdir -p /sys/fs/cgroup/systemd
-    mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
-    mkdir -p /sys/fs/cgroup/systemd/k3s
-
-    rm /tmp/cgroups
-}
-
-mketc()
-{
-    mkdir -p /etc
-    cp -rf usr/etc/* /etc/
-    for i in  /.root/usr/lib/os-release /.root/etc/os-release; do
-        if [ -e $i ]; then
-            cp -f $i /etc/os-release
-        fi
-    done
-
-    if [ -e /.root/etc/machine-id ]; then
-        cp -f /.root/etc/machine-id /etc/machine-id
-    fi
-
-    hostname > /etc/hostname
-
-    cat > /etc/hosts << EOF
-127.0.0.1	localhost $NODE_NAME $(hostname)
-::1		localhost ip6-localhost ip6-loopback
-ff02::1		ip6-allnodes
-ff02::2		ip6-allrouters
-EOF
-
-    cat > /etc/resolv.conf << EOF
-nameserver 1.1.1.1
-EOF
-}
-
-nodename()
-{
-    if [ ! -e $K3S_DATA_DIR/id ]; then
-        echo $RANDOM > $K3S_DATA_DIR/id
-    fi
-    export NODE_NAME="$(hostname | cut -f1 -d.)-$(<$K3S_DATA_DIR/id)"
-}
-
-layout
-nodename
-mketc
-sysfs
-cgroups
-
-umount -l .root
-rmdir .root
-
-if [ "$1" = "--" ]; then
-    shift 1
-    exec "$@"
-fi
-
-exec env -i -- \
-    HOME=/root \
-    NODE_NAME=$NODE_NAME \
-    PATH=/sbin:/bin \
-    K3S_DATA_DIR=$K3S_DATA_DIR \
-    K3S_NODE_IP=$K3S_NODE_IP \
-    K3S_TOKEN=$K3S_TOKEN \
-    K3S_URL=$K3S_URL \
-    agent

main.go

@@ -4,68 +4,25 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"path/filepath"
 
-	"github.com/docker/docker/pkg/reexec"
+	"github.com/rancher/k3s/pkg/cli/agent"
-	"github.com/rancher/rio/cli/cmd/agent"
+	"github.com/rancher/k3s/pkg/cli/cmds"
-	"github.com/rancher/rio/cli/cmd/kubectl"
+	"github.com/rancher/k3s/pkg/cli/kubectl"
-	"github.com/rancher/rio/cli/cmd/server"
+	"github.com/rancher/k3s/pkg/cli/server"
-	"github.com/rancher/rio/cli/pkg/builder"
-	"github.com/rancher/rio/version"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
-
-	_ "github.com/rancher/rio/pkg/kubectl"
-)
-
-var (
-	appName = filepath.Base(os.Args[0])
-	debug   bool
 )
 
 func main() {
-	old := os.Args[0]
+	app := cmds.NewApp()
-	os.Args[0] = filepath.Base(os.Args[0])
-	if reexec.Init() {
-		return
-	}
-	os.Args[0] = old
-
-	app := cli.NewApp()
-	app.Name = appName
-	app.Usage = "Kubernetes, but small and simple"
-	app.Version = version.Version
-	cli.VersionPrinter = func(c *cli.Context) {
-		fmt.Printf("%s version %s\n", app.Name, app.Version)
-	}
-	app.Flags = []cli.Flag{
-		cli.BoolFlag{
-			Name:        "debug",
-			Usage:       "Turn on debug logs",
-			Destination: &debug,
-		},
-	}
-
 	app.Commands = []cli.Command{
-		server.ServerCommand,
+		cmds.NewServerCommand(server.Run),
-		builder.Command(&agent.Agent{},
+		cmds.NewAgentCommand(agent.Run),
-			"Run node agent",
+		cmds.NewKubectlCommand(kubectl.Run),
-			appName+" agent [OPTIONS]",
-			""),
-
-		kubectl.NewKubectlCommand(),
-	}
-	app.Before = func(ctx *cli.Context) error {
-		if debug {
-			logrus.SetLevel(logrus.DebugLevel)
-		}
-		return nil
 	}
 
-	err := app.Run(os.Args)
+	if err := app.Run(os.Args); err != nil {
-	if err != nil {
 		logrus.Fatal(err)
 	}
 }

package/Dockerfile

@@ -1,5 +1,16 @@
-FROM alpine:3.8
+FROM alpine:3.8 as base
-RUN apk -U --no-cache add ca-certificates
+RUN apk add -U ca-certificates
+ADD data.tar.gz /image
+RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \
+    cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
+RUN cd image/bin && \
+    rm -f k3s && \
+    ln -s k3s-server k3s
 
-COPY rio /
+FROM scratch
-WORKDIR /var/lib/rancher/rio
+COPY --from=base /image /
+VOLUME /var/lib/rancher/k3s 
+VOLUME /var/lib/cni
+VOLUME /var/log
+ENTRYPOINT ["/bin/k3s"]
+CMD ["agent"]

pkg/agent/config/config.go

@@ -1,79 +1,47 @@
 package config
 
 import (
-	"crypto/md5"
+	"context"
 	"crypto/tls"
-	"encoding/hex"
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
-	"strings"
 	"time"
 
+	"k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
+
 	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/norman/pkg/clientaccess"
-	"github.com/rancher/rio/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/json"
-	net2 "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/client-go/util/cert"
 )
 
-type envInfo struct {
+func Get(ctx context.Context, agent cmds.Agent) *config.Node {
-	ServerURL string
-	Token     string
-	DataDir   string
-	NodeIP    string
-	NodeName  string
-}
-
-func Get() *config.Node {
 	for {
-		agentConfig, err := get()
+		agentConfig, err := get(&agent)
 		if err != nil {
 			logrus.Error(err)
-			time.Sleep(5 * time.Second)
+			select {
-			continue
+			case <-time.After(5 * time.Second):
+				continue
+			case <-ctx.Done():
+				logrus.Fatalf("Interrupted")
+			}
 		}
 		return agentConfig
 	}
 }
 
-func getEnvInfo() (*envInfo, error) {
-	u := os.Getenv("K3S_URL")
-	if u == "" {
-		return nil, fmt.Errorf("K3S_URL env var is required")
-	}
-
-	_, err := url.Parse(u)
-	if err != nil {
-		return nil, fmt.Errorf("K3S_URL [%s] is invalid: %v", u, err)
-	}
-
-	t := os.Getenv("K3S_TOKEN")
-	if t == "" {
-		return nil, fmt.Errorf("K3S_TOKEN env var is required")
-	}
-
-	dataDir := os.Getenv("K3S_DATA_DIR")
-	if dataDir == "" {
-		return nil, fmt.Errorf("K3S_DATA_DIR is required")
-	}
-
-	return &envInfo{
-		ServerURL: u,
-		Token:     t,
-		DataDir:   dataDir,
-		NodeIP:    os.Getenv("K3S_NODE_IP"),
-		NodeName:  os.Getenv("NODE_NAME"),
-	}, nil
-}
-
 func getNodeCert(info *clientaccess.Info) (*tls.Certificate, error) {
-	nodeCert, err := clientaccess.Get("/v1-k3s/node.cert", info)
+	nodeCert, err := clientaccess.Get("/v1-k3s/node.crt", info)
 	if err != nil {
 		return nil, err
 	}
@@ -104,10 +72,10 @@ func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
 	return clientCA, nil
 }
 
-func getHostnameAndIP(info envInfo) (string, string, error) {
+func getHostnameAndIP(info cmds.Agent) (string, string, error) {
 	ip := info.NodeIP
 	if ip == "" {
-		hostIP, err := net2.ChooseHostInterface()
+		hostIP, err := net.ChooseHostInterface()
 		if err != nil {
 			return "", "", err
 		}
@@ -120,10 +88,7 @@ func getHostnameAndIP(info envInfo) (string, string, error) {
 		if err != nil {
 			return "", "", err
 		}
-		hostname = strings.Split(hostname, ".")[0]
+		name = hostname
-
-		d := md5.Sum([]byte(ip))
-		name = hostname + "-" + hex.EncodeToString(d[:])[:8]
 	}
 
 	return name, ip, nil
@@ -133,7 +98,7 @@ func localAddress(controlConfig *config.Control) string {
 	return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
 }
 
-func writeKubeConfig(envInfo *envInfo, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
+func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
 	os.MkdirAll(envInfo.DataDir, 0700)
 	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
 
@@ -146,10 +111,9 @@ func writeKubeConfig(envInfo *envInfo, info clientaccess.Info, controlConfig *co
 	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
 }
 
-func get() (*config.Node, error) {
+func get(envInfo *cmds.Agent) (*config.Node, error) {
-	envInfo, err := getEnvInfo()
+	if envInfo.Debug {
-	if err != nil {
+		logrus.SetLevel(logrus.DebugLevel)
-		return nil, err
 	}
 
 	serverURLParsed, err := url.Parse(envInfo.ServerURL)
@@ -187,20 +151,45 @@ func get() (*config.Node, error) {
 		return nil, err
 	}
 
-	nodeConfig := &controlConfig.NodeConfig
+	hostLocal, err := exec.LookPath("host-local")
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find host-local")
+	}
+
+	nodeConfig := &config.Node{
+		Docker:    envInfo.Docker,
+		NoFlannel: envInfo.NoFlannel,
+	}
 	nodeConfig.LocalAddress = localAddress(controlConfig)
-	nodeConfig.AgentConfig.NodeIP = defString(nodeConfig.AgentConfig.NodeIP, nodeIP)
+	nodeConfig.AgentConfig.NodeIP = nodeIP
-	nodeConfig.AgentConfig.NodeName = defString(nodeConfig.AgentConfig.NodeName, nodeName)
+	nodeConfig.AgentConfig.NodeName = nodeName
-	nodeConfig.AgentConfig.CNIBinDir = defString(nodeConfig.AgentConfig.CNIBinDir, "/usr/share/cni")
+	nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS
 	nodeConfig.AgentConfig.CACertPath = clientCA
-	nodeConfig.AgentConfig.ListenAddress = defString(nodeConfig.AgentConfig.ListenAddress, "127.0.0.1")
+	nodeConfig.AgentConfig.ListenAddress = "127.0.0.1"
 	nodeConfig.AgentConfig.KubeConfig = kubeConfig
+	nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet")
 	nodeConfig.CACerts = info.CACerts
+	nodeConfig.Containerd.Config = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml")
+	nodeConfig.Containerd.Root = filepath.Join(envInfo.DataDir, "containerd")
+	nodeConfig.Containerd.Opt = filepath.Join(envInfo.DataDir, "containerd")
+	nodeConfig.Containerd.State = "/run/k3s/containerd"
+	nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock")
 	nodeConfig.ServerAddress = serverURLParsed.Host
 	nodeConfig.Certificate = nodeCert
-	if !nodeConfig.Docker {
+	if !nodeConfig.NoFlannel {
-		nodeConfig.AgentConfig.RuntimeSocket = "/run/k3s/containerd.sock"
+		nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json")
+		nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal)
+		nodeConfig.AgentConfig.CNIConfDir = filepath.Join(envInfo.DataDir, "etc/cni/net.d")
 	}
+	if !nodeConfig.Docker {
+		nodeConfig.AgentConfig.RuntimeSocket = "unix://" + nodeConfig.Containerd.Address
+	}
+	if controlConfig.ClusterIPRange != nil {
+		nodeConfig.AgentConfig.ClusterCIDR = *controlConfig.ClusterIPRange
+	}
+
+	os.Setenv("NODE_NAME", nodeConfig.AgentConfig.NodeName)
+	v1beta1.KubeletSocket = filepath.Join(envInfo.DataDir, "kubelet/device-plugins/kubelet.sock")
 
 	return nodeConfig, nil
 }

pkg/agent/containerd/containerd.go

@@ -9,8 +9,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/rancher/rio/agent/config"
+	util2 "github.com/rancher/k3s/pkg/agent/util"
-	util2 "github.com/rancher/rio/agent/util"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
@@ -18,34 +18,50 @@ import (
 )
 
 const (
-	address    = "/run/k3s/containerd.sock"
 	maxMsgSize = 1024 * 1024 * 16
-	configToml = `[plugins.cri]
+	configToml = `
+[plugins.opt]
+	path = "%OPT%"
+[plugins.cri]
   stream_server_address = "%NODE%"
   stream_server_port = "10010"
+`
+	configCNIToml = `
   [plugins.cri.cni]
-    bin_dir = "/usr/share/cni/bin"
+    bin_dir = "%CNIBIN%"
-    conf_dir = "/etc/cni/net.d"
+    conf_dir = "%CNICFG%"
 `
 )
 
-func Run(ctx context.Context, config *config.NodeConfig) error {
+func Run(ctx context.Context, cfg *config.Node) error {
 	args := []string{
 		"containerd",
-		"-a", address,
+		"-c", cfg.Containerd.Config,
-		"--state", "/run/k3s/containerd",
+		"-a", cfg.Containerd.Address,
+		"--state", cfg.Containerd.State,
+		"--root", cfg.Containerd.Root,
 	}
 
-	if err := util2.WriteFile("/etc/containerd/config.toml",
+	template := configToml
-		strings.Replace(configToml, "%NODE%", config.AgentConfig.NodeName, -1)); err != nil {
+	if !cfg.NoFlannel {
+		template += configCNIToml
+	}
+
+	template = strings.Replace(template, "%OPT%", cfg.Containerd.Opt, -1)
+	template = strings.Replace(template, "%CNIBIN%", cfg.AgentConfig.CNIBinDir, -1)
+	template = strings.Replace(template, "%CNICFG%", cfg.AgentConfig.CNIConfDir, -1)
+	template = strings.Replace(template, "%NODE%", cfg.AgentConfig.NodeName, -1)
+
+	if err := util2.WriteFile(cfg.Containerd.Config, template); err != nil {
 		return err
 	}
 
 	if logrus.GetLevel() >= logrus.DebugLevel {
-		args = append(args, "--verbose")
+		args = append(args, "-l", "debug")
 	}
 
 	go func() {
+		logrus.Infof("Running containerd %s", config.ArgString(args[1:]))
 		cmd := exec.Command(args[0], args[1:]...)
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
@@ -59,7 +75,7 @@ func Run(ctx context.Context, config *config.NodeConfig) error {
 	}()
 
 	for {
-		addr, dailer, err := util.GetAddressAndDialer("unix://" + address)
+		addr, dailer, err := util.GetAddressAndDialer("unix://" + cfg.Containerd.Address)
 		if err != nil {
 			time.Sleep(1 * time.Second)
 			continue
@@ -81,8 +97,12 @@ func Run(ctx context.Context, config *config.NodeConfig) error {
 			break
 		}
 		conn.Close()
-		logrus.Infof("Waiting for containerd startup")
+		logrus.Infof("Waiting for containerd startup: %v", err)
-		time.Sleep(1 * time.Second)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second):
+		}
 	}
 
 	return nil

pkg/agent/flannel/flannel.go

@@ -1,5 +1,5 @@
-// Copyright 2015 flannel authors
 //
+// Copyright 2015 flannel authors
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -36,13 +36,13 @@ const (
 	subnetFile = "/run/flannel/subnet.env"
 )
 
-func flannel(ctx context.Context, kubeConfigFile string) error {
+func flannel(ctx context.Context, flannelConf, kubeConfigFile string) error {
 	extIface, err := LookupExtIface()
 	if err != nil {
 		return err
 	}
 
-	sm, err := kube.NewSubnetManager("", kubeConfigFile)
+	sm, err := kube.NewSubnetManager("", flannelConf, kubeConfigFile, "flannel.alpha.coreos.com")
 	if err != nil {
 		return err
 	}

pkg/agent/flannel/setup.go

@@ -2,12 +2,12 @@ package flannel
 
 import (
 	"context"
+	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/rancher/rio/agent/util"
+	"github.com/rancher/k3s/pkg/agent/util"
-
+	"github.com/rancher/k3s/pkg/daemons/config"
-	"github.com/rancher/rio/agent/config"
 	"github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
@@ -44,7 +44,15 @@ const (
 `
 )
 
-func Run(ctx context.Context, config *config.NodeConfig) error {
+func Prepare(ctx context.Context, config *config.Node) error {
+	if err := createCNIConf(config.AgentConfig.CNIConfDir); err != nil {
+		return err
+	}
+
+	return createFlannelConf(config)
+}
+
+func Run(ctx context.Context, config *config.Node) error {
 	nodeName := config.AgentConfig.NodeName
 
 	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
@@ -70,22 +78,26 @@ func Run(ctx context.Context, config *config.NodeConfig) error {
 		time.Sleep(2 * time.Second)
 	}
 
-	if err := createCNIConf(); err != nil {
+	go func() {
-		return err
+		err := flannel(ctx, config.FlannelConf, config.AgentConfig.KubeConfig)
+		logrus.Fatalf("fannel exited: %v", err)
+	}()
+
+	return err
+}
+
+func createCNIConf(dir string) error {
+	if dir == "" {
+		return nil
 	}
+	p := filepath.Join(dir, "10-flannel.conflist")
+	return util.WriteFile(p, cniConf)
+}
 
-	if err := createFlannelConf(config); err != nil {
+func createFlannelConf(config *config.Node) error {
-		return err
+	if config.FlannelConf == "" {
+		return nil
 	}
-
+	return util.WriteFile(config.FlannelConf,
-	return flannel(ctx, config.AgentConfig.KubeConfig)
+		strings.Replace(netJson, "%CIDR%", config.AgentConfig.ClusterCIDR.String(), -1))
-}
-
-func createCNIConf() error {
-	return util.WriteFile("/etc/cni/net.d/10-flannel.conflist", cniConf)
-}
-
-func createFlannelConf(config *config.NodeConfig) error {
-	return util.WriteFile("/etc/kube-flannel/net-conf.json",
-		strings.Replace(netJson, "%CIDR", config.AgentConfig.ClusterCIDR.String(), -1))
 }

pkg/agent/proxy/proxy.go

@@ -5,12 +5,12 @@ import (
 	"net/http"
 
 	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/norman/pkg/proxy"
-	"github.com/rancher/rio/agent/config"
 	"github.com/sirupsen/logrus"
 )
 
-func Run(config *config.NodeConfig) error {
+func Run(config *config.Node) error {
 	proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
 	if err != nil {
 		return err

pkg/agent/run.go

@@ -0,0 +1,85 @@
+package agent
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/rancher/k3s/pkg/agent/config"
+	"github.com/rancher/k3s/pkg/agent/containerd"
+	"github.com/rancher/k3s/pkg/agent/flannel"
+	"github.com/rancher/k3s/pkg/agent/proxy"
+	"github.com/rancher/k3s/pkg/agent/syssetup"
+	"github.com/rancher/k3s/pkg/agent/tunnel"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/daemons/agent"
+	"github.com/rancher/norman/pkg/clientaccess"
+)
+
+func run(ctx context.Context, cfg cmds.Agent) error {
+	nodeConfig := config.Get(ctx, cfg)
+
+	if !nodeConfig.NoFlannel {
+		if err := flannel.Prepare(ctx, nodeConfig); err != nil {
+			return err
+		}
+	}
+
+	if nodeConfig.Docker {
+		nodeConfig.AgentConfig.RuntimeSocket = ""
+	} else {
+		if err := containerd.Run(ctx, nodeConfig); err != nil {
+			return err
+		}
+	}
+
+	if err := syssetup.Configure(); err != nil {
+		return err
+	}
+
+	if err := tunnel.Setup(nodeConfig); err != nil {
+		return err
+	}
+
+	if err := proxy.Run(nodeConfig); err != nil {
+		return err
+	}
+
+	if err := agent.Agent(&nodeConfig.AgentConfig); err != nil {
+		return err
+	}
+
+	if !nodeConfig.NoFlannel {
+		if err := flannel.Run(ctx, nodeConfig); err != nil {
+			return err
+		}
+	}
+
+	<-ctx.Done()
+	return ctx.Err()
+}
+
+func Run(ctx context.Context, cfg cmds.Agent) error {
+	cfg.DataDir = filepath.Join(cfg.DataDir, "agent")
+
+	for {
+		tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", cfg.ServerURL, cfg.Token)
+		if err != nil {
+			logrus.Error(err)
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			case <-time.After(2 * time.Second):
+			}
+			continue
+		}
+		os.Remove(tmpFile)
+		break
+	}
+
+	os.MkdirAll(cfg.DataDir, 0700)
+	return run(ctx, cfg)
+}

pkg/agent/tunnel/tunnel.go

@@ -12,8 +12,8 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/norman/pkg/remotedialer"
-	"github.com/rancher/rio/agent/config"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/tools/clientcmd"
 )
@@ -25,7 +25,7 @@ var (
 	}
 )
 
-func Setup(config *config.NodeConfig) error {
+func Setup(config *config.Node) error {
 	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
 	if err != nil {
 		return err
@@ -36,7 +36,7 @@ func Setup(config *config.NodeConfig) error {
 		return err
 	}
 
-	wsURL := fmt.Sprintf("wss://%s/v1/connect", config.ServerAddress)
+	wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", config.ServerAddress)
 	headers := map[string][]string{
 		"X-K3s-NodeName": {config.AgentConfig.NodeName},
 	}

pkg/agent/util/file.go

@@ -9,7 +9,7 @@ import (
 )
 
 func WriteFile(name string, content string) error {
-	os.Mkdir(filepath.Dir(name), 0755)
+	os.MkdirAll(filepath.Dir(name), 0755)
 	err := ioutil.WriteFile(name, []byte(content), 0644)
 	if err != nil {
 		return errors.Wrapf(err, "writing %s", name)

pkg/cli/agent/agent.go

@@ -0,0 +1,38 @@
+package agent
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/rancher/k3s/pkg/agent"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/norman/pkg/resolvehome"
+	"github.com/rancher/norman/signal"
+	"github.com/urfave/cli"
+)
+
+func Run(ctx *cli.Context) error {
+	if os.Getuid() != 0 {
+		return fmt.Errorf("agent must be ran as root")
+	}
+
+	if cmds.AgentConfig.Token == "" {
+		return fmt.Errorf("--token is required")
+	}
+
+	if cmds.AgentConfig.ServerURL == "" {
+		return fmt.Errorf("--server is required")
+	}
+
+	dataDir, err := resolvehome.Resolve(cmds.AgentConfig.DataDir)
+	if err != nil {
+		return err
+	}
+
+	cfg := cmds.AgentConfig
+	cfg.DataDir = dataDir
+
+	contextCtx := signal.SigTermCancelContext(context.Background())
+	return agent.Run(contextCtx, cfg)
+}

pkg/cli/cmds/agent.go

@@ -0,0 +1,81 @@
+package cmds
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/urfave/cli"
+)
+
+type Agent struct {
+	Token     string
+	ServerURL string
+	DataDir   string
+	NodeIP    string
+	NodeName  string
+	Docker    bool
+	NoFlannel bool
+	Debug     bool
+	AgentShared
+}
+
+type AgentShared struct {
+	NodeIP string
+}
+
+var (
+	appName     = filepath.Base(os.Args[0])
+	AgentConfig Agent
+	NodeIPFlag  = cli.StringFlag{
+		Name:        "node-ip,i",
+		Usage:       "IP address to advertise for node",
+		Destination: &AgentConfig.NodeIP,
+	}
+	NodeNameFlag = cli.StringFlag{
+		Name:        "node-name",
+		Usage:       "Node name",
+		EnvVar:      "K3S_NODE_NAME",
+		Destination: &AgentConfig.NodeName,
+	}
+)
+
+func NewAgentCommand(action func(ctx *cli.Context) error) cli.Command {
+	return cli.Command{
+		Name:      "agent",
+		Usage:     "Run node agent",
+		UsageText: appName + " agent [OPTIONS]",
+		Action:    action,
+		Flags: []cli.Flag{
+			cli.StringFlag{
+				Name:        "token,t",
+				Usage:       "Token to use for authentication",
+				EnvVar:      "K3S_TOKEN",
+				Destination: &AgentConfig.Token,
+			},
+			cli.StringFlag{
+				Name:        "server,s",
+				Usage:       "Server to connect to",
+				EnvVar:      "K3S_URL",
+				Destination: &AgentConfig.ServerURL,
+			},
+			cli.StringFlag{
+				Name:        "data-dir,d",
+				Usage:       "Folder to hold state",
+				Destination: &AgentConfig.DataDir,
+				Value:       "/var/lib/rancher/k3s",
+			},
+			cli.BoolFlag{
+				Name:        "docker",
+				Usage:       "Use docker instead of containerd",
+				Destination: &AgentConfig.Docker,
+			},
+			cli.BoolFlag{
+				Name:        "no-flannel",
+				Usage:       "Disable embedded flannel",
+				Destination: &AgentConfig.NoFlannel,
+			},
+			NodeNameFlag,
+			NodeIPFlag,
+		},
+	}
+}

pkg/cli/cmds/kubectl.go

@@ -0,0 +1,15 @@
+package cmds
+
+import (
+	"github.com/urfave/cli"
+)
+
+func NewKubectlCommand(action func(*cli.Context) error) cli.Command {
+	return cli.Command{
+		Name:            "kubectl",
+		Usage:           "Run kubectl",
+		SkipFlagParsing: true,
+		SkipArgReorder:  true,
+		Action:          action,
+	}
+}

pkg/cli/cmds/root.go

@@ -0,0 +1,39 @@
+package cmds
+
+import (
+	"fmt"
+
+	"github.com/rancher/k3s/pkg/version"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+var (
+	debug bool
+)
+
+func NewApp() *cli.App {
+	app := cli.NewApp()
+	app.Name = appName
+	app.Usage = "Kubernetes, but small and simple"
+	app.Version = version.Version
+	cli.VersionPrinter = func(c *cli.Context) {
+		fmt.Printf("%s version %s\n", app.Name, app.Version)
+	}
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "Turn on debug logs",
+			Destination: &debug,
+		},
+	}
+
+	app.Before = func(ctx *cli.Context) error {
+		if debug {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+		return nil
+	}
+
+	return app
+}

pkg/cli/cmds/server.go

@@ -0,0 +1,62 @@
+package cmds
+
+import (
+	"github.com/urfave/cli"
+)
+
+type Server struct {
+	Log          string
+	ClusterCIDR  string
+	ServiceCIDR  string
+	HTTPSPort    int
+	HTTPPort     int
+	DataDir      string
+	DisableAgent bool
+}
+
+var ServerConfig Server
+
+func NewServerCommand(action func(*cli.Context) error) cli.Command {
+	return cli.Command{
+		Name:      "server",
+		Usage:     "Run management server",
+		UsageText: appName + " server [OPTIONS]",
+		Action:    action,
+		Flags: []cli.Flag{
+			cli.IntFlag{
+				Name:        "https-listen-port",
+				Usage:       "HTTPS listen port",
+				Value:       6443,
+				Destination: &ServerConfig.HTTPSPort,
+			},
+			cli.IntFlag{
+				Name:        "http-listen-port",
+				Usage:       "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)",
+				Value:       0,
+				Destination: &ServerConfig.HTTPPort,
+			},
+			cli.StringFlag{
+				Name:        "data-dir,d",
+				Usage:       "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
+				Destination: &ServerConfig.DataDir,
+			},
+			cli.BoolFlag{
+				Name:        "disable-agent",
+				Usage:       "Do not run a local agent and register a local kubelet",
+				Destination: &ServerConfig.DisableAgent,
+			},
+			cli.StringFlag{
+				Name:        "log,l",
+				Usage:       "Log to file",
+				Destination: &ServerConfig.Log,
+			},
+			cli.StringFlag{
+				Name:        "cluster-cidr",
+				Usage:       "Network CIDR to use for pod IPs",
+				Destination: &ServerConfig.ClusterCIDR,
+			},
+			NodeIPFlag,
+			NodeNameFlag,
+		},
+	}
+}

pkg/cli/kubectl/kubectl.go

@@ -0,0 +1,11 @@
+package kubectl
+
+import (
+	"github.com/rancher/k3s/pkg/kubectl"
+	"github.com/urfave/cli"
+)
+
+func Run(ctx *cli.Context) error {
+	kubectl.Main()
+	return nil
+}

pkg/cli/server/server.go

@@ -0,0 +1,90 @@
+package server
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/natefinch/lumberjack"
+	"github.com/rancher/k3s/pkg/agent"
+	"github.com/rancher/k3s/pkg/cli/cmds"
+	"github.com/rancher/k3s/pkg/server"
+	"github.com/rancher/norman/signal"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func setupLogging(app *cli.Context) {
+	if !app.GlobalBool("debug") {
+		flag.Set("stderrthreshold", "3")
+		flag.Set("alsologtostderr", "false")
+		flag.Set("logtostderr", "false")
+	}
+}
+
+func runWithLogging(app *cli.Context, cfg *cmds.Server) error {
+	l := &lumberjack.Logger{
+		Filename:   cfg.Log,
+		MaxSize:    50,
+		MaxBackups: 3,
+		MaxAge:     28,
+		Compress:   true,
+	}
+
+	args := append([]string{"k3s"}, os.Args[1:]...)
+	cmd := reexec.Command(args...)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, "_RIO_REEXEC_=true")
+	cmd.Stderr = l
+	cmd.Stdout = l
+	cmd.Stdin = os.Stdin
+	return cmd.Run()
+}
+
+func Run(app *cli.Context) error {
+	return run(app, &cmds.ServerConfig)
+}
+
+func run(app *cli.Context, cfg *cmds.Server) error {
+	if cfg.Log != "" && os.Getenv("_RIO_REEXEC_") == "" {
+		return runWithLogging(app, cfg)
+	}
+
+	setupLogging(app)
+
+	if !cfg.DisableAgent && os.Getuid() != 0 {
+		return fmt.Errorf("must run as root unless --disable-agent is specified")
+	}
+
+	serverConfig := server.Config{}
+	serverConfig.ControlConfig.DataDir = cfg.DataDir
+	serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort
+	serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort
+
+	logrus.Info("Starting k3s ", app.App.Version)
+	ctx := signal.SigTermCancelContext(context.Background())
+	certs, err := server.StartServer(ctx, &serverConfig)
+	if err != nil {
+		return err
+	}
+
+	if cfg.DisableAgent {
+		<-ctx.Done()
+		return nil
+	}
+
+	logFile := filepath.Join(serverConfig.ControlConfig.DataDir, "agent/agent.log")
+	url := fmt.Sprintf("https://localhost:%d", serverConfig.TLSConfig.HTTPSPort)
+	logrus.Infof("Agent starting, logging to %s", logFile)
+	token := server.FormatToken(serverConfig.ControlConfig.Runtime.NodeToken, certs)
+
+	agentConfig := cmds.AgentConfig
+	agentConfig.DataDir = serverConfig.ControlConfig.DataDir
+	agentConfig.ServerURL = url
+	agentConfig.Token = token
+
+	return agent.Run(ctx, agentConfig)
+}

pkg/containerd/builtins.go

@@ -0,0 +1,39 @@
+// +build ctrd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+// register containerd builtins here
+import (
+	_ "github.com/containerd/containerd/diff/walking/plugin"
+	_ "github.com/containerd/containerd/gc/scheduler"
+	_ "github.com/containerd/containerd/runtime/restart/monitor"
+	_ "github.com/containerd/containerd/services/containers"
+	_ "github.com/containerd/containerd/services/content"
+	_ "github.com/containerd/containerd/services/diff"
+	_ "github.com/containerd/containerd/services/events"
+	_ "github.com/containerd/containerd/services/healthcheck"
+	_ "github.com/containerd/containerd/services/images"
+	_ "github.com/containerd/containerd/services/introspection"
+	_ "github.com/containerd/containerd/services/leases"
+	_ "github.com/containerd/containerd/services/namespaces"
+	_ "github.com/containerd/containerd/services/opt"
+	_ "github.com/containerd/containerd/services/snapshots"
+	_ "github.com/containerd/containerd/services/tasks"
+	_ "github.com/containerd/containerd/services/version"
+)

pkg/containerd/builtins_cri_linux.go

@@ -0,0 +1,21 @@
+// +build ctrd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+import _ "github.com/containerd/cri"

pkg/containerd/builtins_linux.go

@@ -0,0 +1,28 @@
+// +build ctrd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+import (
+	_ "github.com/containerd/containerd/metrics/cgroups"
+	_ "github.com/containerd/containerd/runtime/v1/linux"
+	_ "github.com/containerd/containerd/runtime/v2"
+	_ "github.com/containerd/containerd/runtime/v2/runc/options"
+	_ "github.com/containerd/containerd/snapshots/native"
+	_ "github.com/containerd/containerd/snapshots/overlay"
+)

pkg/containerd/main.go

@@ -0,0 +1,21 @@
+// +build ctrd
+
+package containerd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/cmd/containerd/command"
+	"github.com/containerd/containerd/pkg/seed"
+)
+
+func Main() {
+	//klog.InitFlags(nil)
+	seed.WithTimeAndRand()
+	app := command.App()
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
+		os.Exit(1)
+	}
+}

pkg/containerd/none.go

@@ -0,0 +1,6 @@
+// +build !ctrd
+
+package containerd
+
+func Main() {
+}

pkg/controller/tls/handler.go

@@ -1,12 +0,0 @@
-package tls
-
-import (
-	"context"
-
-	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
-)
-
-func Register(ctx context.Context) error {
-	clients := v1.ClientsFrom(ctx)
-	clients.ListenerConfig.OnChange(ctx)
-}

pkg/daemons/agent/agent.go

@@ -3,10 +3,12 @@ package agent
 import (
 	"context"
 	"math/rand"
+	"path/filepath"
 	"time"
 
-	"github.com/rancher/rio/pkg/daemons/config"
+	"k8s.io/apimachinery/pkg/util/net"
 
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apiserver/pkg/util/logs"
 	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
@@ -18,23 +20,12 @@ import (
 func Agent(config *config.Agent) error {
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	prepare(config)
-
 	kubelet(config)
 	kubeProxy(config)
 
 	return nil
 }
 
-func prepare(config *config.Agent) {
-	if config.CNIBinDir == "" {
-		config.CNIBinDir = "/opt/cni/bin"
-	}
-	if config.CNIConfDir == "" {
-		config.CNIConfDir = "/etc/cni/net.d"
-	}
-}
-
 func kubeProxy(config *config.Agent) {
 	args := []string{
 		"--proxy-mode", "iptables",
@@ -52,7 +43,7 @@ func kubeProxy(config *config.Agent) {
 	}()
 }
 
-func kubelet(config *config.Agent) {
+func kubelet(cfg *config.Agent) {
 	command := app.NewKubeletCommand(context.Background().Done())
 	logs.InitLogs()
 	defer logs.FlushLogs()
@@ -62,37 +53,51 @@ func kubelet(config *config.Agent) {
 		"--read-only-port", "0",
 		"--allow-privileged=true",
 		"--cluster-domain", "cluster.local",
-		"--kubeconfig", config.KubeConfig,
+		"--kubeconfig", cfg.KubeConfig,
 		"--eviction-hard", "imagefs.available<5%,nodefs.available<5%",
 		"--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%",
-		"--feature-gates=MountPropagation=true",
+		"--node-ip", cfg.NodeIP,
-		"--node-ip", config.NodeIP,
 		"--fail-swap-on=false",
-		"--cgroup-root", "/k3s",
+		//"--cgroup-root", "/k3s",
 		"--cgroup-driver", "cgroupfs",
-		"--cni-conf-dir", config.CNIConfDir,
-		"--cni-bin-dir", config.CNIBinDir,
 	}
-	if len(config.ClusterDNS) > 0 {
+	if cfg.RootDir != "" {
-		args = append(args, "--cluster-dns", config.ClusterDNS.String())
+		args = append(args, "--root-dir", cfg.RootDir)
+		args = append(args, "--cert-dir", filepath.Join(cfg.RootDir, "pki"))
+		args = append(args, "--seccomp-profile-root", filepath.Join(cfg.RootDir, "seccomp"))
 	}
-	if config.RuntimeSocket != "" {
+	if cfg.CNIConfDir != "" {
-		args = append(args, "--container-runtime-endpoint", config.RuntimeSocket)
+		args = append(args, "--cni-conf-dir", cfg.CNIConfDir)
 	}
-	if config.ListenAddress != "" {
+	if cfg.CNIBinDir != "" {
-		args = append(args, "--address", config.ListenAddress)
+		args = append(args, "--cni-bin-dir", cfg.CNIBinDir)
 	}
-	if config.CACertPath != "" {
+	if len(cfg.ClusterDNS) > 0 {
-		args = append(args, "--anonymous-auth=false", "--client-ca-file", config.CACertPath)
+		args = append(args, "--cluster-dns", cfg.ClusterDNS.String())
 	}
-	if config.NodeName != "" {
+	if cfg.RuntimeSocket != "" {
-		args = append(args, "--hostname-override", config.NodeName)
+		args = append(args, "--container-runtime", "remote")
+		args = append(args, "--container-runtime-endpoint", cfg.RuntimeSocket)
 	}
-	args = append(args, config.ExtraKubeletArgs...)
+	if cfg.ListenAddress != "" {
+		args = append(args, "--address", cfg.ListenAddress)
+	}
+	if cfg.CACertPath != "" {
+		args = append(args, "--anonymous-auth=false", "--client-ca-file", cfg.CACertPath)
+	}
+	if cfg.NodeName != "" {
+		args = append(args, "--hostname-override", cfg.NodeName)
+	}
+	defaultIP, err := net.ChooseHostInterface()
+	if err != nil || defaultIP.String() != cfg.NodeIP {
+		args = append(args, "--node-ip", cfg.NodeIP)
+	}
+	args = append(args, cfg.ExtraKubeletArgs...)
 
 	command.SetArgs(args)
 
 	go func() {
+		logrus.Infof("Running kubelet %s", config.ArgString(args))
 		logrus.Fatalf("kubelet exited: %v", command.Execute())
 	}()
 }

pkg/daemons/config/types.go

@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"net"
 	"net/http"
+	"strings"
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 )
@@ -11,18 +12,28 @@ import (
 type Node struct {
 	Docker        bool
 	NoFlannel     bool
-	NoCoreDNS     bool
+	FlannelConf   string
 	LocalAddress  string
+	Containerd    Containerd
 	AgentConfig   Agent
 	CACerts       []byte
 	ServerAddress string
 	Certificate   *tls.Certificate
 }
 
+type Containerd struct {
+	Address string
+	Root    string
+	State   string
+	Config  string
+	Opt     string
+}
+
 type Agent struct {
 	NodeName           string
 	ClusterCIDR        net.IPNet
 	ClusterDNS         net.IP
+	RootDir            string
 	KubeConfig         string
 	NodeIP             string
 	RuntimeSocket      string
@@ -39,6 +50,8 @@ type Control struct {
 	ListenPort            int
 	ClusterIPRange        *net.IPNet
 	ServiceIPRange        *net.IPNet
+	ClusterDNS            net.IP
+	NoCoreDNS             bool
 	DataDir               string
 	ETCDEndpoints         []string
 	ETCDKeyFile           string
@@ -48,7 +61,7 @@ type Control struct {
 	ExtraAPIArgs          []string
 	ExtraControllerArgs   []string
 	ExtraSchedulerAPIArgs []string
-	NodeConfig            Node
+	//NodeConfig            Node
 
 	Runtime *ControlRuntime `json:"-"`
 }
@@ -72,3 +85,16 @@ type ControlRuntime struct {
 	Tunnel        http.Handler
 	Authenticator authenticator.Request
 }
+
+type ArgString []string
+
+func (a ArgString) String() string {
+	b := strings.Builder{}
+	for _, s := range a {
+		if b.Len() > 0 {
+			b.WriteString(" ")
+		}
+		b.WriteString(s)
+	}
+	return b.String()
+}

pkg/daemons/control/server.go

@@ -22,7 +22,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/rancher/rio/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/config"
 
 	_ "github.com/mattn/go-sqlite3" // sqlite
 	"github.com/sirupsen/logrus"
@@ -93,81 +93,68 @@ func Server(ctx context.Context, cfg *config.Control) error {
 	return nil
 }
 
-func controllerManager(config *config.Control, runtime *config.ControlRuntime) {
+func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) {
 	args := []string{
 		"--kubeconfig", runtime.KubeConfigSystem,
 		"--leader-elect=true",
 		"--service-account-private-key-file", runtime.ServiceKey,
 		"--allocate-node-cidrs",
-		"--cluster-cidr", config.ClusterIPRange.String(),
+		"--cluster-cidr", cfg.ClusterIPRange.String(),
 		"--root-ca-file", runtime.TokenCA,
 		"--port", "0",
 		"--secure-port", "0",
 	}
-	args = append(args, config.ExtraControllerArgs...)
+	args = append(args, cfg.ExtraControllerArgs...)
 	command := cmapp.NewControllerManagerCommand()
 	command.SetArgs(args)
 
 	go func() {
-		logrus.Infof("Running kube-controller-manager %s", argString(args))
+		logrus.Infof("Running kube-controller-manager %s", config.ArgString(args))
 		logrus.Fatalf("controller-manager exited: %v", command.Execute())
 	}()
 }
 
-type argString []string
+func scheduler(cfg *config.Control, runtime *config.ControlRuntime) {
-
-func (a argString) String() string {
-	b := strings.Builder{}
-	for _, s := range a {
-		if b.Len() > 0 {
-			b.WriteString(" ")
-		}
-		b.WriteString(s)
-	}
-	return b.String()
-}
-
-func scheduler(config *config.Control, runtime *config.ControlRuntime) {
 	args := []string{
 		"--kubeconfig", runtime.KubeConfigSystem,
 		"--port", "0",
 		"--secure-port", "0",
 	}
-	args = append(args, config.ExtraSchedulerAPIArgs...)
+	args = append(args, cfg.ExtraSchedulerAPIArgs...)
 	command := sapp.NewSchedulerCommand()
 	command.SetArgs(args)
 
 	go func() {
-		logrus.Infof("Running kube-scheduler %s", argString(args))
+		logrus.Infof("Running kube-scheduler %s", config.ArgString(args))
 		logrus.Fatalf("scheduler exited: %v", command.Execute())
 	}()
 }
 
-func apiServer(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
+func apiServer(ctx context.Context, cfg *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
 	var args []string
 
-	if len(config.ETCDEndpoints) > 0 {
+	if len(cfg.ETCDEndpoints) > 0 {
 		args = append(args, "--storage-backend", "etcd3")
-		args = append(args, "--etcd-servers", strings.Join(config.ETCDEndpoints, ","))
+		args = append(args, "--etcd-servers", strings.Join(cfg.ETCDEndpoints, ","))
-		if config.ETCDKeyFile != "" {
+		if cfg.ETCDKeyFile != "" {
-			args = append(args, "--etcd-keyfile", config.ETCDKeyFile)
+			args = append(args, "--etcd-keyfile", cfg.ETCDKeyFile)
 		}
-		if config.ETCDCAFile != "" {
+		if cfg.ETCDCAFile != "" {
-			args = append(args, "--etcd-cafile", config.ETCDCAFile)
+			args = append(args, "--etcd-cafile", cfg.ETCDCAFile)
 		}
-		if config.ETCDCertFile != "" {
+		if cfg.ETCDCertFile != "" {
-			args = append(args, "--etcd-certfile", config.ETCDCertFile)
+			args = append(args, "--etcd-certfile", cfg.ETCDCertFile)
 		}
 	}
 
 	args = append(args, "--allow-privileged=true")
 	args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ","))
 	args = append(args, "--service-account-signing-key-file", runtime.ServiceKey)
-	args = append(args, "--service-cluster-ip-range", config.ServiceIPRange.String())
+	args = append(args, "--service-cluster-ip-range", cfg.ServiceIPRange.String())
-	args = append(args, "--advertise-port", strconv.Itoa(config.AdvertisePort))
+	args = append(args, "--advertise-port", strconv.Itoa(cfg.AdvertisePort))
 	args = append(args, "--advertise-address", localhostIP.String())
 	args = append(args, "--insecure-port", "0")
-	args = append(args, "--secure-port", strconv.Itoa(config.ListenPort))
+	args = append(args, "--secure-port", strconv.Itoa(cfg.ListenPort))
 	args = append(args, "--bind-address", localhostIP.String())
 	args = append(args, "--tls-cert-file", runtime.TLSCert)
 	args = append(args, "--tls-private-key-file", runtime.TLSKey)
@@ -175,15 +162,15 @@ func apiServer(ctx context.Context, config *config.Control, runtime *config.Cont
 	args = append(args, "--service-account-issuer", "k3s")
 	args = append(args, "--api-audiences", "unknown")
 	args = append(args, "--basic-auth-file", runtime.PasswdFile)
-	//args = append(args, "--kubelet-client-certificate", runtime.NodeCert)
+	args = append(args, "--kubelet-client-certificate", runtime.NodeCert)
-	//args = append(args, "--kubelet-client-key", runtime.NodeKey)
+	args = append(args, "--kubelet-client-key", runtime.NodeKey)
 
-	args = append(args, config.ExtraAPIArgs...)
+	args = append(args, cfg.ExtraAPIArgs...)
 	command := app.NewAPIServerCommand(ctx.Done())
 	command.SetArgs(args)
 
 	go func() {
-		logrus.Infof("Running kube-apiserver %s", argString(args))
+		logrus.Infof("Running kube-apiserver %s", config.ArgString(args))
 		logrus.Fatalf("apiserver exited: %v", command.Execute())
 	}()
 
@@ -203,6 +190,10 @@ func defaults(config *config.Control) {
 		config.ServiceIPRange = serviceIPNet
 	}
 
+	if len(config.ClusterDNS) == 0 {
+		config.ClusterDNS = net.ParseIP("10.43.0.10")
+	}
+
 	if config.AdvertisePort == 0 {
 		config.AdvertisePort = 6445
 	}

pkg/datadir/datadir.go

@@ -0,0 +1,25 @@
+package datadir
+
+import (
+	"os"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman/pkg/resolvehome"
+)
+
+func Resolve(dataDir string) (string, error) {
+	if dataDir == "" {
+		if os.Getuid() == 0 {
+			dataDir = "/var/lib/rancher/k3s"
+		} else {
+			dataDir = "${HOME}/.rancher/k3s"
+		}
+	}
+
+	dataDir, err := resolvehome.Resolve(dataDir)
+	if err != nil {
+		return "", errors.Wrapf(err, "resolving %s", dataDir)
+	}
+
+	return dataDir, nil
+}

pkg/enterchroot/enter.go

@@ -1,254 +0,0 @@
-package enterchroot
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-	"syscall"
-	"time"
-
-	"github.com/docker/docker/pkg/mount"
-	"github.com/docker/docker/pkg/reexec"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	losetup "gopkg.in/freddierice/go-losetup.v1"
-)
-
-const (
-	magic = "_SQMAGIC_"
-)
-
-var (
-	symlinks = []string{"lib", "bin", "sbin", "lib64"}
-)
-
-func init() {
-	reexec.Register("enter-root", enter)
-}
-
-func enter() {
-	if os.Getenv("ENTER_DEBUG") == "true" {
-		logrus.SetLevel(logrus.DebugLevel)
-	}
-
-	logrus.Debug("Running bootstrap")
-	err := run(os.Getenv("ENTER_DATA"))
-	if err != nil {
-		logrus.Fatal(err)
-	}
-}
-
-func Mount(dataDir string, stdout, stderr io.Writer, args []string) error {
-	if logrus.GetLevel() >= logrus.DebugLevel {
-		os.Setenv("ENTER_DEBUG", "true")
-	}
-
-	root, offset, err := findRoot()
-	if err != nil {
-		return err
-	}
-
-	os.Setenv("ENTER_DATA", dataDir)
-	os.Setenv("ENTER_ROOT", root)
-
-	logrus.Debugf("Using data [%s] root [%s]", dataDir, root)
-
-	stat, err := os.Stat(root)
-	if err != nil {
-		return fmt.Errorf("failed to find %s: %v", root, err)
-	}
-
-	if !stat.IsDir() {
-		logrus.Debugf("Attaching file [%s] offset [%d]", root, offset)
-		dev, err := losetup.Attach(root, offset, true)
-		if err != nil {
-			return errors.Wrap(err, "creating loopback device")
-		}
-		defer dev.Detach()
-		os.Setenv("ENTER_DEVICE", dev.Path())
-
-		go func() {
-			// Assume that after 3 seconds loop back device has been mounted
-			time.Sleep(3 * time.Second)
-			info, err := dev.GetInfo()
-			if err != nil {
-				return
-			}
-
-			info.Flags |= losetup.FlagsAutoClear
-			err = dev.SetInfo(info)
-			if err != nil {
-				return
-			}
-		}()
-	}
-
-	logrus.Debugf("Running enter-root %v", args)
-	cmd := &exec.Cmd{
-		Path: reexec.Self(),
-		Args: append([]string{"enter-root"}, args...),
-		SysProcAttr: &syscall.SysProcAttr{
-			//Cloneflags:   syscall.CLONE_NEWPID | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC,
-			Unshareflags: syscall.CLONE_NEWNS,
-			Pdeathsig:    syscall.SIGKILL,
-		},
-		Stdout: stdout,
-		Stdin:  os.Stdin,
-		Stderr: stderr,
-		Env:    os.Environ(),
-	}
-	return cmd.Run()
-}
-
-func findRoot() (string, uint64, error) {
-	root := os.Getenv("ENTER_ROOT")
-	if root != "" {
-		return root, 0, nil
-	}
-
-	for _, suffix := range []string{".root", ".squashfs"} {
-		test := os.Args[0] + suffix
-		if _, err := os.Stat(test); err == nil {
-			return test, 0, nil
-		}
-	}
-
-	return inFile()
-}
-
-func inFile() (string, uint64, error) {
-	f, err := os.Open(reexec.Self())
-	if err != nil {
-		return "", 0, err
-	}
-	defer f.Close()
-
-	buf := make([]byte, 8192)
-	test := []byte(strings.ToLower(magic))
-	testLength := len(test)
-	offset := uint64(0)
-	found := 0
-
-	for {
-		n, err := f.Read(buf)
-		if err == io.EOF && n == 0 {
-			break
-		} else if err != nil {
-			return "", 0, err
-		}
-
-		for _, b := range buf[:n] {
-			if b == test[found] {
-				found++
-				if found == testLength {
-					return reexec.Self(), offset + 1, nil
-				}
-			} else {
-				found = 0
-			}
-			offset++
-		}
-	}
-
-	return "", 0, fmt.Errorf("failed to find image in file %s", os.Args[0])
-}
-
-func run(data string) error {
-	os.MkdirAll(data, 0755)
-
-	if err := mount.Mount("tmpfs", data, "tmpfs", ""); err != nil {
-		return errors.Wrapf(err, "remounting data %s", data)
-	}
-
-	root := os.Getenv("ENTER_ROOT")
-	device := os.Getenv("ENTER_DEVICE")
-
-	logrus.Debugf("Using root %s %s", root, device)
-
-	usr := filepath.Join(data, "usr")
-	dotRoot := filepath.Join(data, ".root")
-
-	for _, d := range []string{usr, dotRoot} {
-		if err := os.MkdirAll(d, 0755); err != nil {
-			return fmt.Errorf("failed to make dir %s: %v", data, err)
-		}
-	}
-
-	if device == "" {
-		logrus.Debugf("Bind mounting %s to %s", root, usr)
-		if err := mount.Mount(root, usr, "none", "bind"); err != nil {
-			return fmt.Errorf("failed to bind mount")
-		}
-	} else {
-		logrus.Debugf("Mounting squashfs %s to %s", device, usr)
-		squashErr := checkSquashfs()
-		if err := mount.Mount(device, usr, "squashfs", "ro"); err != nil {
-			err = errors.Wrap(err, "mounting squashfs")
-			if squashErr != nil {
-				err = errors.Wrap(err, squashErr.Error())
-			}
-			return err
-		}
-	}
-
-	if err := os.Chdir(data); err != nil {
-		return err
-	}
-
-	for _, p := range symlinks {
-		if _, err := os.Lstat(p); os.IsNotExist(err) {
-			if err := os.Symlink(filepath.Join("usr", p), p); err != nil {
-				return errors.Wrapf(err, "failed to symlink %s", p)
-			}
-		}
-	}
-
-	logrus.Debugf("pivoting to . .root")
-	if err := syscall.PivotRoot(".", ".root"); err != nil {
-		return errors.Wrap(err, "pivot_root failed")
-	}
-
-	if err := mount.ForceMount("", ".", "none", "rprivate"); err != nil {
-		return errors.Wrapf(err, "making . private %s", data)
-	}
-
-	if err := syscall.Chroot("/"); err != nil {
-		return err
-	}
-
-	if err := os.Chdir("/"); err != nil {
-		return err
-	}
-
-	if _, err := os.Stat("/usr/init"); err != nil {
-		return errors.Wrap(err, "failed to find /usr/init")
-	}
-
-	return syscall.Exec("/usr/init", os.Args, os.Environ())
-}
-
-func checkSquashfs() error {
-	if !inProcFS() {
-		exec.Command("modprobe", "squashfs").Run()
-	}
-
-	if !inProcFS() {
-		return errors.New("This kernel does not support squashfs, please enable. " +
-			"On Fedora you may need to run \"dnf install kernel-modules-$(uname -r)\"")
-	}
-
-	return nil
-}
-
-func inProcFS() bool {
-	bytes, err := ioutil.ReadFile("/proc/filesystems")
-	if err != nil {
-		logrus.Errorf("Failed to read /proc/filesystems: %v", err)
-		return false
-	}
-	return strings.Contains(string(bytes), "squashfs")
-}

pkg/kubectl/main.go

@@ -7,18 +7,13 @@ import (
 	"os"
 	"time"
 
-	"github.com/docker/docker/pkg/reexec"
+	"github.com/rancher/k3s/pkg/server"
-	"github.com/rancher/rio/pkg/server"
 	"github.com/spf13/pflag"
 	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/pkg/kubectl/cmd"
 )
 
-func init() {
-	reexec.Register("kubectl", Main)
-}
-
 func Main() {
 	kubenv := os.Getenv("KUBECONFIG")
 	if kubenv == "" {

pkg/server/auth.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/rancher/rio/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apiserver/pkg/endpoints/request"
 )

pkg/server/router.go

@@ -3,11 +3,9 @@ package server
 import (
 	"net/http"
 
-	"github.com/rancher/rio/pkg/daemons/config"
-
-	"k8s.io/apimachinery/pkg/util/json"
-
 	"github.com/gorilla/mux"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"k8s.io/apimachinery/pkg/util/json"
 )
 
 type CACertsGetter func() (string, error)

pkg/server/server.go

@@ -6,21 +6,22 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io/ioutil"
+	net2 "net"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/rancher/k3s/pkg/daemons/config"
+	"github.com/rancher/k3s/pkg/daemons/control"
+	"github.com/rancher/k3s/pkg/tls"
+	v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
 	"github.com/rancher/norman"
 	"github.com/rancher/norman/pkg/clientaccess"
 	"github.com/rancher/norman/pkg/dynamiclistener"
 	"github.com/rancher/norman/pkg/resolvehome"
 	"github.com/rancher/norman/types"
-	"github.com/rancher/rio/pkg/daemons/config"
-	"github.com/rancher/rio/pkg/daemons/control"
-	"github.com/rancher/rio/pkg/tls"
-	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
 	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/net"
 )
@@ -38,25 +39,29 @@ func resolveDataDir(dataDir string) (string, error) {
 	return resolvehome.Resolve(dataDir)
 }
 
-func StartServer(ctx context.Context, config *Config) error {
+func StartServer(ctx context.Context, config *Config) (string, error) {
 	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
-		return err
+		return "", err
 	}
 
 	if err := control.Server(ctx, &config.ControlConfig); err != nil {
-		return errors.Wrap(err, "starting kubernetes")
+		return "", errors.Wrap(err, "starting kubernetes")
 	}
 
 	certs, err := startNorman(ctx, &config.TLSConfig, &config.ControlConfig)
 	if err != nil {
-		return errors.Wrap(err, "starting tls server")
+		return "", errors.Wrap(err, "starting tls server")
 	}
 
-	printTokens(certs, config.ControlConfig.NodeConfig.AgentConfig.NodeIP, &config.TLSConfig, &config.ControlConfig)
+	ip, err := net.ChooseHostInterface()
+	if err != nil {
+		ip = net2.ParseIP("127.0.0.1")
+	}
+	printTokens(certs, ip.String(), &config.TLSConfig, &config.ControlConfig)
 
 	writeKubeConfig(certs, &config.TLSConfig, &config.ControlConfig)
 
-	return nil
+	return certs, nil
 }
 
 func startNorman(ctx context.Context, tlsConfig *dynamiclistener.UserConfig, config *config.Control) (string, error) {
@@ -203,7 +208,7 @@ func printToken(httpsPort int, advertiseIP, prefix, file, cmd string) {
 		ip = hostIP.String()
 	}
 
-	logrus.Infof("%s rio %s -s https://%s:%d -t %s", prefix, cmd, ip, httpsPort, token)
+	logrus.Infof("%s k3s %s -s https://%s:%d -t %s", prefix, cmd, ip, httpsPort, token)
 }
 
 func FormatToken(token string, certs string) string {

pkg/server/types.go

@@ -1,8 +1,8 @@
 package server
 
 import (
+	"github.com/rancher/k3s/pkg/daemons/config"
 	"github.com/rancher/norman/pkg/dynamiclistener"
-	"github.com/rancher/rio/pkg/daemons/config"
 )
 
 type Config struct {

pkg/tls/storage.go

@@ -3,8 +3,8 @@ package tls
 import (
 	"context"
 
+	v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
 	"github.com/rancher/norman/pkg/dynamiclistener"
-	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"

pkg/untar/untar.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package untar untars a tarball to disk.
+package untar
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// TODO(bradfitz): this was copied from x/build/cmd/buildlet/buildlet.go
+// but there were some buildlet-specific bits in there, so the code is
+// forked for now.  Unfork and add some opts arguments here, so the
+// buildlet can use this code somehow.
+
+// Untar reads the gzip-compressed tar file from r and writes it into dir.
+func Untar(r io.Reader, dir string) error {
+	return untar(r, dir)
+}
+
+func untar(r io.Reader, dir string) (err error) {
+	t0 := time.Now()
+	nFiles := 0
+	madeDir := map[string]bool{}
+	defer func() {
+		td := time.Since(t0)
+		if err != nil {
+			logrus.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
+		}
+	}()
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("requires gzip-compressed body: %v", err)
+	}
+	tr := tar.NewReader(zr)
+	loggedChtimesError := false
+	for {
+		f, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			logrus.Printf("tar reading error: %v", err)
+			return fmt.Errorf("tar error: %v", err)
+		}
+		if !validRelPath(f.Name) {
+			return fmt.Errorf("tar contained invalid name error %q", f.Name)
+		}
+		rel := filepath.FromSlash(f.Name)
+		abs := filepath.Join(dir, rel)
+
+		fi := f.FileInfo()
+		mode := fi.Mode()
+		switch {
+		case mode.IsRegular():
+			// Make the directory. This is redundant because it should
+			// already be made by a directory entry in the tar
+			// beforehand. Thus, don't check for errors; the next
+			// write will fail with the same error.
+			dir := filepath.Dir(abs)
+			if !madeDir[dir] {
+				if err := os.MkdirAll(filepath.Dir(abs), 0755); err != nil {
+					return err
+				}
+				madeDir[dir] = true
+			}
+			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+			if err != nil {
+				return err
+			}
+			n, err := io.Copy(wf, tr)
+			if closeErr := wf.Close(); closeErr != nil && err == nil {
+				err = closeErr
+			}
+			if err != nil {
+				return fmt.Errorf("error writing to %s: %v", abs, err)
+			}
+			if n != f.Size {
+				return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
+			}
+			modTime := f.ModTime
+			if modTime.After(t0) {
+				// Clamp modtimes at system time. See
+				// golang.org/issue/19062 when clock on
+				// buildlet was behind the gitmirror server
+				// doing the git-archive.
+				modTime = t0
+			}
+			if !modTime.IsZero() {
+				if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError {
+					// benign error. Gerrit doesn't even set the
+					// modtime in these, and we don't end up relying
+					// on it anywhere (the gomote push command relies
+					// on digests only), so this is a little pointless
+					// for now.
+					logrus.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err)
+					loggedChtimesError = true // once is enough
+				}
+			}
+			nFiles++
+		case mode.IsDir():
+			if err := os.MkdirAll(abs, 0755); err != nil {
+				return err
+			}
+			madeDir[abs] = true
+		case f.Linkname != "":
+			if err := os.Symlink(f.Linkname, abs); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("tar file entry %s contained unsupported file type %v", f.Name, mode)
+		}
+	}
+	return nil
+}
+
+func validRelPath(p string) bool {
+	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+		return false
+	}
+	return true
+}

scripts/build

@@ -1,31 +1,41 @@
 #!/bin/bash
-set -e
+set -e -x
 
 source $(dirname $0)/version
 
 cd $(dirname $0)/..
 
-LDFLAGS="-X github.com/rancher/rio/version.Version=$VERSION -w -s"
+LDFLAGS="-X github.com/rancher/k3s/version.Version=$VERSION -w -s"
-STATIC="-extldflags -static"
+STATIC="-extldflags '-static'"
 STATIC_SQLITE="-extldflags '-static -lm -ldl -lz -lpthread'"
+TAGS="ctrd apparmor seccomp no_btrfs netgo osusergo"
 
-cross()
+if [ "$STATIC_BUILD" != "true" ]; then
-{
+    STATIC=""
-    echo Building windows CLI
+    STATIC_SQLITE=""
-    GOOS=windows GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-windows ./cli/main.go
+else
-    echo Building mac CLI
+    TAGS="static_build libsqlite3 $TAGS"
-    GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-darwin ./cli/main.go
+fi
-}
 
 mkdir -p bin
 
-if [ -n "$CROSS" ]; then
+rm -f bin/k3s-agent bin/hyperkube bin/containerd bin/cni ./bin/runc bin/containerd-shim bin/k3s-server bin/kubectl
-    cross
+# echo Building agent
-fi
+# CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
-
+echo Building server
-echo Building incluster
+CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/containerd ./cmd/server/main.go
-GOOS=linux GOARCH=amd64
+ln -s containerd ./bin/k3s-agent
-CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o bin/rio-incluster
+ln -s containerd ./bin/k3s-server
-
+ln -s containerd ./bin/kubectl
-echo Building cli full
+echo Building hyperkube
-CGO_ENABLED=1 go build -tags "static_build libsqlite3 k8s ctr no_etcd netgo osusergo" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/rio-full ./cli/main.go
+CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/hyperkube ./vendor/k8s.io/kubernetes/cmd/hyperkube/
+# echo Building containerd
+# CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
+echo Building cni
+CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o bin/cni ./cmd/cni/main.go
+echo Building runc
+make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
+cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
+echo Building containerd-shim
+make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
+cp -f ./vendor/github.com/containerd/containerd/bin/containerd-shim ./bin/containerd-shim

scripts/ci

@@ -6,4 +6,3 @@ cd $(dirname $0)
 ./validate
 ./build
 ./package
-./test

scripts/dev-agent.sh

@@ -1,14 +1,18 @@
 #!/bin/bash
 set -e
 
-cd $(dirname $0)/../bin
+cd $(dirname $0)/..
 
 # Prime sudo
-sudo echo Compiling CLI
+sudo echo Compiling
-go build -tags "k8s no_etcd" -o rio-agent ../cli/main.go
 
-echo Building image and agent
+if [ ! -e bin/containerd ]; then
-../image/build
+    ./scripts/build
+    ./scripts/package
+else
+    rm -f ./bin/k3s-agent
+    go build -tags "apparmor seccomp" -o ./bin/k3s-agent ./cmd/agent/main.go
+fi
 
-echo Running
+echo Starting agent
-exec sudo ENTER_ROOT=../image/main.squashfs ./rio-agent --debug agent -s https://localhost:7443 -t $(<${HOME}/.rancher/rio/server/node-token)
+sudo env "PATH=$(pwd)/bin:$PATH" ./bin/k3s-agent agent -s https://localhost:6443 -t $(<${HOME}/.rancher/k3s/server/node-token) "$@"

scripts/dev-docker-agent.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+cd $(dirname $0)/..
+IP=$(ip addr show dev docker0 | grep -w inet | awk '{print $2}' | cut -f1 -d/)
+docker run \
+    --read-only \
+    --tmpfs /run \
+    --tmpfs /var/run \
+    --tmpfs /tmp \
+    -v /lib/modules:/lib/modules:ro \
+    -v /lib/firmware:/lib/firmware:ro \
+    -v /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro \
+    -v $(pwd)/bin:/usr/bin \
+    -v /var/log \
+    -v /var/lib/rancher/k3s \
+    -v /var/lib/cni \
+    -v /usr/lib/x86_64-linux-gnu/libsqlite3.so.0:/usr/lib/x86_64-linux-gnu/libsqlite3.so.0:ro \
+    --privileged \
+    ubuntu:18.04 /usr/bin/k3s-agent agent -t $(<~/.rancher/k3s/server/node-token) -s https://${IP}:6443

scripts/dev-k8s-only-server.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -e
-
-cd $(dirname $0)/../bin
-
-echo Running
-go run -tags "k8s no_etcd" ../cli/main.go --debug server --disable-controllers --disable-agent

scripts/dev-login.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-set -e
-
-cd $(dirname $0)/../bin
-
-rio login -s https://localhost:5443 -t $(<${HOME}/.rancher/rio/server/client-token)

scripts/dev-server.sh

@@ -4,4 +4,4 @@ set -e
 cd $(dirname $0)/../bin
 
 echo Running
-go run -tags k3s ../cli/main.go --debug server --disable-agent
+go run -tags "apparmor" ../cmd/server/main.go --debug server --disable-agent

scripts/package

@@ -3,6 +3,9 @@ set -e
 
 cd $(dirname $0)
 
+if [ ! -e ../bin/k3s-server ]; then
+    ./build
+fi
+
 ./package-cli
 ./package-image
-./package-tar

scripts/package-cli

@@ -1,11 +1,42 @@
 #!/bin/bash
-set -e
+set -e -x
+
+ROOT_VERSION=v0.0.1
 
 source $(dirname $0)/version
 
-cd $(dirname $0)/../image
+cd $(dirname $0)/..
-./build
 
-cp ../bin/rio-full ../bin/rio
+curl --compressed -sfL https://github.com/ibuildthecloud/k3s-root/releases/download/${ROOT_VERSION}/k3s-root-${ARCH}.tar | tar xf -
-echo -n "_sqmagic_" >> ../bin/rio
+
-cat main.squashfs >> ../bin/rio
+rm -rf bin/kubectl bin/k3s-agent bin/k3s build/data
+ln -s containerd bin/k3s-agent
+ln -s containerd bin/k3s-server
+ln -s containerd bin/kubectl
+for i in bridge flannel host-local loopback portmap; do
+    if [ -e ./bin/$i ]; then
+        rm -f ./bin/$i
+    fi
+    ln -s cni ./bin/$i
+done
+
+
+mkdir -p build/data
+
+tar cvzf ./build/data.tar.gz --exclude ./bin/hyperkube ./bin
+HASH=$(sha256sum ./build/data.tar.gz | awk '{print $1}')
+
+cp ./build/data.tar.gz ./build/data/${HASH}.tgz
+
+if [ ${ARCH} = amd64 ]; then
+    cp -f ./bin/hyperkube dist/hyperkube
+    CMD_NAME=dist/k3s
+else
+    cp -f ./bin/hyperkube dist/hyperkube-${ARCH}
+    CMD_NAME=dist/k3s-${ARCH}
+fi
+
+go generate
+LDFLAGS="-X github.com/rancher/k3s/version.Version=$VERSION -w -s"
+STATIC="-extldflags '-static'"
+CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go

scripts/package-image

@@ -7,10 +7,11 @@ cd $(dirname $0)/../package
 
 TAG=${TAG:-${VERSION}${SUFFIX}}
 REPO=${REPO:-rancher}
+IMAGE_NAME=${IMAGE_NAME:-k3s}
 
-cp ../bin/rio ./rio
+cp -f ../build/data.tar.gz .
 
-IMAGE=${REPO}/rio:${TAG}
+IMAGE=${REPO}/${IMAGE_NAME}:${TAG}
 docker build -t ${IMAGE} .
 mkdir -p ../dist
 echo ${IMAGE} > ../dist/images

scripts/package-tar

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-cd $(dirname $0)/..
-
-. ./scripts/version
-
-mkdir -p dist/artifacts
-
-tar cvzf dist/artifacts/rio-${VERSION}-linux-amd64.tar.gz -h bin/rio --xform='s!^bin!rio-'${VERSION}'-linux-amd64!'
-tar cvzf dist/artifacts/rio-${VERSION}-darwin.tar.gz bin/rio-darwin --xform='s!.*!rio-'${VERSION}'-darwin/rio!'
-
-W=rio-${VERSION}-windows
-mkdir -p $W
-trap "rm -rf $W" EXIT
-
-cp -f bin/rio-windows $W/rio.exe
-zip dist/artifacts/rio-${VERSION}-windows.zip $W/rio.exe

scripts/test

@@ -1,50 +0,0 @@
-#!/bin/bash
-set -e
-
-cd $(dirname $0)/..
-
-echo Running tests
-
-mkdir -p ./build
-
-mkdir -p /var/lib/rancher/rio/agent
-mount -t tmpfs none /var/lib/rancher/rio/agent
-
-./bin/rio server --disable-agent | grep -v level=info > /var/lib/rancher/rio/agent/agent.log 2>&1 &
-
-for i in {1..120}; do
-    if [ ! -e /var/lib/rancher/rio/server/node-token ]; then
-        sleep .5
-        continue
-    fi
-
-    curl -sf http://localhost:7080/healthz >/dev/null && break
-    sleep .5
-done
-curl -sf http://localhost:7080/healthz >/dev/null
-
-rm -rf ./image/root
-unsquashfs -d ./image/root ./image/main.squashfs
-ENTER_ROOT=$(pwd)/image/root ./bin/rio --debug agent -s https://localhost:7443 -t $(</var/lib/rancher/rio/server/node-token) >>/var/lib/rancher/rio/agent/agent.log 2>&1 &
-
-export PATH=$(pwd)/bin:$PATH
-
-rio login -s https://localhost:7443 -t /var/lib/rancher/rio/server/client-token
-
-echo Waiting for istio/istio-gateway
-rio --project=rio-system wait istio/istio-gateway
-rio --project=rio-system ps
-rio --project=rio-system ps -c
-
-chmod +x ./tests/init-nfs.bash
-./tests/init-nfs.bash
-export RUN_NFS_TEST=true
-
-cd ./tests
-tox -- -n $(nproc)
-cd ..
-
-bats -r ./tests || {
-    tail -n 100 /var/lib/rancher/rio/agent/agent.log
-    exit 1
-}

scripts/validate

@@ -5,18 +5,14 @@ cd $(dirname $0)/..
 
 echo Running validation
 
-if grep -r '^[[:space:]]*\[\[.*]][[:space:]]*$' tests; then
-    echo "Add \"|| false\" to bats tests in any place you use [[ ... ]]"
-    exit 1
-fi
-
 PACKAGES="$(go list ./...)"
 
-echo Running: go vet
+#echo Running: go vet
-go vet ${PACKAGES}
+#go vet ${PACKAGES}
 echo Running: gometalinter
 for i in ${PACKAGES}; do
     if [ -n "$(gometalinter $i | \
+        grep -v 'pkg/data/zz_generated_bindata.go' | \
         grep -v 'should have comment.*or be unexported' | \
         grep -v 'cli/cmd.*don.t use underscores in Go name' | \
         grep -v 'cli/cmd.*should be DNS' | \
@@ -26,4 +22,6 @@ for i in ${PACKAGES}; do
 done
 test -z "$failed"
 echo Running: go fmt
-test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)"
+test -z "$(go fmt ${PACKAGES} | \
+        grep -v 'pkg/data/zz_generated_bindata.go' | \
+        tee /dev/stderr)"

scripts/version

@@ -13,6 +13,5 @@ else
     VERSION="${COMMIT}${DIRTY}"
 fi
 
-if [ -z "$ARCH" ]; then
+ARCH=$(go env GOARCH)
-    ARCH=amd64
+SUFFIX="-${ARCH}"
-fi

types/codegen/main.go

@@ -1,16 +1,33 @@
 package main
 
 import (
+	bindata "github.com/jteeuwen/go-bindata"
+	v1 "github.com/rancher/k3s/types/apis/k3s.cattle.io/v1"
 	"github.com/rancher/norman/generator"
-	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
 	"github.com/sirupsen/logrus"
 )
 
 var (
-	basePackage = "github.com/rancher/rio/types"
+	basePackage = "github.com/rancher/k3s/types"
 )
 
 func main() {
+	bc := &bindata.Config{
+		Input: []bindata.InputConfig{
+			{
+				Path:      "build/data",
+				Recursive: true,
+			},
+		},
+		Package:    "data",
+		NoCompress: true,
+		NoMemCopy:  true,
+		NoMetadata: true,
+		Output:     "pkg/data/zz_generated_bindata.go",
+	}
+	if err := bindata.Translate(bc); err != nil {
+		logrus.Fatal(err)
+	}
 	if err := generator.DefaultGenerate(v1.Schemas, basePackage, false, nil); err != nil {
 		logrus.Fatal(err)
 	}

vendor.conf

@@ -1,11 +1,18 @@
-package=github.com/rancher/rio
+package=github.com/rancher/k3s
 package=github.com/jteeuwen/go-bindata
 package=github.com/jteeuwen/go-bindata/go-bindata
+package=github.com/containerd/containerd/cmd/containerd-shim
+package=k8s.io/kubernetes/cmd/hyperkube
+package=github.com/opencontainers/runc
+package=github.com/coreos/go-systemd/activation
+package=github.com/opencontainers/runc/libcontainer/nsenter
+package=github.com/opencontainers/runc/libcontainer/specconv
+package=github.com/opencontainers/runc/contrib/cmd/recvtty
 
-k8s.io/kubernetes                   a1d7d1b140f43b6503311ffc1dd80017c553bf8e  file:///home/darren/src/kuberlite/.git            transitive=true,staging=true
+k8s.io/kubernetes                   v1.13.1-k3s2                              https://github.com/ibuildthecloud/k3s.git          transitive=true,staging=true
 
-github.com/rancher/norman           5726ebfba191eef161af9638e85bf4fcaa58e008  https://github.com/ibuildthecloud/norman.git
+github.com/rancher/norman           29915f8336c0a242560a9fef1d11bbaf04660915  https://github.com/ibuildthecloud/norman.git
-github.com/coreos/flannel           39af3d7e46f2efa156644e247bdcf3b5bc5f1394
+github.com/coreos/flannel           3d7cff78e2ca4cade87c6c7d44adf27fe3de2709  https://github.com/ibuildthecloud/flannel.git
 github.com/natefinch/lumberjack     aee4629129445bbdfb69aa565537dcfa16544311
 github.com/gorilla/mux              v1.6.2
 github.com/gorilla/websocket        v1.2.0
@@ -15,3 +22,93 @@ golang.org/x/crypto                 a49355c7e3f8fe157a85be2f77e6e269a0f89602
 gopkg.in/freddierice/go-losetup.v1  fc9adea44124401d8bfef3a97eaf61b5d44cc2c6
 github.com/urfave/cli               8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff
 golang.org/x/sync                   450f422ab23cf9881c94e2db30cac0eb1b7cf80c
+
+# flannel
+github.com/golang/glog              23def4e6c14b4da8ac2ed8007337bc5eb5007998
+github.com/coreos/go-iptables       47f22b0dd3355c0ba570ba12b0b8a36bf214c04b
+
+# cni
+github.com/containernetworking/plugins  9810b7d5137b171c4e07ce59bb18be9feccec557  https://github.com/ibuildthecloud/plugins.git
+github.com/j-keck/arping                2cf9dc699c5640a7e2c81403a44127bf28033600
+github.com/alexflint/go-filemutex       72bdc8eae2aef913234599b837f5dda445ca9bd9
+
+# runc
+github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294
+
+# containerd
+github.com/containerd/containerd v1.2.1
+github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3
+github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23
+github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2
+github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40
+github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
+github.com/containerd/btrfs 2e1aa0ddf94f91fa282b6ed87c23bf0d64911244
+github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4
+github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6
+github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098
+github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
+github.com/docker/go-units v0.3.1
+github.com/godbus/dbus c7fdd8b5cd55e87b4e1f4e372cdb1db61dd6c66f
+#github.com/prometheus/client_golang f4fb1b73fb099f396a7f0036bf86aa8def4ed823
+#github.com/prometheus/client_model 99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c
+#github.com/prometheus/common 89604d197083d4781071d3c65855d24ecfb0a563
+#github.com/prometheus/procfs cb4147076ac75738c9a7d279075a253c0cc5acbd
+github.com/beorn7/perks 4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9
+github.com/matttproud/golang_protobuf_extensions v1.0.0
+github.com/gogo/protobuf v1.0.0
+github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef
+github.com/golang/protobuf v1.1.0
+#github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d
+github.com/opencontainers/runc 96ec2177ae841256168fcf76954f7177af9446eb
+github.com/sirupsen/logrus v1.0.3
+github.com/urfave/cli 7bc6a0acffa589f415f88aca16cc1de5ffd66f9c
+golang.org/x/net b3756b4b77d7b13260a0a2ec658753cf48922eac
+google.golang.org/grpc v1.12.0
+github.com/pkg/errors v0.8.0
+github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7
+#golang.org/x/sys 1b2967e3c290b7c545b3db0deeda16e9be4f98a2 https://github.com/golang/sys
+github.com/opencontainers/image-spec v1.0.1
+golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
+github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
+github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
+github.com/Microsoft/go-winio v0.4.11
+github.com/Microsoft/hcsshim v0.8.3
+google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944
+golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4
+github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a
+github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
+gotest.tools v2.1.0
+github.com/google/go-cmp v0.1.0
+go.etcd.io/bbolt v1.3.1-etcd.8
+
+# cri dependencies
+github.com/containerd/cri 0ca1e3c2b73b5c38e72f29bb76338d0078b23d6c # release/1.2 branch
+github.com/containerd/go-cni 40bcf8ec8acd7372be1d77031d585d5d8e561c90
+github.com/blang/semver v3.1.0
+github.com/containernetworking/cni v0.6.0
+#github.com/containernetworking/plugins v0.7.0
+github.com/davecgh/go-spew v1.1.0
+github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621
+#github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00
+github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528
+github.com/emicklei/go-restful v2.2.1
+github.com/ghodss/yaml v1.0.0
+#github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed
+github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c
+github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55
+github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f
+github.com/json-iterator/go 1.1.5
+github.com/modern-go/reflect2 1.0.1
+github.com/modern-go/concurrent 1.0.3
+github.com/opencontainers/runtime-tools v0.6.0
+github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a
+github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
+github.com/tchap/go-patricia v2.2.6
+github.com/xeipuuv/gojsonpointer 4e3ac2762d5f479393488629ee9370b50873b3a6
+github.com/xeipuuv/gojsonreference bd5ef7bd5415a7ac448318e64f11a24cd21e594b
+github.com/xeipuuv/gojsonschema 1d523034197ff1f222f6429836dd36a2457a1874
+golang.org/x/crypto 49796115aa4b964c318aad4f3084fdb41e9aa067
+golang.org/x/oauth2 a6bd8cefa1811bd24b86f8902872e4e8225f74c4
+golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
+gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
+gopkg.in/yaml.v2 v2.2.1