Initial Commit

.dockerignore

@@ -0,0 +1,10 @@
+./bin
+./.vagrant
+./.dapper
+./data-dir
+./dist
+./.trash-cache
+./image/root
+./image/agent
+./image/go_build_agent
+./image/main.squashfs

.drone.yml

@@ -0,0 +1,55 @@
+---
+pipeline:
+  build:
+    privileged: true
+    image: rancher/dapper:1.11.2
+    volumes:
+    - /var/run/docker.sock:/var/run/docker.sock
+    commands:
+    - dapper ci
+
+  stage-binaries:
+    image: rancher/dapper:1.11.2
+    commands:
+    - cp -f ./bin/rio-incluster ./package/rio
+    when:
+      branch: master
+      event: tag
+
+  publish-image:
+    image: plugins/docker
+    dockerfile: package/Dockerfile
+    repo: rancher/rio
+    context: package/
+    tag: ${DRONE_TAG}
+    secrets: [docker_username, docker_password]
+    when:
+      branch: master
+      event: tag
+
+  github_binary_prerelease:
+    image: plugins/github-release
+    prerelease: true
+    files:
+     - dist/artifacts/*
+    checksum:
+     - sha256
+    secrets: [github_token]
+    when:
+      branch: master
+      event: tag
+      ref:
+        include: [ refs/tags/*rc* ]
+
+  github_binary_release:
+    image: plugins/github-release
+    files:
+     - dist/artifacts/*
+    checksum:
+     - sha256
+    secrets: [github_token]
+    when:
+      branch: master
+      event: tag
+      ref:
+        exclude: [ refs/tags/*rc* ]

.gitignore

@@ -0,0 +1,20 @@
+*.swp
+/.dapper
+/.idea
+/.trash-cache
+/.vagrant
+/*.log
+/bin
+/build
+/data-dir
+/dist
+/image/root
+/image/agent
+/image/go_build_agent
+/image/main.squashfs
+/package/rio
+__pycache__
+/tests/.pytest_cache/
+/tests/.tox/
+/tests/.vscode
+

.gometalinter.json

@@ -0,0 +1,11 @@
+{
+  "EnableAll": false,
+  "Enable": [
+    "golint",
+    "goimports",
+    "misspell",
+    "ineffassign",
+    "errcheck"
+  ],
+  "Deadline": "1m"
+}

Dockerfile.dapper

@@ -0,0 +1,29 @@
+FROM golang:1.11-alpine3.8
+
+RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
+    python3 py3-pip python3-dev openssl-dev libffi-dev
+RUN pip3 install 'tox==3.6.0'
+RUN npm install -g 'bats@1.1.0'
+RUN apk -U --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/main/ add sqlite-dev sqlite-static
+RUN go get -d golang.org/x/lint/golint && \
+    git -C /go/src/golang.org/x/lint/golint checkout -b current 06c8688daad7faa9da5a0c2f163a3d14aac986ca && \
+    go install golang.org/x/lint/golint && \
+    rm -rf /go/src /go/pkg
+RUN go get -d github.com/alecthomas/gometalinter && \
+    git -C /go/src/github.com/alecthomas/gometalinter checkout -b current v2.0.11 && \
+    go install github.com/alecthomas/gometalinter && \
+    gometalinter --install && \
+    rm -rf /go/src /go/pkg
+
+ENV DAPPER_RUN_ARGS --privileged
+ENV DAPPER_ENV REPO TAG DRONE_TAG
+ENV DAPPER_SOURCE /go/src/github.com/rancher/rio/
+ENV DAPPER_OUTPUT ./bin ./dist
+ENV DAPPER_DOCKER_SOCKET true
+ENV HOME ${DAPPER_SOURCE}
+ENV CROSS true
+WORKDIR ${DAPPER_SOURCE}
+
+ENTRYPOINT ["./scripts/entry"]
+CMD ["ci"]
+

LICENSE

@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

Makefile

@@ -0,0 +1,23 @@
+TARGETS := $(shell ls scripts)
+
+.dapper:
+	@echo Downloading dapper
+	@curl -sL https://releases.rancher.com/dapper/latest/dapper-`uname -s`-`uname -m` > .dapper.tmp
+	@@chmod +x .dapper.tmp
+	@./.dapper.tmp -v
+	@mv .dapper.tmp .dapper
+
+$(TARGETS): .dapper
+	./.dapper $@
+
+trash: .dapper
+	./.dapper -m bind trash
+
+trash-keep: .dapper
+	./.dapper -m bind trash -k
+
+deps: trash
+
+.DEFAULT_GOAL := ci
+
+.PHONY: $(TARGETS)

agent/config/config.go

@@ -0,0 +1,223 @@
+package config
+
+import (
+	"crypto/md5"
+	"crypto/tls"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman/pkg/clientaccess"
+	"github.com/rancher/rio/pkg/daemons/config"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/json"
+	net2 "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/client-go/util/cert"
+)
+
+type envInfo struct {
+	ServerURL string
+	Token     string
+	DataDir   string
+	NodeIP    string
+	NodeName  string
+}
+
+func Get() *config.Node {
+	for {
+		agentConfig, err := get()
+		if err != nil {
+			logrus.Error(err)
+			time.Sleep(5 * time.Second)
+			continue
+		}
+		return agentConfig
+	}
+}
+
+func getEnvInfo() (*envInfo, error) {
+	u := os.Getenv("K3S_URL")
+	if u == "" {
+		return nil, fmt.Errorf("K3S_URL env var is required")
+	}
+
+	_, err := url.Parse(u)
+	if err != nil {
+		return nil, fmt.Errorf("K3S_URL [%s] is invalid: %v", u, err)
+	}
+
+	t := os.Getenv("K3S_TOKEN")
+	if t == "" {
+		return nil, fmt.Errorf("K3S_TOKEN env var is required")
+	}
+
+	dataDir := os.Getenv("K3S_DATA_DIR")
+	if dataDir == "" {
+		return nil, fmt.Errorf("K3S_DATA_DIR is required")
+	}
+
+	return &envInfo{
+		ServerURL: u,
+		Token:     t,
+		DataDir:   dataDir,
+		NodeIP:    os.Getenv("K3S_NODE_IP"),
+		NodeName:  os.Getenv("NODE_NAME"),
+	}, nil
+}
+
+func getNodeCert(info *clientaccess.Info) (*tls.Certificate, error) {
+	nodeCert, err := clientaccess.Get("/v1-k3s/node.cert", info)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info)
+	if err != nil {
+		return nil, err
+	}
+
+	cert, err := tls.X509KeyPair(nodeCert, nodeKey)
+	if err != nil {
+		return nil, err
+	}
+	return &cert, nil
+}
+
+func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) {
+	clientCABytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: nodeCert.Certificate[1],
+	})
+
+	clientCA := filepath.Join(dataDir, "client-ca.pem")
+	if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil {
+		return "", errors.Wrapf(err, "failed to write client CA")
+	}
+
+	return clientCA, nil
+}
+
+func getHostnameAndIP(info envInfo) (string, string, error) {
+	ip := info.NodeIP
+	if ip == "" {
+		hostIP, err := net2.ChooseHostInterface()
+		if err != nil {
+			return "", "", err
+		}
+		ip = hostIP.String()
+	}
+
+	name := info.NodeName
+	if name == "" {
+		hostname, err := os.Hostname()
+		if err != nil {
+			return "", "", err
+		}
+		hostname = strings.Split(hostname, ".")[0]
+
+		d := md5.Sum([]byte(ip))
+		name = hostname + "-" + hex.EncodeToString(d[:])[:8]
+	}
+
+	return name, ip, nil
+}
+
+func localAddress(controlConfig *config.Control) string {
+	return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort)
+}
+
+func writeKubeConfig(envInfo *envInfo, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) {
+	os.MkdirAll(envInfo.DataDir, 0700)
+	kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml")
+
+	info.URL = "https://" + localAddress(controlConfig)
+	info.CACerts = pem.EncodeToMemory(&pem.Block{
+		Type:  cert.CertificateBlockType,
+		Bytes: nodeCert.Certificate[1],
+	})
+
+	return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath)
+}
+
+func get() (*config.Node, error) {
+	envInfo, err := getEnvInfo()
+	if err != nil {
+		return nil, err
+	}
+
+	serverURLParsed, err := url.Parse(envInfo.ServerURL)
+	if err != nil {
+		return nil, err
+	}
+
+	info, err := clientaccess.ParseAndValidateToken(envInfo.ServerURL, envInfo.Token)
+	if err != nil {
+		return nil, err
+	}
+
+	controlConfig, err := getConfig(info)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeCert, err := getNodeCert(info)
+	if err != nil {
+		return nil, err
+	}
+
+	clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeName, nodeIP, err := getHostnameAndIP(*envInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeConfig := &controlConfig.NodeConfig
+	nodeConfig.LocalAddress = localAddress(controlConfig)
+	nodeConfig.AgentConfig.NodeIP = defString(nodeConfig.AgentConfig.NodeIP, nodeIP)
+	nodeConfig.AgentConfig.NodeName = defString(nodeConfig.AgentConfig.NodeName, nodeName)
+	nodeConfig.AgentConfig.CNIBinDir = defString(nodeConfig.AgentConfig.CNIBinDir, "/usr/share/cni")
+	nodeConfig.AgentConfig.CACertPath = clientCA
+	nodeConfig.AgentConfig.ListenAddress = defString(nodeConfig.AgentConfig.ListenAddress, "127.0.0.1")
+	nodeConfig.AgentConfig.KubeConfig = kubeConfig
+	nodeConfig.CACerts = info.CACerts
+	nodeConfig.ServerAddress = serverURLParsed.Host
+	nodeConfig.Certificate = nodeCert
+	if !nodeConfig.Docker {
+		nodeConfig.AgentConfig.RuntimeSocket = "/run/k3s/containerd.sock"
+	}
+
+	return nodeConfig, nil
+}
+
+func defString(val, newVal string) string {
+	if val == "" {
+		return newVal
+	}
+	return val
+}
+
+func getConfig(info *clientaccess.Info) (*config.Control, error) {
+	data, err := clientaccess.Get("/v1-k3s/config", info)
+	if err != nil {
+		return nil, err
+	}
+
+	controlControl := &config.Control{}
+	return controlControl, json.Unmarshal(data, controlControl)
+}

agent/containerd/containerd.go

@@ -0,0 +1,89 @@
+package containerd
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/rancher/rio/agent/config"
+	util2 "github.com/rancher/rio/agent/util"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
+	"k8s.io/kubernetes/pkg/kubelet/util"
+)
+
+const (
+	address    = "/run/k3s/containerd.sock"
+	maxMsgSize = 1024 * 1024 * 16
+	configToml = `[plugins.cri]
+  stream_server_address = "%NODE%"
+  stream_server_port = "10010"
+  [plugins.cri.cni]
+    bin_dir = "/usr/share/cni/bin"
+    conf_dir = "/etc/cni/net.d"
+`
+)
+
+func Run(ctx context.Context, config *config.NodeConfig) error {
+	args := []string{
+		"containerd",
+		"-a", address,
+		"--state", "/run/k3s/containerd",
+	}
+
+	if err := util2.WriteFile("/etc/containerd/config.toml",
+		strings.Replace(configToml, "%NODE%", config.AgentConfig.NodeName, -1)); err != nil {
+		return err
+	}
+
+	if logrus.GetLevel() >= logrus.DebugLevel {
+		args = append(args, "--verbose")
+	}
+
+	go func() {
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Pdeathsig: syscall.SIGKILL,
+		}
+		if err := cmd.Run(); err != nil {
+			fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
+		}
+		os.Exit(1)
+	}()
+
+	for {
+		addr, dailer, err := util.GetAddressAndDialer("unix://" + address)
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(3*time.Second), grpc.WithDialer(dailer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
+		if err != nil {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		c := runtimeapi.NewRuntimeServiceClient(conn)
+
+		_, err = c.Version(ctx, &runtimeapi.VersionRequest{
+			Version: "0.1.0",
+		})
+		if err == nil {
+			conn.Close()
+			break
+		}
+		conn.Close()
+		logrus.Infof("Waiting for containerd startup")
+		time.Sleep(1 * time.Second)
+	}
+
+	return nil
+}

agent/flannel/flannel.go

@@ -0,0 +1,140 @@
+// Copyright 2015 flannel authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flannel
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/coreos/flannel/backend"
+	"github.com/coreos/flannel/network"
+	"github.com/coreos/flannel/pkg/ip"
+	"github.com/coreos/flannel/subnet/kube"
+	"golang.org/x/net/context"
+	log "k8s.io/klog"
+
+	// Backends need to be imported for their init() to get executed and them to register
+	_ "github.com/coreos/flannel/backend/vxlan"
+)
+
+const (
+	subnetFile = "/run/flannel/subnet.env"
+)
+
+func flannel(ctx context.Context, kubeConfigFile string) error {
+	extIface, err := LookupExtIface()
+	if err != nil {
+		return err
+	}
+
+	sm, err := kube.NewSubnetManager("", kubeConfigFile)
+	if err != nil {
+		return err
+	}
+
+	config, err := sm.GetNetworkConfig(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Create a backend manager then use it to create the backend and register the network with it.
+	bm := backend.NewManager(ctx, sm, extIface)
+
+	be, err := bm.GetBackend(config.BackendType)
+	if err != nil {
+		return err
+	}
+
+	bn, err := be.RegisterNetwork(ctx, sync.WaitGroup{}, config)
+	if err != nil {
+		return err
+	}
+
+	go network.SetupAndEnsureIPTables(network.MasqRules(config.Network, bn.Lease()), 60)
+	go network.SetupAndEnsureIPTables(network.ForwardRules(config.Network.String()), 50)
+
+	if err := WriteSubnetFile(subnetFile, config.Network, true, bn); err != nil {
+		// Continue, even though it failed.
+		log.Warningf("Failed to write subnet file: %s", err)
+	} else {
+		log.Infof("Wrote subnet file to %s", subnetFile)
+	}
+
+	// Start "Running" the backend network. This will block until the context is done so run in another goroutine.
+	log.Info("Running backend.")
+	bn.Run(ctx)
+	return nil
+}
+
+func LookupExtIface() (*backend.ExternalInterface, error) {
+	var iface *net.Interface
+	var ifaceAddr net.IP
+	var err error
+
+	log.Info("Determining IP address of default interface")
+	if iface, err = ip.GetDefaultGatewayIface(); err != nil {
+		return nil, fmt.Errorf("failed to get default interface: %s", err)
+	}
+
+	ifaceAddr, err = ip.GetIfaceIP4Addr(iface)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find IPv4 address for interface %s", iface.Name)
+	}
+
+	log.Infof("Using interface with name %s and address %s", iface.Name, ifaceAddr)
+
+	if iface.MTU == 0 {
+		return nil, fmt.Errorf("failed to determine MTU for %s interface", ifaceAddr)
+	}
+
+	return &backend.ExternalInterface{
+		Iface:     iface,
+		IfaceAddr: ifaceAddr,
+		ExtAddr:   ifaceAddr,
+	}, nil
+}
+
+func WriteSubnetFile(path string, nw ip.IP4Net, ipMasq bool, bn backend.Network) error {
+	dir, name := filepath.Split(path)
+	os.MkdirAll(dir, 0755)
+
+	tempFile := filepath.Join(dir, "."+name)
+	f, err := os.Create(tempFile)
+	if err != nil {
+		return err
+	}
+
+	// Write out the first usable IP by incrementing
+	// sn.IP by one
+	sn := bn.Lease().Subnet
+	sn.IP += 1
+
+	fmt.Fprintf(f, "FLANNEL_NETWORK=%s\n", nw)
+	fmt.Fprintf(f, "FLANNEL_SUBNET=%s\n", sn)
+	fmt.Fprintf(f, "FLANNEL_MTU=%d\n", bn.MTU())
+	_, err = fmt.Fprintf(f, "FLANNEL_IPMASQ=%v\n", ipMasq)
+	f.Close()
+	if err != nil {
+		return err
+	}
+
+	// rename(2) the temporary file to the desired location so that it becomes
+	// atomically visible with the contents
+	return os.Rename(tempFile, path)
+	//TODO - is this safe? What if it's not on the same FS?
+}

agent/flannel/setup.go

@@ -0,0 +1,91 @@
+package flannel
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/rancher/rio/agent/util"
+
+	"github.com/rancher/rio/agent/config"
+	"github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const (
+	cniConf = `{
+  "name":"cbr0",
+  "cniVersion":"0.3.1",
+  "plugins":[
+    {
+      "type":"flannel",
+      "delegate":{
+        "forceAddress":true,
+        "isDefaultGateway":true
+      }
+    },
+    {
+      "type":"portmap",
+      "capabilities":{
+        "portMappings":true
+      }
+    }
+  ]
+}
+`
+	netJson = `{
+    "Network": "%CIDR%",
+    "Backend": {
+    "Type": "vxlan"
+    }
+}
+`
+)
+
+func Run(ctx context.Context, config *config.NodeConfig) error {
+	nodeName := config.AgentConfig.NodeName
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	if err != nil {
+		return err
+	}
+
+	client, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	for {
+		node, err := client.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
+		if err == nil && node.Spec.PodCIDR != "" {
+			break
+		}
+		if err == nil {
+			logrus.Infof("waiting for node %s CIDR not assigned yet", nodeName)
+		} else {
+			logrus.Infof("waiting for node %s: %v", nodeName, err)
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+	if err := createCNIConf(); err != nil {
+		return err
+	}
+
+	if err := createFlannelConf(config); err != nil {
+		return err
+	}
+
+	return flannel(ctx, config.AgentConfig.KubeConfig)
+}
+
+func createCNIConf() error {
+	return util.WriteFile("/etc/cni/net.d/10-flannel.conflist", cniConf)
+}
+
+func createFlannelConf(config *config.NodeConfig) error {
+	return util.WriteFile("/etc/kube-flannel/net-conf.json",
+		strings.Replace(netJson, "%CIDR", config.AgentConfig.ClusterCIDR.String(), -1))
+}

agent/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+
+	"github.com/rancher/norman/signal"
+	"github.com/rancher/rio/agent/config"
+	"github.com/rancher/rio/agent/containerd"
+	"github.com/rancher/rio/agent/flannel"
+	"github.com/rancher/rio/agent/proxy"
+	"github.com/rancher/rio/agent/syssetup"
+	"github.com/rancher/rio/agent/tunnel"
+	"github.com/rancher/rio/pkg/daemons/agent"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	if err := run(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func run() error {
+	ctx := signal.SigTermCancelContext(context.Background())
+
+	nodeConfig := config.Get()
+
+	if nodeConfig.Docker {
+		nodeConfig.AgentConfig.RuntimeSocket = ""
+	} else {
+		containerd.Run(ctx, nodeConfig)
+	}
+
+	if err := syssetup.Configure(); err != nil {
+		return err
+	}
+
+	if err := tunnel.Setup(nodeConfig); err != nil {
+		return err
+	}
+
+	if err := proxy.Run(nodeConfig); err != nil {
+		return err
+	}
+
+	if err := agent.Agent(&nodeConfig.AgentConfig); err != nil {
+		return err
+	}
+
+	if !nodeConfig.NoFlannel {
+		if err := flannel.Run(ctx, nodeConfig); err != nil {
+			return err
+		}
+	}
+
+	<-ctx.Done()
+	return ctx.Err()
+}

agent/proxy/proxy.go

@@ -0,0 +1,35 @@
+package proxy
+
+import (
+	"crypto/tls"
+	"net/http"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman/pkg/proxy"
+	"github.com/rancher/rio/agent/config"
+	"github.com/sirupsen/logrus"
+)
+
+func Run(config *config.NodeConfig) error {
+	proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true)
+	if err != nil {
+		return err
+	}
+
+	listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{
+		Certificates: []tls.Certificate{
+			*config.Certificate,
+		},
+	})
+
+	if err != nil {
+		return errors.Wrap(err, "Failed to start tls listener")
+	}
+
+	go func() {
+		err := http.Serve(listener, proxy)
+		logrus.Fatalf("TLS proxy stopped: %v", err)
+	}()
+
+	return nil
+}

agent/syssetup/setup.go

@@ -0,0 +1,18 @@
+package syssetup
+
+import (
+	"io/ioutil"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	callIPTablesFile = "/proc/sys/net/bridge/bridge-nf-call-iptables"
+)
+
+func Configure() error {
+	if err := ioutil.WriteFile(callIPTablesFile, []byte("1"), 0640); err != nil {
+		logrus.Warnf("failed to write value 1 at %s: %v", callIPTablesFile, err)
+	}
+	return nil
+}

agent/tunnel/tunnel.go

@@ -0,0 +1,79 @@
+package tunnel
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/rancher/norman/pkg/remotedialer"
+	"github.com/rancher/rio/agent/config"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var (
+	ports = map[string]bool{
+		"10250": true,
+		"10010": true,
+	}
+)
+
+func Setup(config *config.NodeConfig) error {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig)
+	if err != nil {
+		return err
+	}
+
+	transportConfig, err := restConfig.TransportConfig()
+	if err != nil {
+		return err
+	}
+
+	wsURL := fmt.Sprintf("wss://%s/v1/connect", config.ServerAddress)
+	headers := map[string][]string{
+		"X-K3s-NodeName": {config.AgentConfig.NodeName},
+	}
+	ws := &websocket.Dialer{}
+
+	if len(config.CACerts) > 0 {
+		pool := x509.NewCertPool()
+		pool.AppendCertsFromPEM(config.CACerts)
+		ws.TLSClientConfig = &tls.Config{
+			RootCAs: pool,
+		}
+	}
+
+	if transportConfig.Username != "" {
+		auth := transportConfig.Username + ":" + transportConfig.Password
+		auth = base64.StdEncoding.EncodeToString([]byte(auth))
+		headers["Authorization"] = []string{"Basic " + auth}
+	}
+
+	once := sync.Once{}
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+
+	go func() {
+		for {
+			logrus.Infof("Connecting to %s", wsURL)
+			remotedialer.ClientConnect(wsURL, http.Header(headers), ws, func(proto, address string) bool {
+				host, port, err := net.SplitHostPort(address)
+				return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
+			}, func(_ context.Context) error {
+				once.Do(wg.Done)
+				return nil
+			})
+			time.Sleep(5 * time.Second)
+		}
+	}()
+
+	wg.Wait()
+	return nil
+}

agent/util/file.go

@@ -0,0 +1,18 @@
+package util
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+func WriteFile(name string, content string) error {
+	os.Mkdir(filepath.Dir(name), 0755)
+	err := ioutil.WriteFile(name, []byte(content), 0644)
+	if err != nil {
+		return errors.Wrapf(err, "writing %s", name)
+	}
+	return nil
+}

cli/cmd/agent/agent.go

@@ -0,0 +1,19 @@
+package agent
+
+import "github.com/urfave/cli"
+
+type Agent struct {
+	T_Token   string `desc:"Token to use for authentication" env:"K3S_TOKEN"`
+	S_Server  string `desc:"Server to connect to" env:"K3S_URL"`
+	D_DataDir string `desc:"Folder to hold state" default:"/var/lib/rancher/k3s"`
+	L_Log     string `desc:"log to file"`
+	AgentShared
+}
+
+type AgentShared struct {
+	I_NodeIp string `desc:"IP address to advertise for node"`
+}
+
+func (a *Agent) Customize(command *cli.Command) {
+	command.Category = "CLUSTER RUNTIME"
+}

cli/cmd/agent/agent_k3s.go

@@ -0,0 +1,81 @@
+// +build k8s
+
+package agent
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/natefinch/lumberjack"
+	"github.com/rancher/norman/pkg/clientaccess"
+	"github.com/rancher/norman/pkg/resolvehome"
+	"github.com/rancher/rio/pkg/enterchroot"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+)
+
+func (a *Agent) Run(ctx *cli.Context) error {
+	if os.Getuid() != 0 {
+		return fmt.Errorf("agent must be ran as root")
+	}
+
+	if len(a.T_Token) == 0 {
+		return fmt.Errorf("--token is required")
+	}
+
+	if len(a.S_Server) == 0 {
+		return fmt.Errorf("--server is required")
+	}
+
+	dataDir, err := resolvehome.Resolve(a.D_DataDir)
+	if err != nil {
+		return err
+	}
+
+	return RunAgent(a.S_Server, a.T_Token, dataDir, a.L_Log, a.I_NodeIp)
+}
+
+func RunAgent(server, token, dataDir, logFile, ipAddress string) error {
+	dataDir = filepath.Join(dataDir, "agent")
+
+	for {
+		tmpFile, err := clientaccess.AgentAccessInfoToTempKubeConfig("", server, token)
+		if err != nil {
+			logrus.Error(err)
+			time.Sleep(2 * time.Second)
+			continue
+		}
+		os.Remove(tmpFile)
+		break
+	}
+
+	os.Setenv("K3S_URL", server)
+	os.Setenv("K3S_TOKEN", token)
+	os.Setenv("K3S_DATA_DIR", dataDir)
+	os.Setenv("K3S_NODE_IP", ipAddress)
+
+	os.MkdirAll(dataDir, 0700)
+
+	stdout := io.Writer(os.Stdout)
+	stderr := io.Writer(os.Stderr)
+
+	if logFile == "" {
+		stdout = os.Stdout
+		stderr = os.Stderr
+	} else {
+		l := &lumberjack.Logger{
+			Filename:   logFile,
+			MaxSize:    50,
+			MaxBackups: 3,
+			MaxAge:     28,
+			Compress:   true,
+		}
+		stdout = l
+		stderr = l
+	}
+
+	return enterchroot.Mount(filepath.Join(dataDir, "root"), stdout, stderr, os.Args[1:])
+}

cli/cmd/agent/agent_none.go

@@ -0,0 +1,17 @@
+// +build !k8s
+
+package agent
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+func (a *Agent) Run(ctx *cli.Context) error {
+	return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"")
+}
+
+func RunAgent(server, token, dataDir, logFile string) error {
+	return fmt.Errorf("agent support is not compiled in, add \"-tags k8s\" to \"go build\"")
+}

cli/cmd/kubectl/kubectl.go

@@ -0,0 +1,24 @@
+package kubectl
+
+import (
+	"os"
+
+	"github.com/rancher/rio/pkg/kubectl"
+	"github.com/urfave/cli"
+)
+
+func NewKubectlCommand() cli.Command {
+	return cli.Command{
+		Name:            "kubectl",
+		Usage:           "Run kubectl",
+		SkipFlagParsing: true,
+		SkipArgReorder:  true,
+		Action:          run,
+	}
+}
+
+func run(ctx *cli.Context) error {
+	os.Args = append([]string{"kubectl"}, ctx.Args()...)
+	kubectl.Main()
+	return nil
+}

cli/cmd/server/server.go

@@ -0,0 +1,128 @@
+package server
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/natefinch/lumberjack"
+	"github.com/rancher/norman/signal"
+	"github.com/rancher/rio/pkg/server"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+	"k8s.io/apimachinery/pkg/util/net"
+)
+
+var (
+	appName = filepath.Base(os.Args[0])
+
+	config server.Config
+	log    string
+)
+
+var ServerCommand = cli.Command{
+	Name:      "server",
+	Usage:     "Run management server",
+	UsageText: appName + " server [OPTIONS]",
+	Action:    Run,
+	Flags: []cli.Flag{
+		cli.IntFlag{
+			Name:        "https-listen-port",
+			Usage:       "HTTPS listen port",
+			Value:       6443,
+			Destination: &config.TLSConfig.HTTPSPort,
+		},
+		cli.IntFlag{
+			Name:        "http-listen-port",
+			Usage:       "HTTP listen port (for /healthz, HTTPS redirect, and port for TLS terminating LB)",
+			Value:       0,
+			Destination: &config.TLSConfig.HTTPPort,
+		},
+		cli.StringFlag{
+			Name:        "data-dir",
+			Usage:       "Folder to hold state default /var/lib/rancher/k3s or ${HOME}/.rancher/k3s if not root",
+			Destination: &config.ControlConfig.DataDir,
+		},
+		//cli.StringFlag{
+		//	Name:        "advertise-address",
+		//	Usage:       "Address of the server to put in the generated kubeconfig",
+		//	Destination: &config.AdvertiseIP,
+		//},
+		cli.BoolFlag{
+			Name:        "disable-agent",
+			Usage:       "Do not run a local agent and register a local kubelet",
+			Destination: &config.DisableAgent,
+		},
+		cli.StringFlag{
+			Name:        "log",
+			Usage:       "Log to file",
+			Destination: &log,
+		},
+	},
+}
+
+func setupLogging(app *cli.Context) {
+	if !app.GlobalBool("debug") {
+		flag.Set("stderrthreshold", "3")
+		flag.Set("alsologtostderr", "false")
+		flag.Set("logtostderr", "false")
+	}
+}
+
+func runWithLogging(app *cli.Context) error {
+	l := &lumberjack.Logger{
+		Filename:   log,
+		MaxSize:    50,
+		MaxBackups: 3,
+		MaxAge:     28,
+		Compress:   true,
+	}
+
+	args := append([]string{"k3s"}, os.Args[1:]...)
+	cmd := reexec.Command(args...)
+	cmd.Env = os.Environ()
+	cmd.Env = append(cmd.Env, "_RIO_REEXEC_=true")
+	cmd.Stderr = l
+	cmd.Stdout = l
+	cmd.Stdin = os.Stdin
+	return cmd.Run()
+}
+
+func Run(app *cli.Context) error {
+	if log != "" && os.Getenv("_RIO_REEXEC_") == "" {
+		return runWithLogging(app)
+	}
+
+	setupLogging(app)
+
+	if !config.DisableAgent && os.Getuid() != 0 {
+		return fmt.Errorf("must run as root unless --disable-agent is specified")
+	}
+
+	if config.ControlConfig.NodeConfig.AgentConfig.NodeIP == "" {
+		ip, err := net.ChooseHostInterface()
+		if err == nil {
+			config.ControlConfig.NodeConfig.AgentConfig.NodeIP = ip.String()
+		}
+	}
+
+	logrus.Info("Starting k3s ", app.App.Version)
+	ctx := signal.SigTermCancelContext(context.Background())
+	if err := server.StartServer(ctx, &config); err != nil {
+		return err
+	}
+
+	if config.DisableAgent {
+		<-ctx.Done()
+		return nil
+	}
+
+	return nil
+	//logFile := filepath.Join(serverConfig.DataDir, "agent/agent.log")
+	//url := fmt.Sprintf("https://localhost:%d", httpsListenPort)
+	//logrus.Infof("Agent starting, logging to %s", logFile)
+	//return agent.RunAgent(url, server2.FormatToken(serverConfig.Runtime.NodeToken), serverConfig.DataDir, logFile, "")
+}

cli/pkg/builder/builder.go

@@ -0,0 +1,151 @@
+package builder
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"unsafe"
+
+	"github.com/urfave/cli"
+)
+
+var (
+	caseRegexp = regexp.MustCompile("([a-z])([A-Z])")
+)
+
+type clirunnable interface {
+	Run(app *cli.Context) error
+}
+
+type customizer interface {
+	Customize(cmd *cli.Command)
+}
+
+type fieldInfo struct {
+	FieldType  reflect.StructField
+	FieldValue reflect.Value
+}
+
+func fields(obj interface{}) []fieldInfo {
+	ptrValue := reflect.ValueOf(obj)
+	objValue := ptrValue.Elem()
+
+	var result []fieldInfo
+
+	for i := 0; i < objValue.NumField(); i++ {
+		fieldType := objValue.Type().Field(i)
+		if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct {
+			result = append(result, fields(objValue.Field(i).Addr().Interface())...)
+		} else if !fieldType.Anonymous {
+			result = append(result, fieldInfo{
+				FieldValue: objValue.Field(i),
+				FieldType:  objValue.Type().Field(i),
+			})
+		}
+	}
+
+	return result
+}
+
+func Command(obj interface{}, usage, usageText, description string) cli.Command {
+	slices := map[string]reflect.Value{}
+	maps := map[string]reflect.Value{}
+	ptrValue := reflect.ValueOf(obj)
+	objValue := ptrValue.Elem()
+
+	c := cli.Command{
+		Name:                   strings.ToLower(strings.Replace(objValue.Type().Name(), "Command", "", 1)),
+		Usage:                  usage,
+		UsageText:              usageText,
+		Description:            description,
+		UseShortOptionHandling: true,
+		SkipArgReorder:         true,
+	}
+
+	for _, info := range fields(obj) {
+		defMessage := ""
+		fieldType := info.FieldType
+		v := info.FieldValue
+
+		switch fieldType.Type.Kind() {
+		case reflect.Int:
+			flag := cli.IntFlag{
+				Name:        name(fieldType.Name),
+				Usage:       fieldType.Tag.Get("desc"),
+				EnvVar:      fieldType.Tag.Get("env"),
+				Destination: (*int)(unsafe.Pointer(v.Addr().Pointer())),
+			}
+			defValue := fieldType.Tag.Get("default")
+			if defValue != "" {
+				n, err := strconv.Atoi(defValue)
+				if err != nil {
+					panic("bad default " + defValue + " on field " + fieldType.Name)
+				}
+				flag.Value = n
+			}
+			c.Flags = append(c.Flags, flag)
+		case reflect.String:
+			flag := cli.StringFlag{
+				Name:        name(fieldType.Name),
+				Usage:       fieldType.Tag.Get("desc"),
+				Value:       fieldType.Tag.Get("default"),
+				EnvVar:      fieldType.Tag.Get("env"),
+				Destination: (*string)(unsafe.Pointer(v.Addr().Pointer())),
+			}
+			c.Flags = append(c.Flags, flag)
+		case reflect.Slice:
+			slices[name(fieldType.Name)] = v
+			defMessage = " "
+			fallthrough
+		case reflect.Map:
+			if defMessage == "" {
+				maps[name(fieldType.Name)] = v
+				defMessage = " "
+			}
+			flag := cli.StringSliceFlag{
+				Name:   name(fieldType.Name),
+				Usage:  fieldType.Tag.Get("desc") + defMessage,
+				EnvVar: fieldType.Tag.Get("env"),
+				Value:  &cli.StringSlice{},
+			}
+			c.Flags = append(c.Flags, flag)
+		case reflect.Bool:
+			flag := cli.BoolFlag{
+				Name:        name(fieldType.Name),
+				Usage:       fieldType.Tag.Get("desc"),
+				EnvVar:      fieldType.Tag.Get("env"),
+				Destination: (*bool)(unsafe.Pointer(v.Addr().Pointer())),
+			}
+			c.Flags = append(c.Flags, flag)
+		default:
+			panic("Unknown kind on field " + fieldType.Name + " on " + objValue.Type().Name())
+		}
+	}
+
+	if run, ok := obj.(clirunnable); ok {
+		c.Action = run.Run
+	} else {
+		panic(fmt.Sprintf("failed to find Action function for %T", obj))
+	}
+
+	cust, ok := obj.(customizer)
+	if ok {
+		cust.Customize(&c)
+	}
+
+	return c
+}
+
+func name(name string) string {
+	parts := strings.Split(name, "_")
+	i := len(parts) - 1
+	name = caseRegexp.ReplaceAllString(parts[i], "$1-$2")
+	name = strings.ToLower(name)
+	result := append([]string{name}, parts[0:i]...)
+	for i := 0; i < len(result); i++ {
+		result[i] = strings.ToLower(result[i])
+	}
+	return strings.Join(result, ",")
+}

image/Dockerfile

@@ -0,0 +1,179 @@
+### BASE ###
+FROM alpine:3.8 as base
+RUN apk -U add findutils iptables ipset bash ca-certificates jq iproute2 nfs-utils coreutils libseccomp conntrack-tools
+# RUN apk -U add eudev tinyssh e2fsprogs mdadm rsync nfs-utils parted
+
+FROM base as k3s-build
+COPY --from=base /bin /usr/src/image/bin/
+COPY --from=base /lib /usr/src/image/lib/
+COPY --from=base /sbin /usr/src/image/sbin/
+COPY --from=base /etc/ssl/certs/ca-certificates.crt /usr/src/image/etc/ssl/certs/ca-certificates.crt
+COPY --from=base /etc/terminfo /usr/src/image/etc/terminfo
+COPY --from=base /usr /usr/src/image/usr/
+
+WORKDIR /usr/src/image
+
+RUN rm -rf  usr/bin/iconv \
+            usr/bin/scanelf \
+            usr/bin/ssl_client \
+            usr/bin/pkgconf \
+            usr/bin/getent \
+            usr/bin/locate \
+            usr/bin/updatedb \
+            usr/bin/c_rehash \
+            usr/bin/getconf \
+            usr/etc \
+            usr/include \
+            usr/lib/bash \
+            usr/lib/krb5 \
+            usr/lib/pkgconfig \
+            usr/lib/tc \
+            usr/libexec \
+            usr/local \
+            usr/sbin/nfsiostat \
+            usr/sbin/rpc.gssd \
+            usr/sbin/nfsidmap \
+            usr/sbin/blkmapd \
+            usr/sbin/conntrackd \
+            usr/sbin/nfct \
+            usr/sbin/nfsstat \
+            usr/sbin/mountstats \
+            usr/sbin/setcap \
+            usr/sbin/exportfs \
+            usr/sbin/update-ca-certificates \
+            usr/sbin/capsh \
+            usr/sbin/getcap \
+            usr/sbin/rpcdebug \
+            usr/sbin/start-statd \
+            usr/sbin/getpcaps \
+            usr/sbin/sm-notify \
+            usr/share/aclocal \
+            usr/share/apk \
+            usr/share/ca-certificates \
+            usr/share/man \
+            usr/share/misc && \
+            find usr/share/terminfo -type f -exec rm {} \; && \
+            ln -s xterm-color usr/share/terminfo/x/xterm-256color && \
+            rmdir usr/share/terminfo/* || true
+RUN rm -rf  bin/sh \
+            lib/apk \
+            lib/mdev \
+            sbin/ss \
+            sbin/routel \
+            sbin/*-compat* \
+            sbin/genl \
+            sbin/lnstat \
+            sbin/ifstat \
+            sbin/mkmntdirs \
+            sbin/nfsdcltrack \
+            sbin/rtacct \
+            sbin/nstat \
+            sbin/routef \
+            sbin/apk \
+            sbin/tc \
+            sbin/ifcfg \
+            sbin/setup-udev \
+            sbin/rtpr \
+            sbin/osd_login \
+            sbin/bridge \
+            sbin/rtmon && \
+    ln -s bash bin/sh && \
+    mkdir -p lib/modules
+
+RUN mv sbin/* bin/ && \
+    rmdir sbin && \
+    ln -s bin sbin
+
+RUN mkdir lib2 && \
+    mv usr/lib/* lib2/ && \
+    mv lib2/* lib/ && \
+    mv usr/bin/* bin/ && \
+    mv usr/sbin/* bin/ && \
+    mv usr/share . && \
+    rm -rf usr lib2 && \
+    for i in $(ls -l bin | grep usr/bin/coreutils | awk '{print $(NF-2)}'); do \
+        rm bin/$i && ln -s coreutils bin/$i \
+    ;done && \
+    find -L bin -type l -exec rm {} \; -print
+
+RUN apk add upx && \
+    upx $(find bin -type f -executable \! -name coreutils) || true
+
+RUN echo '#### LAYOUT #####' && \
+    find -type d && \
+    echo '#### BIN #####' && \
+    find bin -type f -executable && \
+    du -x -s -h
+
+RUN tar cf ../rootfs.tar * && \
+    ls -la ../rootfs.tar
+
+CMD ["sh"]
+
+### BUILD IMAGE ###
+FROM golang:1.11-alpine AS gobuild
+RUN apk -U add git gcc linux-headers musl-dev make libseccomp libseccomp-dev bash
+RUN rm -f /bin/sh && ln -s /bin/bash /bin/sh
+
+### CNI ###
+FROM gobuild AS cni
+RUN mkdir -p $GOPATH/src/github.com/containernetworking && \
+    cd $GOPATH/src/github.com/containernetworking && \
+    git clone https://github.com/ibuildthecloud/plugins.git && \
+    cd plugins && \
+    git checkout 9810b7d5137b171c4e07ce59bb18be9feccec557
+RUN go build -buildmode=pie -ldflags -s -o /usr/bin/cni github.com/containernetworking/plugins
+
+### RUNC ###
+FROM gobuild AS runc
+RUN go get -d github.com/opencontainers/runc && \
+    git -C $GOPATH/src/github.com/opencontainers/runc checkout -b build v1.0.0-rc5
+WORKDIR $GOPATH/src/github.com/opencontainers/runc
+RUN make runc && \
+    cp runc /usr/bin/
+
+### CONTAINERD ###
+FROM gobuild AS containerd
+RUN go get -d github.com/containerd/containerd && \
+    git -C $GOPATH/src/github.com/containerd/containerd checkout -b build v1.1.4
+WORKDIR $GOPATH/src/github.com/containerd/containerd
+RUN sed -i -e '/aufs/d' -e '/zfs/d' cmd/containerd/builtins_linux.go
+RUN make BUILDTAGS="apparmor seccomp no_btrfs netgo osusergo" bin/containerd bin/containerd-shim && \
+    cp bin/containerd bin/containerd-shim /usr/bin/
+
+### AGENT ###
+FROM gobuild AS agent
+ADD /build/vendor.tar $GOPATH/src/github.com/rancher/rio/
+COPY /agent $GOPATH/src/github.com/rancher/rio/agent
+WORKDIR $GOPATH/src/github.com/rancher/rio
+RUN go build -buildmode=pie -tags k3s -ldflags -s -o /usr/bin/agent ./agent
+
+### ASSEMBLE IMAGE ###
+FROM gobuild
+
+RUN apk add -U squashfs-tools
+
+COPY --from=k3s-build /usr/src/rootfs.tar /usr/src/rootfs.tar
+RUN mkdir /usr/src/image && \
+    tar xf /usr/src/rootfs.tar -C /usr/src/image
+
+COPY --from=runc /usr/bin/runc /usr/src/image/bin/runc
+RUN strip --strip-unneeded /usr/src/image/bin/runc
+
+COPY --from=agent /usr/bin/agent /usr/src/image/bin/
+RUN strip --strip-unneeded /usr/src/image/bin/agent
+
+COPY --from=containerd /usr/bin/containerd-shim /usr/bin/containerd /usr/src/image/bin/
+#COPY containerd /usr/src/image/bin
+RUN strip --strip-unneeded /usr/src/image/bin/containerd /usr/src/image/bin/containerd-shim
+
+RUN mkdir -p /usr/src/image/share/cni/bin
+COPY --from=cni /usr/bin/cni /usr/src/image/share/cni/bin
+RUN cd /usr/src/image/share/cni/bin && \
+    for i in ./bridge ./flannel ./host-local ./loopback ./portmap; do \
+        ln -s cni $i \
+    ;done
+
+COPY image/init /usr/src/image/init
+
+RUN mksquashfs /usr/src/image main.squashfs

image/build

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+rm -rf main.squashfs
+
+pushd ..
+mkdir -p build
+echo Copying vendor
+tar chf build/vendor.tar \
+    --exclude vendor/github.com/coreos/etcd/cmd \
+    --exclude vendor/github.com/jteeuwen/go-bindata/testdata \
+    --exclude vendor/github.com/karrick/godirwalk/testdata \
+    vendor
+echo Running agent docker build
+docker build -t bb -f image/Dockerfile .
+popd
+
+docker run --rm bb cat main.squashfs > main.squashfs

image/init

@@ -0,0 +1,147 @@
+#!/usr/bin/sh
+set -e
+
+if [ "$ENTER_DEBUG" = true ]; then
+    set -x
+fi
+
+layout()
+{
+    mount --make-rshared /
+
+    mkdir -p /proc
+    mount -t proc -o nodev,nosuid,noexec,relatime none /proc
+
+    for i in cache empty lib local local log opt spool tmp; do
+        mkdir -p /var/$i
+    done
+
+    for i in run dev home mnt media opt root lib/modules lib/firmware var/lib/docker; do
+        if [ -d /.root/$i ]; then
+            mkdir -p /$i
+            mount --rbind /.root/$i /$i
+        fi
+    done
+
+    if [ -L /.root/var/run ]; then
+        ln -sf /run /var/run
+    else
+        mkdir -p /var/run
+        mount --rbind /.root/var/run /var/run
+    fi
+
+    mkdir -p $K3S_DATA_DIR
+    mount --rbind /.root/$K3S_DATA_DIR /$K3S_DATA_DIR
+
+    mkdir -p /run/k3s/containerd
+}
+
+sysfs()
+{
+    mkdir -p /sys
+    mount -t sysfs none /sys
+
+    mount -t securityfs -o noexec,nosuid,nodev none /sys/kernel/security 2>/dev/null|| true
+    mount -t configfs -o noexec,nosuid,nodev none /sys/kernel/config 2>/dev/null || true
+    mount -t fusectl -o noexec,nosuid,nodev none /sys/fs/fuse/connections 2>/dev/null || true
+    mount -t binfmt_misc -o noexec,nosuid,nodev none /proc/sys/fs/binfmt_misc 2>/dev/null || true
+}
+
+cgroups()
+{
+    mount -t tmpfs -o mode=755,size=10m none /sys/fs/cgroup
+    cat /proc/cgroups > /tmp/cgroups
+
+    for i in $(seq 0 20); do
+        t=""
+        l="$(cat /tmp/cgroups | grep '1$' | awk '{print $1 " " $2}' | grep -w $i | awk '{print $1}')"
+        for j in $l; do
+            if [ -z "$t" ]; then
+                t=$j
+            else
+                t="${t},$j"
+            fi
+        done
+
+        if [ -z "$t" ]; then
+            continue
+        fi
+
+        mkdir -p /sys/fs/cgroup/${t}
+        mount -t cgroup -o ${t},noexec,nosuid,nodev none /sys/fs/cgroup/${t}
+        mkdir -p /sys/fs/cgroup/${t}/k3s
+
+        for j in $l; do
+            if [ $j != $t ]; then
+                ln -s $t /sys/fs/cgroup/$j
+            fi
+        done
+    done
+
+    # good ole systemd
+    mkdir -p /sys/fs/cgroup/systemd
+    mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
+    mkdir -p /sys/fs/cgroup/systemd/k3s
+
+    rm /tmp/cgroups
+}
+
+mketc()
+{
+    mkdir -p /etc
+    cp -rf usr/etc/* /etc/
+    for i in  /.root/usr/lib/os-release /.root/etc/os-release; do
+        if [ -e $i ]; then
+            cp -f $i /etc/os-release
+        fi
+    done
+
+    if [ -e /.root/etc/machine-id ]; then
+        cp -f /.root/etc/machine-id /etc/machine-id
+    fi
+
+    hostname > /etc/hostname
+
+    cat > /etc/hosts << EOF
+127.0.0.1	localhost $NODE_NAME $(hostname)
+::1		localhost ip6-localhost ip6-loopback
+ff02::1		ip6-allnodes
+ff02::2		ip6-allrouters
+EOF
+
+    cat > /etc/resolv.conf << EOF
+nameserver 1.1.1.1
+EOF
+}
+
+nodename()
+{
+    if [ ! -e $K3S_DATA_DIR/id ]; then
+        echo $RANDOM > $K3S_DATA_DIR/id
+    fi
+    export NODE_NAME="$(hostname | cut -f1 -d.)-$(<$K3S_DATA_DIR/id)"
+}
+
+layout
+nodename
+mketc
+sysfs
+cgroups
+
+umount -l .root
+rmdir .root
+
+if [ "$1" = "--" ]; then
+    shift 1
+    exec "$@"
+fi
+
+exec env -i -- \
+    HOME=/root \
+    NODE_NAME=$NODE_NAME \
+    PATH=/sbin:/bin \
+    K3S_DATA_DIR=$K3S_DATA_DIR \
+    K3S_NODE_IP=$K3S_NODE_IP \
+    K3S_TOKEN=$K3S_TOKEN \
+    K3S_URL=$K3S_URL \
+    agent

main.go

@@ -0,0 +1,71 @@
+//go:generate go run types/codegen/cleanup/main.go
+//go:generate go run types/codegen/main.go
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/rancher/rio/cli/cmd/agent"
+	"github.com/rancher/rio/cli/cmd/kubectl"
+	"github.com/rancher/rio/cli/cmd/server"
+	"github.com/rancher/rio/cli/pkg/builder"
+	"github.com/rancher/rio/version"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
+
+	_ "github.com/rancher/rio/pkg/kubectl"
+)
+
+var (
+	appName = filepath.Base(os.Args[0])
+	debug   bool
+)
+
+func main() {
+	old := os.Args[0]
+	os.Args[0] = filepath.Base(os.Args[0])
+	if reexec.Init() {
+		return
+	}
+	os.Args[0] = old
+
+	app := cli.NewApp()
+	app.Name = appName
+	app.Usage = "Kubernetes, but small and simple"
+	app.Version = version.Version
+	cli.VersionPrinter = func(c *cli.Context) {
+		fmt.Printf("%s version %s\n", app.Name, app.Version)
+	}
+	app.Flags = []cli.Flag{
+		cli.BoolFlag{
+			Name:        "debug",
+			Usage:       "Turn on debug logs",
+			Destination: &debug,
+		},
+	}
+
+	app.Commands = []cli.Command{
+		server.ServerCommand,
+		builder.Command(&agent.Agent{},
+			"Run node agent",
+			appName+" agent [OPTIONS]",
+			""),
+
+		kubectl.NewKubectlCommand(),
+	}
+	app.Before = func(ctx *cli.Context) error {
+		if debug {
+			logrus.SetLevel(logrus.DebugLevel)
+		}
+		return nil
+	}
+
+	err := app.Run(os.Args)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}

package/Dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.8
+RUN apk -U --no-cache add ca-certificates
+
+COPY rio /
+WORKDIR /var/lib/rancher/rio

pkg/controller/tls/handler.go

@@ -0,0 +1,12 @@
+package tls
+
+import (
+	"context"
+
+	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
+)
+
+func Register(ctx context.Context) error {
+	clients := v1.ClientsFrom(ctx)
+	clients.ListenerConfig.OnChange(ctx)
+}

pkg/daemons/agent/agent.go

@@ -0,0 +1,98 @@
+package agent
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"github.com/rancher/rio/pkg/daemons/config"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/apiserver/pkg/util/logs"
+	app2 "k8s.io/kubernetes/cmd/kube-proxy/app"
+	"k8s.io/kubernetes/cmd/kubelet/app"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+)
+
+func Agent(config *config.Agent) error {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	prepare(config)
+
+	kubelet(config)
+	kubeProxy(config)
+
+	return nil
+}
+
+func prepare(config *config.Agent) {
+	if config.CNIBinDir == "" {
+		config.CNIBinDir = "/opt/cni/bin"
+	}
+	if config.CNIConfDir == "" {
+		config.CNIConfDir = "/etc/cni/net.d"
+	}
+}
+
+func kubeProxy(config *config.Agent) {
+	args := []string{
+		"--proxy-mode", "iptables",
+		"--healthz-bind-address", "127.0.0.1",
+		"--kubeconfig", config.KubeConfig,
+		"--cluster-cidr", config.ClusterCIDR.String(),
+	}
+	args = append(args, config.ExtraKubeletArgs...)
+
+	command := app2.NewProxyCommand()
+	command.SetArgs(args)
+	go func() {
+		err := command.Execute()
+		logrus.Fatalf("kube-proxy exited: %v", err)
+	}()
+}
+
+func kubelet(config *config.Agent) {
+	command := app.NewKubeletCommand(context.Background().Done())
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	args := []string{
+		"--healthz-bind-address", "127.0.0.1",
+		"--read-only-port", "0",
+		"--allow-privileged=true",
+		"--cluster-domain", "cluster.local",
+		"--kubeconfig", config.KubeConfig,
+		"--eviction-hard", "imagefs.available<5%,nodefs.available<5%",
+		"--eviction-minimum-reclaim", "imagefs.available=10%,nodefs.available=10%",
+		"--feature-gates=MountPropagation=true",
+		"--node-ip", config.NodeIP,
+		"--fail-swap-on=false",
+		"--cgroup-root", "/k3s",
+		"--cgroup-driver", "cgroupfs",
+		"--cni-conf-dir", config.CNIConfDir,
+		"--cni-bin-dir", config.CNIBinDir,
+	}
+	if len(config.ClusterDNS) > 0 {
+		args = append(args, "--cluster-dns", config.ClusterDNS.String())
+	}
+	if config.RuntimeSocket != "" {
+		args = append(args, "--container-runtime-endpoint", config.RuntimeSocket)
+	}
+	if config.ListenAddress != "" {
+		args = append(args, "--address", config.ListenAddress)
+	}
+	if config.CACertPath != "" {
+		args = append(args, "--anonymous-auth=false", "--client-ca-file", config.CACertPath)
+	}
+	if config.NodeName != "" {
+		args = append(args, "--hostname-override", config.NodeName)
+	}
+	args = append(args, config.ExtraKubeletArgs...)
+
+	command.SetArgs(args)
+
+	go func() {
+		logrus.Fatalf("kubelet exited: %v", command.Execute())
+	}()
+}

pkg/daemons/config/types.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"crypto/tls"
+	"net"
+	"net/http"
+
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+)
+
+type Node struct {
+	Docker        bool
+	NoFlannel     bool
+	NoCoreDNS     bool
+	LocalAddress  string
+	AgentConfig   Agent
+	CACerts       []byte
+	ServerAddress string
+	Certificate   *tls.Certificate
+}
+
+type Agent struct {
+	NodeName           string
+	ClusterCIDR        net.IPNet
+	ClusterDNS         net.IP
+	KubeConfig         string
+	NodeIP             string
+	RuntimeSocket      string
+	ListenAddress      string
+	CACertPath         string
+	CNIBinDir          string
+	CNIConfDir         string
+	ExtraKubeletArgs   []string
+	ExtraKubeProxyArgs []string
+}
+
+type Control struct {
+	AdvertisePort         int
+	ListenPort            int
+	ClusterIPRange        *net.IPNet
+	ServiceIPRange        *net.IPNet
+	DataDir               string
+	ETCDEndpoints         []string
+	ETCDKeyFile           string
+	ETCDCertFile          string
+	ETCDCAFile            string
+	NoScheduler           bool
+	ExtraAPIArgs          []string
+	ExtraControllerArgs   []string
+	ExtraSchedulerAPIArgs []string
+	NodeConfig            Node
+
+	Runtime *ControlRuntime `json:"-"`
+}
+
+type ControlRuntime struct {
+	TLSCert          string
+	TLSKey           string
+	TLSCA            string
+	TLSCAKey         string
+	TokenCA          string
+	TokenCAKey       string
+	ServiceKey       string
+	PasswdFile       string
+	KubeConfigSystem string
+
+	NodeCert      string
+	NodeKey       string
+	ClientToken   string
+	NodeToken     string
+	Handler       http.Handler
+	Tunnel        http.Handler
+	Authenticator authenticator.Request
+}

pkg/daemons/control/server.go

@@ -0,0 +1,560 @@
+package control
+
+import (
+	"context"
+	cryptorand "crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/csv"
+	"encoding/hex"
+	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rancher/rio/pkg/daemons/config"
+
+	_ "github.com/mattn/go-sqlite3" // sqlite
+	"github.com/sirupsen/logrus"
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
+	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
+	sapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	"k8s.io/kubernetes/pkg/kubeapiserver/authorizer/modes"
+	"k8s.io/kubernetes/pkg/master"
+	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
+	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+)
+
+var (
+	localhostIP        = net.ParseIP("127.0.0.1")
+	kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1
+clusters:
+- cluster:
+    server: {{.URL}}
+    certificate-authority-data: {{.CACert}}
+  name: local
+contexts:
+- context:
+    cluster: local
+    namespace: default
+    user: user
+  name: Default
+current-context: Default
+kind: Config
+preferences: {}
+users:
+- name: user
+  user:
+    username: {{.User}}
+    password: {{.Password}}
+`))
+)
+
+func Server(ctx context.Context, cfg *config.Control) error {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	runtime := &config.ControlRuntime{}
+	cfg.Runtime = runtime
+
+	if err := prepare(cfg, runtime); err != nil {
+		return err
+	}
+
+	cfg.Runtime.Tunnel = setupTunnel()
+
+	auth, handler, err := apiServer(ctx, cfg, runtime)
+	if err != nil {
+		return err
+	}
+
+	runtime.Handler = handler
+	runtime.Authenticator = auth
+
+	if !cfg.NoScheduler {
+		scheduler(cfg, runtime)
+	}
+
+	controllerManager(cfg, runtime)
+
+	return nil
+}
+
+func controllerManager(config *config.Control, runtime *config.ControlRuntime) {
+	args := []string{
+		"--kubeconfig", runtime.KubeConfigSystem,
+		"--leader-elect=true",
+		"--service-account-private-key-file", runtime.ServiceKey,
+		"--allocate-node-cidrs",
+		"--cluster-cidr", config.ClusterIPRange.String(),
+		"--root-ca-file", runtime.TokenCA,
+		"--port", "0",
+		"--secure-port", "0",
+	}
+	args = append(args, config.ExtraControllerArgs...)
+	command := cmapp.NewControllerManagerCommand()
+	command.SetArgs(args)
+
+	go func() {
+		logrus.Infof("Running kube-controller-manager %s", argString(args))
+		logrus.Fatalf("controller-manager exited: %v", command.Execute())
+	}()
+}
+
+type argString []string
+
+func (a argString) String() string {
+	b := strings.Builder{}
+	for _, s := range a {
+		if b.Len() > 0 {
+			b.WriteString(" ")
+		}
+		b.WriteString(s)
+	}
+	return b.String()
+}
+
+func scheduler(config *config.Control, runtime *config.ControlRuntime) {
+	args := []string{
+		"--kubeconfig", runtime.KubeConfigSystem,
+		"--port", "0",
+		"--secure-port", "0",
+	}
+	args = append(args, config.ExtraSchedulerAPIArgs...)
+	command := sapp.NewSchedulerCommand()
+	command.SetArgs(args)
+
+	go func() {
+		logrus.Infof("Running kube-scheduler %s", argString(args))
+		logrus.Fatalf("scheduler exited: %v", command.Execute())
+	}()
+}
+
+func apiServer(ctx context.Context, config *config.Control, runtime *config.ControlRuntime) (authenticator.Request, http.Handler, error) {
+	var args []string
+
+	if len(config.ETCDEndpoints) > 0 {
+		args = append(args, "--storage-backend", "etcd3")
+		args = append(args, "--etcd-servers", strings.Join(config.ETCDEndpoints, ","))
+		if config.ETCDKeyFile != "" {
+			args = append(args, "--etcd-keyfile", config.ETCDKeyFile)
+		}
+		if config.ETCDCAFile != "" {
+			args = append(args, "--etcd-cafile", config.ETCDCAFile)
+		}
+		if config.ETCDCertFile != "" {
+			args = append(args, "--etcd-certfile", config.ETCDCertFile)
+		}
+	}
+
+	args = append(args, "--allow-privileged=true")
+	args = append(args, "--authorization-mode", strings.Join([]string{modes.ModeNode, modes.ModeRBAC}, ","))
+	args = append(args, "--service-account-signing-key-file", runtime.ServiceKey)
+	args = append(args, "--service-cluster-ip-range", config.ServiceIPRange.String())
+	args = append(args, "--advertise-port", strconv.Itoa(config.AdvertisePort))
+	args = append(args, "--advertise-address", localhostIP.String())
+	args = append(args, "--insecure-port", "0")
+	args = append(args, "--secure-port", strconv.Itoa(config.ListenPort))
+	args = append(args, "--bind-address", localhostIP.String())
+	args = append(args, "--tls-cert-file", runtime.TLSCert)
+	args = append(args, "--tls-private-key-file", runtime.TLSKey)
+	args = append(args, "--service-account-key-file", runtime.ServiceKey)
+	args = append(args, "--service-account-issuer", "k3s")
+	args = append(args, "--api-audiences", "unknown")
+	args = append(args, "--basic-auth-file", runtime.PasswdFile)
+	//args = append(args, "--kubelet-client-certificate", runtime.NodeCert)
+	//args = append(args, "--kubelet-client-key", runtime.NodeKey)
+
+	args = append(args, config.ExtraAPIArgs...)
+	command := app.NewAPIServerCommand(ctx.Done())
+	command.SetArgs(args)
+
+	go func() {
+		logrus.Infof("Running kube-apiserver %s", argString(args))
+		logrus.Fatalf("apiserver exited: %v", command.Execute())
+	}()
+
+	startupConfig := <-app.StartupConfig
+
+	return startupConfig.Authenticator, startupConfig.Handler, nil
+}
+
+func defaults(config *config.Control) {
+	if config.ClusterIPRange == nil {
+		_, clusterIPNet, _ := net.ParseCIDR("10.42.0.0/16")
+		config.ClusterIPRange = clusterIPNet
+	}
+
+	if config.ServiceIPRange == nil {
+		_, serviceIPNet, _ := net.ParseCIDR("10.43.0.0/16")
+		config.ServiceIPRange = serviceIPNet
+	}
+
+	if config.AdvertisePort == 0 {
+		config.AdvertisePort = 6445
+	}
+
+	if config.ListenPort == 0 {
+		config.ListenPort = 6444
+	}
+
+	if config.DataDir == "" {
+		config.DataDir = "./management-state"
+	}
+}
+
+func prepare(config *config.Control, runtime *config.ControlRuntime) error {
+	var err error
+
+	defaults(config)
+
+	if _, err := os.Stat(config.DataDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(config.DataDir, 0700); err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	config.DataDir, err = filepath.Abs(config.DataDir)
+	if err != nil {
+		return err
+	}
+
+	os.MkdirAll(path.Join(config.DataDir, "tls"), 0700)
+	os.MkdirAll(path.Join(config.DataDir, "cred"), 0700)
+
+	name := "localhost"
+	runtime.TLSCert = path.Join(config.DataDir, "tls", name+".crt")
+	runtime.TLSKey = path.Join(config.DataDir, "tls", name+".key")
+	runtime.TLSCA = path.Join(config.DataDir, "tls", "ca.crt")
+	runtime.TLSCAKey = path.Join(config.DataDir, "tls", "ca.key")
+	runtime.TokenCA = path.Join(config.DataDir, "tls", "token-ca.crt")
+	runtime.TokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key")
+	runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key")
+	runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd")
+	runtime.KubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml")
+	runtime.NodeKey = path.Join(config.DataDir, "tls", "token-node.key")
+	runtime.NodeCert = path.Join(config.DataDir, "tls", "token-node.crt")
+
+	regen := false
+	if _, err := os.Stat(runtime.TLSCA); err != nil {
+		regen = true
+		if err := genCA(runtime); err != nil {
+			return err
+		}
+	}
+
+	if err := genServiceAccount(runtime); err != nil {
+		return err
+	}
+
+	if err := genTLS(regen, config, runtime); err != nil {
+		return err
+	}
+
+	if err := genTokenTLS(config, runtime); err != nil {
+		return err
+	}
+
+	if err := genUsers(config, runtime); err != nil {
+		return err
+	}
+
+	return readTokens(runtime)
+}
+
+func readTokens(runtime *config.ControlRuntime) error {
+	f, err := os.Open(runtime.PasswdFile)
+	if err != nil {
+		return err
+	}
+	reader := csv.NewReader(f)
+	reader.FieldsPerRecord = -1
+
+	for {
+		record, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		if len(record) < 2 {
+			continue
+		}
+
+		switch record[1] {
+		case "node":
+			runtime.NodeToken = "node:" + record[0]
+		case "admin":
+			runtime.ClientToken = "admin:" + record[0]
+		}
+	}
+
+	return nil
+}
+
+func genUsers(config *config.Control, runtime *config.ControlRuntime) error {
+	if s, err := os.Stat(runtime.PasswdFile); err == nil && s.Size() > 0 {
+		return nil
+	}
+
+	adminToken, err := getToken()
+	if err != nil {
+		return err
+	}
+	systemToken, err := getToken()
+	if err != nil {
+		return err
+	}
+	nodeToken, err := getToken()
+	if err != nil {
+		return err
+	}
+
+	passwd := fmt.Sprintf(`%s,admin,admin,system:masters
+%s,system,system,system:masters
+%s,node,node,system:masters
+`, adminToken, systemToken, nodeToken)
+
+	caCertBytes, err := ioutil.ReadFile(runtime.TLSCA)
+	if err != nil {
+		return err
+	}
+
+	caCert := base64.StdEncoding.EncodeToString(caCertBytes)
+
+	if err := kubeConfig(runtime.KubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert,
+		"system", systemToken); err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(runtime.PasswdFile, []byte(passwd), 0600)
+}
+
+func getToken() (string, error) {
+	token := make([]byte, 16, 16)
+	_, err := cryptorand.Read(token)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(token), err
+}
+
+func genTokenTLS(config *config.Control, runtime *config.ControlRuntime) error {
+	regen := false
+	if _, err := os.Stat(runtime.TokenCA); err != nil {
+		regen = true
+		if err := genTokenCA(runtime); err != nil {
+			return err
+		}
+	}
+
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: "kubernetes",
+		AltNames: certutil.AltNames{
+			DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
+			IPs:      []net.IP{net.ParseIP("127.0.0.1"), apiServerServiceIP},
+		},
+		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+	}
+
+	if _, err := os.Stat(runtime.NodeCert); err == nil && !regen {
+		return nil
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(runtime.TokenCAKey)
+	if err != nil {
+		return err
+	}
+
+	caBytes, err := ioutil.ReadFile(runtime.TokenCA)
+	if err != nil {
+		return err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return err
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(runtime.NodeKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(runtime.NodeCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+}
+
+func genTLS(regen bool, config *config.Control, runtime *config.ControlRuntime) error {
+	if !regen {
+		_, certErr := os.Stat(runtime.TLSCert)
+		_, keyErr := os.Stat(runtime.TLSKey)
+		if certErr == nil && keyErr == nil {
+			return nil
+		}
+	}
+
+	_, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange)
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: "localhost",
+		AltNames: certutil.AltNames{
+			DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"},
+			IPs:      []net.IP{apiServerServiceIP, localhostIP},
+		},
+		Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+
+	caKeyBytes, err := ioutil.ReadFile(runtime.TLSCAKey)
+	if err != nil {
+		return err
+	}
+
+	caBytes, err := ioutil.ReadFile(runtime.TLSCA)
+	if err != nil {
+		return err
+	}
+
+	caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes)
+	if err != nil {
+		return err
+	}
+
+	caCert, err := certutil.ParseCertsPEM(caBytes)
+	if err != nil {
+		return err
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey))
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(runtime.TLSKey, certutil.EncodePrivateKeyPEM(key)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(runtime.TLSCert, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...))
+}
+
+func genServiceAccount(runtime *config.ControlRuntime) error {
+	_, keyErr := os.Stat(runtime.ServiceKey)
+	if keyErr == nil {
+		return nil
+	}
+
+	key, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	return certutil.WriteKey(runtime.ServiceKey, certutil.EncodePrivateKeyPEM(key))
+}
+
+func genTokenCA(runtime *config.ControlRuntime) error {
+	caKey, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("%s-ca@%d", "k3s-token", time.Now().Unix()),
+	}
+
+	cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(runtime.TokenCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(runtime.TokenCA, certutil.EncodeCertPEM(cert))
+}
+
+func genCA(runtime *config.ControlRuntime) error {
+	caKey, err := certutil.NewPrivateKey()
+	if err != nil {
+		return err
+	}
+
+	cfg := certutil.Config{
+		CommonName: fmt.Sprintf("%s-ca@%d", "k3s", time.Now().Unix()),
+	}
+
+	cert, err := certutil.NewSelfSignedCACert(cfg, caKey)
+	if err != nil {
+		return err
+	}
+
+	if err := certutil.WriteKey(runtime.TLSCAKey, certutil.EncodePrivateKeyPEM(caKey)); err != nil {
+		return err
+	}
+
+	return certutil.WriteCert(runtime.TLSCA, certutil.EncodeCertPEM(cert))
+}
+
+func kubeConfig(dest, url, cert, user, password string) error {
+	data := struct {
+		URL      string
+		CACert   string
+		User     string
+		Password string
+	}{
+		URL:      url,
+		CACert:   cert,
+		User:     user,
+		Password: password,
+	}
+
+	output, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	return kubeconfigTemplate.Execute(output, &data)
+}

pkg/daemons/control/tunnel.go

@@ -0,0 +1,50 @@
+package control
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/rancher/norman/pkg/kv"
+	"github.com/rancher/norman/pkg/remotedialer"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/kubernetes/cmd/kube-apiserver/app"
+)
+
+func setupTunnel() http.Handler {
+	tunnelServer := remotedialer.New(authorizer, remotedialer.DefaultErrorWriter)
+	setupProxyDialer(tunnelServer)
+	return tunnelServer
+}
+
+func setupProxyDialer(tunnelServer *remotedialer.Server) {
+	app.DefaultProxyDialerFn = utilnet.DialFunc(func(_ context.Context, network, address string) (net.Conn, error) {
+		_, port, _ := net.SplitHostPort(address)
+		addr := "127.0.0.1"
+		if port != "" {
+			addr += ":" + port
+		}
+		nodeName, _ := kv.Split(address, ":")
+		return tunnelServer.Dial(nodeName, 15*time.Second, "tcp", addr)
+	})
+}
+
+func authorizer(req *http.Request) (clientKey string, authed bool, err error) {
+	user, ok := request.UserFrom(req.Context())
+	if !ok {
+		return "", false, nil
+	}
+
+	if user.GetName() != "node" {
+		return "", false, nil
+	}
+
+	nodeName := req.Header.Get("X-K3s-NodeName")
+	if nodeName == "" {
+		return "", false, nil
+	}
+
+	return nodeName, true, nil
+}

pkg/enterchroot/enter.go

@@ -0,0 +1,254 @@
+package enterchroot
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	losetup "gopkg.in/freddierice/go-losetup.v1"
+)
+
+const (
+	magic = "_SQMAGIC_"
+)
+
+var (
+	symlinks = []string{"lib", "bin", "sbin", "lib64"}
+)
+
+func init() {
+	reexec.Register("enter-root", enter)
+}
+
+func enter() {
+	if os.Getenv("ENTER_DEBUG") == "true" {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
+	logrus.Debug("Running bootstrap")
+	err := run(os.Getenv("ENTER_DATA"))
+	if err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func Mount(dataDir string, stdout, stderr io.Writer, args []string) error {
+	if logrus.GetLevel() >= logrus.DebugLevel {
+		os.Setenv("ENTER_DEBUG", "true")
+	}
+
+	root, offset, err := findRoot()
+	if err != nil {
+		return err
+	}
+
+	os.Setenv("ENTER_DATA", dataDir)
+	os.Setenv("ENTER_ROOT", root)
+
+	logrus.Debugf("Using data [%s] root [%s]", dataDir, root)
+
+	stat, err := os.Stat(root)
+	if err != nil {
+		return fmt.Errorf("failed to find %s: %v", root, err)
+	}
+
+	if !stat.IsDir() {
+		logrus.Debugf("Attaching file [%s] offset [%d]", root, offset)
+		dev, err := losetup.Attach(root, offset, true)
+		if err != nil {
+			return errors.Wrap(err, "creating loopback device")
+		}
+		defer dev.Detach()
+		os.Setenv("ENTER_DEVICE", dev.Path())
+
+		go func() {
+			// Assume that after 3 seconds loop back device has been mounted
+			time.Sleep(3 * time.Second)
+			info, err := dev.GetInfo()
+			if err != nil {
+				return
+			}
+
+			info.Flags |= losetup.FlagsAutoClear
+			err = dev.SetInfo(info)
+			if err != nil {
+				return
+			}
+		}()
+	}
+
+	logrus.Debugf("Running enter-root %v", args)
+	cmd := &exec.Cmd{
+		Path: reexec.Self(),
+		Args: append([]string{"enter-root"}, args...),
+		SysProcAttr: &syscall.SysProcAttr{
+			//Cloneflags:   syscall.CLONE_NEWPID | syscall.CLONE_NEWUTS | syscall.CLONE_NEWIPC,
+			Unshareflags: syscall.CLONE_NEWNS,
+			Pdeathsig:    syscall.SIGKILL,
+		},
+		Stdout: stdout,
+		Stdin:  os.Stdin,
+		Stderr: stderr,
+		Env:    os.Environ(),
+	}
+	return cmd.Run()
+}
+
+func findRoot() (string, uint64, error) {
+	root := os.Getenv("ENTER_ROOT")
+	if root != "" {
+		return root, 0, nil
+	}
+
+	for _, suffix := range []string{".root", ".squashfs"} {
+		test := os.Args[0] + suffix
+		if _, err := os.Stat(test); err == nil {
+			return test, 0, nil
+		}
+	}
+
+	return inFile()
+}
+
+func inFile() (string, uint64, error) {
+	f, err := os.Open(reexec.Self())
+	if err != nil {
+		return "", 0, err
+	}
+	defer f.Close()
+
+	buf := make([]byte, 8192)
+	test := []byte(strings.ToLower(magic))
+	testLength := len(test)
+	offset := uint64(0)
+	found := 0
+
+	for {
+		n, err := f.Read(buf)
+		if err == io.EOF && n == 0 {
+			break
+		} else if err != nil {
+			return "", 0, err
+		}
+
+		for _, b := range buf[:n] {
+			if b == test[found] {
+				found++
+				if found == testLength {
+					return reexec.Self(), offset + 1, nil
+				}
+			} else {
+				found = 0
+			}
+			offset++
+		}
+	}
+
+	return "", 0, fmt.Errorf("failed to find image in file %s", os.Args[0])
+}
+
+func run(data string) error {
+	os.MkdirAll(data, 0755)
+
+	if err := mount.Mount("tmpfs", data, "tmpfs", ""); err != nil {
+		return errors.Wrapf(err, "remounting data %s", data)
+	}
+
+	root := os.Getenv("ENTER_ROOT")
+	device := os.Getenv("ENTER_DEVICE")
+
+	logrus.Debugf("Using root %s %s", root, device)
+
+	usr := filepath.Join(data, "usr")
+	dotRoot := filepath.Join(data, ".root")
+
+	for _, d := range []string{usr, dotRoot} {
+		if err := os.MkdirAll(d, 0755); err != nil {
+			return fmt.Errorf("failed to make dir %s: %v", data, err)
+		}
+	}
+
+	if device == "" {
+		logrus.Debugf("Bind mounting %s to %s", root, usr)
+		if err := mount.Mount(root, usr, "none", "bind"); err != nil {
+			return fmt.Errorf("failed to bind mount")
+		}
+	} else {
+		logrus.Debugf("Mounting squashfs %s to %s", device, usr)
+		squashErr := checkSquashfs()
+		if err := mount.Mount(device, usr, "squashfs", "ro"); err != nil {
+			err = errors.Wrap(err, "mounting squashfs")
+			if squashErr != nil {
+				err = errors.Wrap(err, squashErr.Error())
+			}
+			return err
+		}
+	}
+
+	if err := os.Chdir(data); err != nil {
+		return err
+	}
+
+	for _, p := range symlinks {
+		if _, err := os.Lstat(p); os.IsNotExist(err) {
+			if err := os.Symlink(filepath.Join("usr", p), p); err != nil {
+				return errors.Wrapf(err, "failed to symlink %s", p)
+			}
+		}
+	}
+
+	logrus.Debugf("pivoting to . .root")
+	if err := syscall.PivotRoot(".", ".root"); err != nil {
+		return errors.Wrap(err, "pivot_root failed")
+	}
+
+	if err := mount.ForceMount("", ".", "none", "rprivate"); err != nil {
+		return errors.Wrapf(err, "making . private %s", data)
+	}
+
+	if err := syscall.Chroot("/"); err != nil {
+		return err
+	}
+
+	if err := os.Chdir("/"); err != nil {
+		return err
+	}
+
+	if _, err := os.Stat("/usr/init"); err != nil {
+		return errors.Wrap(err, "failed to find /usr/init")
+	}
+
+	return syscall.Exec("/usr/init", os.Args, os.Environ())
+}
+
+func checkSquashfs() error {
+	if !inProcFS() {
+		exec.Command("modprobe", "squashfs").Run()
+	}
+
+	if !inProcFS() {
+		return errors.New("This kernel does not support squashfs, please enable. " +
+			"On Fedora you may need to run \"dnf install kernel-modules-$(uname -r)\"")
+	}
+
+	return nil
+}
+
+func inProcFS() bool {
+	bytes, err := ioutil.ReadFile("/proc/filesystems")
+	if err != nil {
+		logrus.Errorf("Failed to read /proc/filesystems: %v", err)
+		return false
+	}
+	return strings.Contains(string(bytes), "squashfs")
+}

pkg/kubectl/main.go

@@ -0,0 +1,52 @@
+package kubectl
+
+import (
+	goflag "flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/docker/docker/pkg/reexec"
+	"github.com/rancher/rio/pkg/server"
+	"github.com/spf13/pflag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/kubernetes/pkg/kubectl/cmd"
+)
+
+func init() {
+	reexec.Register("kubectl", Main)
+}
+
+func Main() {
+	kubenv := os.Getenv("KUBECONFIG")
+	if kubenv == "" {
+		config, err := server.HomeKubeConfig()
+		if _, serr := os.Stat(config); err == nil && serr == nil {
+			os.Setenv("KUBECONFIG", config)
+		}
+	}
+
+	main()
+}
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+
+	command := cmd.NewDefaultKubectlCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := command.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

pkg/server/auth.go

@@ -0,0 +1,41 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/rancher/rio/pkg/daemons/config"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+func doAuth(serverConfig *config.Control, next http.Handler, rw http.ResponseWriter, req *http.Request) {
+	if serverConfig == nil || serverConfig.Runtime.Authenticator == nil {
+		next.ServeHTTP(rw, req)
+		return
+	}
+
+	resp, ok, err := serverConfig.Runtime.Authenticator.AuthenticateRequest(req)
+	if err != nil {
+		logrus.Errorf("failed to authenticate request: %v", err)
+		rw.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if !ok || resp.User.GetName() != "node" {
+		rw.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+
+	ctx := request.WithUser(req.Context(), resp.User)
+	req = req.WithContext(ctx)
+	next.ServeHTTP(rw, req)
+}
+
+func authMiddleware(serverConfig *config.Control) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+			doAuth(serverConfig, next, rw, req)
+		})
+	}
+}

pkg/server/router.go

@@ -0,0 +1,72 @@
+package server
+
+import (
+	"net/http"
+
+	"github.com/rancher/rio/pkg/daemons/config"
+
+	"k8s.io/apimachinery/pkg/util/json"
+
+	"github.com/gorilla/mux"
+)
+
+type CACertsGetter func() (string, error)
+
+func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CACertsGetter) http.Handler {
+	authed := mux.NewRouter()
+	authed.Use(authMiddleware(serverConfig))
+	authed.NotFoundHandler = serverConfig.Runtime.Handler
+	authed.Path("/v1-k3s/connect").Handler(tunnel)
+	authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig))
+	authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig))
+	authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig))
+
+	router := mux.NewRouter()
+	router.NotFoundHandler = authed
+	router.Path("/cacerts").Handler(cacerts(cacertsGetter))
+
+	return router
+}
+
+func cacerts(getter CACertsGetter) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		content, err := getter()
+		if err != nil {
+			resp.WriteHeader(http.StatusInternalServerError)
+			resp.Write([]byte(err.Error()))
+		}
+		resp.Header().Set("content-type", "text/plain")
+		resp.Write([]byte(content))
+	})
+}
+
+func nodeCrt(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		http.ServeFile(resp, req, server.Runtime.NodeCert)
+	})
+}
+
+func nodeKey(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		http.ServeFile(resp, req, server.Runtime.NodeKey)
+	})
+}
+
+func configHandler(server *config.Control) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		if req.TLS == nil {
+			resp.WriteHeader(http.StatusNotFound)
+			return
+		}
+		resp.Header().Set("content-type", "application/json")
+		json.NewEncoder(resp).Encode(server)
+	})
+}

pkg/server/server.go

@@ -0,0 +1,230 @@
+package server
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rancher/norman"
+	"github.com/rancher/norman/pkg/clientaccess"
+	"github.com/rancher/norman/pkg/dynamiclistener"
+	"github.com/rancher/norman/pkg/resolvehome"
+	"github.com/rancher/norman/types"
+	"github.com/rancher/rio/pkg/daemons/config"
+	"github.com/rancher/rio/pkg/daemons/control"
+	"github.com/rancher/rio/pkg/tls"
+	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/net"
+)
+
+func resolveDataDir(dataDir string) (string, error) {
+	if dataDir == "" {
+		if os.Getuid() == 0 {
+			dataDir = "/var/lib/rancher/k3s"
+		} else {
+			dataDir = "${HOME}/.rancher/k3s"
+		}
+	}
+
+	dataDir = filepath.Join(dataDir, "server")
+	return resolvehome.Resolve(dataDir)
+}
+
+func StartServer(ctx context.Context, config *Config) error {
+	if err := setupDataDirAndChdir(&config.ControlConfig); err != nil {
+		return err
+	}
+
+	if err := control.Server(ctx, &config.ControlConfig); err != nil {
+		return errors.Wrap(err, "starting kubernetes")
+	}
+
+	certs, err := startNorman(ctx, &config.TLSConfig, &config.ControlConfig)
+	if err != nil {
+		return errors.Wrap(err, "starting tls server")
+	}
+
+	printTokens(certs, config.ControlConfig.NodeConfig.AgentConfig.NodeIP, &config.TLSConfig, &config.ControlConfig)
+
+	writeKubeConfig(certs, &config.TLSConfig, &config.ControlConfig)
+
+	return nil
+}
+
+func startNorman(ctx context.Context, tlsConfig *dynamiclistener.UserConfig, config *config.Control) (string, error) {
+	var (
+		err       error
+		tlsServer dynamiclistener.ServerInterface
+	)
+
+	tlsConfig.Handler = router(config, config.Runtime.Tunnel, func() (string, error) {
+		if tlsServer == nil {
+			return "", nil
+		}
+		return tlsServer.CACert()
+	})
+
+	normanConfig := &norman.Config{
+		Name:       "k3s",
+		KubeConfig: config.Runtime.KubeConfigSystem,
+		Clients: []norman.ClientFactory{
+			v1.Factory,
+		},
+		Schemas: []*types.Schemas{
+			v1.Schemas,
+		},
+		CRDs: map[*types.APIVersion][]string{
+			&v1.APIVersion: {
+				v1.ListenerConfigGroupVersionKind.Kind,
+			},
+		},
+		IgnoredKubeConfigEnv: true,
+		GlobalSetup: func(ctx context.Context) (context.Context, error) {
+			tlsServer, err = tls.NewServer(ctx, v1.ClientsFrom(ctx).ListenerConfig, *tlsConfig)
+			return ctx, err
+		},
+	}
+
+	ctx, _, err = normanConfig.Build(ctx, nil)
+	if err != nil {
+		return "", err
+	}
+
+	for {
+		certs, err := tlsServer.CACert()
+		if err != nil {
+			logrus.Infof("waiting to generate CA certs")
+			time.Sleep(time.Second)
+			continue
+		}
+		return certs, nil
+	}
+}
+
+func HomeKubeConfig() (string, error) {
+	return resolvehome.Resolve("${HOME}/.kube/k3s.yaml")
+}
+
+func printTokens(certs, advertiseIP string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
+	var (
+		nodeFile string
+	)
+
+	if advertiseIP == "" {
+		advertiseIP = "localhost"
+	}
+
+	if len(config.Runtime.NodeToken) > 0 {
+		p := filepath.Join(config.DataDir, "node-token")
+		if err := writeToken(config.Runtime.NodeToken, p, certs); err == nil {
+			logrus.Infof("Node token is available at %s", p)
+			nodeFile = p
+		}
+	}
+
+	if len(nodeFile) > 0 {
+		printToken(tlsConfig.HTTPSPort, advertiseIP, "To join node to cluster:", nodeFile, "agent")
+	}
+
+}
+
+func writeKubeConfig(certs string, tlsConfig *dynamiclistener.UserConfig, config *config.Control) {
+	clientToken := FormatToken(config.Runtime.ClientToken, certs)
+	url := fmt.Sprintf("https://localhost:%d", tlsConfig.HTTPSPort)
+	kubeConfig, err := HomeKubeConfig()
+	def := true
+	if err != nil {
+		kubeConfig = filepath.Join(config.DataDir, "kubeconfig-k3s.yaml")
+		def = false
+	}
+
+	if err = clientaccess.AgentAccessInfoToKubeConfig(kubeConfig, url, clientToken); err != nil {
+		logrus.Errorf("Failed to generate kubeconfig: %v", err)
+	}
+
+	logrus.Infof("Wrote kubeconfig %s", kubeConfig)
+	if def {
+		logrus.Infof("Run: %s kubectl", filepath.Base(os.Args[0]))
+	}
+}
+
+func setupDataDirAndChdir(config *config.Control) error {
+	var (
+		err error
+	)
+
+	config.DataDir, err = resolveDataDir(config.DataDir)
+	if err != nil {
+		return err
+	}
+
+	dataDir := config.DataDir
+
+	if err := os.MkdirAll(dataDir, 0700); err != nil {
+		return errors.Wrapf(err, "can not mkdir %s", dataDir)
+	}
+
+	if err := os.Chdir(dataDir); err != nil {
+		return errors.Wrapf(err, "can not chdir %s", dataDir)
+	}
+
+	return nil
+}
+
+func readTokenFile(file string) (string, error) {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(content)), nil
+}
+
+func printToken(httpsPort int, advertiseIP, prefix, file, cmd string) {
+	token, err := readTokenFile(file)
+	if err != nil {
+		logrus.Error(err)
+	}
+
+	ip := advertiseIP
+	if ip == "" {
+		hostIP, err := net.ChooseHostInterface()
+		if err != nil {
+			logrus.Error(err)
+		}
+		ip = hostIP.String()
+	}
+
+	logrus.Infof("%s rio %s -s https://%s:%d -t %s", prefix, cmd, ip, httpsPort, token)
+}
+
+func FormatToken(token string, certs string) string {
+	if len(token) == 0 {
+		return token
+	}
+
+	prefix := "K10"
+	if len(certs) > 0 {
+		digest := sha256.Sum256([]byte(certs))
+		prefix = "K10" + hex.EncodeToString(digest[:]) + "::"
+	}
+
+	return prefix + token
+}
+
+func writeToken(token, file, certs string) error {
+	if len(token) == 0 {
+		return nil
+	}
+
+	token = FormatToken(token, certs)
+	return ioutil.WriteFile(file, []byte(token+"\n"), 0600)
+}

pkg/server/types.go

@@ -0,0 +1,12 @@
+package server
+
+import (
+	"github.com/rancher/norman/pkg/dynamiclistener"
+	"github.com/rancher/rio/pkg/daemons/config"
+)
+
+type Config struct {
+	DisableAgent  bool
+	TLSConfig     dynamiclistener.UserConfig
+	ControlConfig config.Control
+}

pkg/tls/storage.go

@@ -0,0 +1,79 @@
+package tls
+
+import (
+	"context"
+
+	"github.com/rancher/norman/pkg/dynamiclistener"
+	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+const (
+	ns   = "kube-system"
+	name = "tls-config"
+)
+
+func NewServer(ctx context.Context, listenerClient v1.ListenerConfigClient, config dynamiclistener.UserConfig) (dynamiclistener.ServerInterface, error) {
+	storage := &listenerConfigStorage{
+		client: listenerClient,
+		cache:  listenerClient.Cache(),
+	}
+
+	server, err := dynamiclistener.NewServer(storage, config)
+	listenerClient.OnChange(ctx, "listen-config", func(obj *v1.ListenerConfig) (runtime.Object, error) {
+		return obj, server.Update(fromStorage(obj))
+	})
+
+	return server, err
+}
+
+type listenerConfigStorage struct {
+	cache  v1.ListenerConfigClientCache
+	client v1.ListenerConfigClient
+}
+
+func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dynamiclistener.ListenerStatus, error) {
+	if config == nil {
+		return nil, nil
+	}
+
+	obj, err := l.cache.Get(ns, name)
+	if errors.IsNotFound(err) {
+		ls := v1.NewListenerConfig(ns, name, v1.ListenerConfig{
+			Status: *config,
+		})
+
+		ls, err := l.client.Create(ls)
+		return fromStorage(ls), err
+	} else if err != nil {
+		return nil, err
+	}
+
+	obj = obj.DeepCopy()
+	obj.ResourceVersion = config.Revision
+	obj.Status = *config
+	obj.Status.Revision = ""
+
+	obj, err = l.client.Update(obj)
+	return fromStorage(obj), err
+}
+
+func (l *listenerConfigStorage) Get() (*dynamiclistener.ListenerStatus, error) {
+	obj, err := l.cache.Get(ns, name)
+	if errors.IsNotFound(err) {
+		obj, err = l.client.Get(ns, name, metav1.GetOptions{})
+	}
+	return fromStorage(obj), err
+}
+
+func fromStorage(obj *v1.ListenerConfig) *dynamiclistener.ListenerStatus {
+	if obj == nil {
+		return nil
+	}
+
+	copy := obj.DeepCopy()
+	copy.Status.Revision = obj.ResourceVersion
+	return &copy.Status
+}

scripts/build

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/version
+
+cd $(dirname $0)/..
+
+LDFLAGS="-X github.com/rancher/rio/version.Version=$VERSION -w -s"
+STATIC="-extldflags -static"
+STATIC_SQLITE="-extldflags '-static -lm -ldl -lz -lpthread'"
+
+cross()
+{
+    echo Building windows CLI
+    GOOS=windows GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-windows ./cli/main.go
+    echo Building mac CLI
+    GOOS=darwin GOARCH=amd64 go build -ldflags "$LDFLAGS" -o bin/rio-darwin ./cli/main.go
+}
+
+mkdir -p bin
+
+if [ -n "$CROSS" ]; then
+    cross
+fi
+
+echo Building incluster
+GOOS=linux GOARCH=amd64
+CGO_ENABLED=0 go build -ldflags "$LDFLAGS $STATIC" -o bin/rio-incluster
+
+echo Building cli full
+CGO_ENABLED=1 go build -tags "static_build libsqlite3 k8s ctr no_etcd netgo osusergo" -ldflags "$LDFLAGS $STATIC_SQLITE" -o bin/rio-full ./cli/main.go

scripts/ci

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+
+./validate
+./build
+./package
+./test

scripts/dev-agent.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/../bin
+
+# Prime sudo
+sudo echo Compiling CLI
+go build -tags "k8s no_etcd" -o rio-agent ../cli/main.go
+
+echo Building image and agent
+../image/build
+
+echo Running
+exec sudo ENTER_ROOT=../image/main.squashfs ./rio-agent --debug agent -s https://localhost:7443 -t $(<${HOME}/.rancher/rio/server/node-token)

scripts/dev-k8s-only-server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/../bin
+
+echo Running
+go run -tags "k8s no_etcd" ../cli/main.go --debug server --disable-controllers --disable-agent

scripts/dev-login.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/../bin
+
+rio login -s https://localhost:5443 -t $(<${HOME}/.rancher/rio/server/client-token)

scripts/dev-server.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/../bin
+
+echo Running
+go run -tags k3s ../cli/main.go --debug server --disable-agent

scripts/entry

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+mkdir -p bin dist
+if [ -e ./scripts/$1 ]; then
+    ./scripts/"$@"
+else
+    exec "$@"
+fi
+
+chown -R $DAPPER_UID:$DAPPER_GID .

scripts/package

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)
+
+./package-cli
+./package-image
+./package-tar

scripts/package-cli

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/version
+
+cd $(dirname $0)/../image
+./build
+
+cp ../bin/rio-full ../bin/rio
+echo -n "_sqmagic_" >> ../bin/rio
+cat main.squashfs >> ../bin/rio

scripts/package-image

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/version
+
+cd $(dirname $0)/../package
+
+TAG=${TAG:-${VERSION}${SUFFIX}}
+REPO=${REPO:-rancher}
+
+cp ../bin/rio ./rio
+
+IMAGE=${REPO}/rio:${TAG}
+docker build -t ${IMAGE} .
+mkdir -p ../dist
+echo ${IMAGE} > ../dist/images
+echo Built ${IMAGE}

scripts/package-tar

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+cd $(dirname $0)/..
+
+. ./scripts/version
+
+mkdir -p dist/artifacts
+
+tar cvzf dist/artifacts/rio-${VERSION}-linux-amd64.tar.gz -h bin/rio --xform='s!^bin!rio-'${VERSION}'-linux-amd64!'
+tar cvzf dist/artifacts/rio-${VERSION}-darwin.tar.gz bin/rio-darwin --xform='s!.*!rio-'${VERSION}'-darwin/rio!'
+
+W=rio-${VERSION}-windows
+mkdir -p $W
+trap "rm -rf $W" EXIT
+
+cp -f bin/rio-windows $W/rio.exe
+zip dist/artifacts/rio-${VERSION}-windows.zip $W/rio.exe

scripts/release

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+exec $(dirname $0)/ci

scripts/symlink-k8s.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ ! -d $1/staging/src/k8s.io ]; then
+    echo Kubernetes source not found at $1
+    exit 1
+fi
+
+cd $(dirname $0)/../vendor/k8s.io
+for i in $1/staging/src/k8s.io/*; do
+    rm -rf $(basename $i)
+    ln -s $i .
+done
+rm -rf kubernetes
+mkdir -p kubernetes
+cd kubernetes
+ln -s $1/{cmd,pkg,plugin,third_party} .

scripts/test

@@ -0,0 +1,50 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/..
+
+echo Running tests
+
+mkdir -p ./build
+
+mkdir -p /var/lib/rancher/rio/agent
+mount -t tmpfs none /var/lib/rancher/rio/agent
+
+./bin/rio server --disable-agent | grep -v level=info > /var/lib/rancher/rio/agent/agent.log 2>&1 &
+
+for i in {1..120}; do
+    if [ ! -e /var/lib/rancher/rio/server/node-token ]; then
+        sleep .5
+        continue
+    fi
+
+    curl -sf http://localhost:7080/healthz >/dev/null && break
+    sleep .5
+done
+curl -sf http://localhost:7080/healthz >/dev/null
+
+rm -rf ./image/root
+unsquashfs -d ./image/root ./image/main.squashfs
+ENTER_ROOT=$(pwd)/image/root ./bin/rio --debug agent -s https://localhost:7443 -t $(</var/lib/rancher/rio/server/node-token) >>/var/lib/rancher/rio/agent/agent.log 2>&1 &
+
+export PATH=$(pwd)/bin:$PATH
+
+rio login -s https://localhost:7443 -t /var/lib/rancher/rio/server/client-token
+
+echo Waiting for istio/istio-gateway
+rio --project=rio-system wait istio/istio-gateway
+rio --project=rio-system ps
+rio --project=rio-system ps -c
+
+chmod +x ./tests/init-nfs.bash
+./tests/init-nfs.bash
+export RUN_NFS_TEST=true
+
+cd ./tests
+tox -- -n $(nproc)
+cd ..
+
+bats -r ./tests || {
+    tail -n 100 /var/lib/rancher/rio/agent/agent.log
+    exit 1
+}

scripts/validate

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+cd $(dirname $0)/..
+
+echo Running validation
+
+if grep -r '^[[:space:]]*\[\[.*]][[:space:]]*$' tests; then
+    echo "Add \"|| false\" to bats tests in any place you use [[ ... ]]"
+    exit 1
+fi
+
+PACKAGES="$(go list ./...)"
+
+echo Running: go vet
+go vet ${PACKAGES}
+echo Running: gometalinter
+for i in ${PACKAGES}; do
+    if [ -n "$(gometalinter $i | \
+        grep -v 'should have comment.*or be unexported' | \
+        grep -v 'cli/cmd.*don.t use underscores in Go name' | \
+        grep -v 'cli/cmd.*should be DNS' | \
+        tee /dev/stderr)" ]; then
+        failed=true
+    fi
+done
+test -z "$failed"
+echo Running: go fmt
+test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)"

scripts/version

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
+    DIRTY="-dirty"
+fi
+
+COMMIT=$(git rev-parse --short HEAD)
+GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)}
+
+if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then
+    VERSION=$GIT_TAG
+else
+    VERSION="${COMMIT}${DIRTY}"
+fi
+
+if [ -z "$ARCH" ]; then
+    ARCH=amd64
+fi

trash.lock

@@ -0,0 +1,246 @@
+package: package=github.com/rancher/rio
+import:
+- package: github.com/coreos/flannel
+  version: 39af3d7e46f2efa156644e247bdcf3b5bc5f1394
+- package: github.com/gorilla/mux
+  version: v1.6.2
+- package: github.com/gorilla/websocket
+  version: v1.2.0
+- package: github.com/mattn/go-sqlite3
+  version: v1.9.0
+- package: github.com/natefinch/lumberjack
+  version: aee4629129445bbdfb69aa565537dcfa16544311
+- package: github.com/rancher/norman
+  version: 5726ebfba191eef161af9638e85bf4fcaa58e008
+  repo: https://github.com/ibuildthecloud/norman.git
+- package: github.com/urfave/cli
+  version: 8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff
+- package: golang.org/x/crypto
+  version: a49355c7e3f8fe157a85be2f77e6e269a0f89602
+- package: golang.org/x/sync
+  version: 450f422ab23cf9881c94e2db30cac0eb1b7cf80c
+- package: gopkg.in/freddierice/go-losetup.v1
+  version: fc9adea44124401d8bfef3a97eaf61b5d44cc2c6
+- package: k8s.io/kubernetes
+  version: a1d7d1b140f43b6503311ffc1dd80017c553bf8e
+  repo: file:///home/darren/src/kuberlite/.git
+  transitive: true
+  staging: true
+- package: github.com/pborman/uuid
+  version: ca53cad383cad2479bbba7f7a1a05797ec1386e4
+- package: github.com/prometheus/client_model
+  version: model-0.0.2-12-gfa8ad6fec33561
+- package: github.com/prometheus/common
+  version: 13ba4ddd0caa9c28ca7b7bffe1dfa9ed8d5ef207
+- package: github.com/golang/protobuf
+  version: v1.1.0
+- package: golang.org/x/sys
+  version: 95c6576299259db960f6c5b9b69ea52422860fce
+- package: github.com/ugorji/go
+  version: bdcc60b419d136a85cdf2e7cbcac34b3f1cd6e57
+- package: github.com/vishvananda/netns
+  version: be1fbeda19366dea804f00efff2dd73a1642fdcc
+- package: gopkg.in/natefinch/lumberjack.v2
+  version: v1.0-16-g20b71e5b60d756
+- package: vbom.ml/util
+  version: db5cfe13f5cc80a4990d98e2e1b0707a4d1a5394
+- package: github.com/exponent-io/jsonpath
+  version: d6023ce2651d8eafb5c75bb0c7167536102ec9f5
+- package: github.com/google/certificate-transparency-go
+  version: v1.0.21
+- package: github.com/coreos/go-semver
+  version: v0.2.0-9-ge214231b295a8e
+- package: github.com/docker/spdystream
+  version: 449fdfce4d962303d702fec724ef0ad181c92528
+- package: github.com/renstrom/dedent
+  version: v1.0.0-3-g020d11c3b9c0c7
+- package: golang.org/x/text
+  version: b19bf474d317b857955b12035d2c5acb57ce8b01
+- package: github.com/container-storage-interface/spec
+  version: v1.0.0
+- package: github.com/evanphx/json-patch
+  version: v4.0.0-3-g36442dbdb58521
+- package: github.com/sigma/go-inotify
+  version: c87b6cf5033d2c6486046f045eeebdc3d910fd38
+- package: github.com/blang/semver
+  version: v3.5.0
+- package: github.com/docker/libnetwork
+  version: v0.8.0-dev.2-1265-ga9cd636e378982
+- package: github.com/fatih/camelcase
+  version: f6a740d52f961c60348ebb109adde9f4635d7540
+- package: github.com/prometheus/client_golang
+  version: v0.8.0-83-ge7e903064f5e9e
+- package: github.com/shurcooL/sanitized_anchor_name
+  version: 10ef21a441db47d8b13ebcc5fd2310f636973c77
+- package: github.com/spf13/cobra
+  version: v0.0.1-34-gc439c4fa093711
+- package: github.com/docker/docker
+  version: docs-v1.12.0-rc4-2016-07-15-9510-ga9fbbdc8dd8794
+- package: golang.org/x/net
+  version: 0ed95abb35c445290478a5348a7b38bb154135fd
+- package: github.com/json-iterator/go
+  version: 1.1.4
+- package: github.com/opencontainers/runc
+  version: v1.0.0-rc5-46-g871ba2e58e2431
+- package: gopkg.in/inf.v0
+  version: v0.9.0
+- package: github.com/containerd/containerd
+  version: v1.0.2
+- package: github.com/ghodss/yaml
+  version: v1.0.0-4-gc7ce16629ff4cd
+- package: github.com/ibuildthecloud/kvsql
+  version: 6bb3d252056655760ed8ca6557d6d5e607b361d2
+- package: google.golang.org/genproto
+  version: 09f6ed296fc66555a25fe4ce95173148778dfa85
+- package: github.com/Microsoft/hcsshim
+  version: v0.6.11
+- package: google.golang.org/grpc
+  version: v1.13.0
+- package: github.com/gregjones/httpcache
+  version: 787624de3eb7bd915c329cba748687a3b22666a6
+- package: github.com/golang/groupcache
+  version: 02826c3e79038b59d737d3b1c0a1d937f71a4433
+- package: github.com/karrick/godirwalk
+  version: v1.7.5
+- package: github.com/mattn/go-shellwords
+  version: v1.0.3-20-gf8471b0a71ded0
+- package: github.com/google/gofuzz
+  version: 44d81051d367757e1c7c6a5a86423ece9afcf63c
+- package: github.com/modern-go/concurrent
+  version: 1.0.3
+- package: github.com/google/cadvisor
+  version: 25bec0e2ace4846e5caaaf69991046c6f44c4bac
+  repo: https://github.com/ibuildthecloud/cadvisor.git
+- package: sigs.k8s.io/yaml
+  version: v1.1.0
+- package: bitbucket.org/ww/goautoneg
+  version: a547fc61f48d567d5b4ec6f8aee5573d8efce11d
+  repo: https://github.com/rancher/goautoneg.git
+- package: github.com/beorn7/perks
+  version: 3ac7bf7a47d159a033b107610db8a1b6575507a4
+- package: github.com/sirupsen/logrus
+  version: v1.0.3-11-g89742aefa4b206
+- package: gopkg.in/square/go-jose.v2
+  version: v2.1.6-4-g89060dee6a84df
+- package: k8s.io/utils
+  version: 66066c83e385e385ccc3c964b44fd7dcd413d0ed
+- package: github.com/daviddengcn/go-colortext
+  version: 511bcaf42ccd42c38aba7427b6673277bf19e2a1
+- package: github.com/prometheus/procfs
+  version: 65c1f6f8f0fc1e2185eb9863a3bc751496404259
+- package: golang.org/x/tools
+  version: 2382e3994d48b1d22acc2c86bcad0a2aff028e32
+- package: github.com/jteeuwen/go-bindata
+  version: v3.0.7-72-ga0ff2567cfb709
+- package: github.com/modern-go/reflect2
+  version: v1.0.1
+- package: golang.org/x/time
+  version: f51c12702a4d776e4c1fa9b0fabab841babae631
+- package: k8s.io/gengo
+  version: 51747d6e00da1fc578d5a333a93bb2abcbce7a95
+- package: github.com/euank/go-kmsg-parser
+  version: v2.0.0
+- package: github.com/chai2010/gettext-go
+  version: c6fed771bfd517099caf0f7a961671fa8ed08723
+- package: github.com/davecgh/go-spew
+  version: v1.1.0-1-g782f4967f2dc45
+- package: github.com/docker/distribution
+  version: v2.6.0-rc.1-209-gedc3ab29cdff86
+- package: github.com/docker/go-units
+  version: v0.3.1-11-g9e638d38cf6977
+- package: github.com/mindprince/gonvml
+  version: fee913ce8fb235edf54739d259ca0ecc226c7b8a
+- package: github.com/MakeNowJust/heredoc
+  version: bb23615498cded5e105af4ce27de75b089cbe851
+- package: github.com/syndtr/gocapability
+  version: e7cb7fa329f456b3855136a2642b197bad7366ba
+- package: k8s.io/klog
+  version: 8139d8cb77af419532b33dfa7dd09fbc5f1d344f
+- package: github.com/emicklei/go-restful
+  version: 2.2.0-4-gff4f55a206334e
+- package: github.com/google/btree
+  version: 7d79101e329e5a3adf994758c578dab82b90c017
+- package: github.com/Azure/go-ansiterm
+  version: d6e3b3328b783f23731bc4d058875b0371ff8109
+- package: github.com/containerd/console
+  version: 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e
+- package: github.com/pkg/errors
+  version: v0.8.0
+- package: github.com/googleapis/gnostic
+  version: 0c5108395e2debce0d731cf0287ddf7242066aba
+- package: github.com/imdario/mergo
+  version: v0.3.5
+- package: github.com/robfig/cron
+  version: v1-53-gdf38d32658d878
+- package: github.com/JeffAshton/win_pdh
+  version: 76bb4ee9f0ab50f77826f2a2ee7fb9d3880d6ec2
+- package: github.com/armon/circbuf
+  version: bbbad097214e2918d8543d5201d12bfd7bca254d
+- package: k8s.io/heapster
+  version: v1.2.0-beta.1
+- package: github.com/matttproud/golang_protobuf_extensions
+  version: v1.0.1
+- package: github.com/mrunalp/fileutils
+  version: 4ee1cc9a80582a0c75febdd5cfa779ee4361cbca
+- package: github.com/opencontainers/runtime-spec
+  version: v1.0.0
+- package: github.com/fsnotify/fsnotify
+  version: v1.3.1-1-gf12c6236fe7b5c
+- package: github.com/mistifyio/go-zfs
+  version: v2.1.1-5-g1b4ae6fb4e77b0
+- package: github.com/russross/blackfriday
+  version: v1.4-2-g300106c228d52c
+- package: github.com/inconshreveable/mousetrap
+  version: v1.0
+- package: github.com/vishvananda/netlink
+  version: b2de5d10e38ecce8607e6b438b6d174f389a004e
+- package: github.com/mitchellh/go-wordwrap
+  version: ad45545899c7b13c020ea92b2072220eefad42b8
+- package: github.com/peterbourgon/diskv
+  version: v2.0.1
+- package: github.com/Microsoft/go-winio
+  version: v0.4.5
+- package: github.com/cyphar/filepath-securejoin
+  version: v0.2.1-1-gae69057f2299fb
+- package: github.com/Nvveen/Gotty
+  version: cd527374f1e5bff4938207604a14f2e38a9cf512
+- package: github.com/containernetworking/cni
+  version: v0.6.0
+- package: github.com/hashicorp/golang-lru
+  version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
+- package: github.com/jonboulle/clockwork
+  version: 72f9bd7c4e0c2a40055ab3d0f09654f730cce982
+- package: github.com/seccomp/libseccomp-golang
+  version: 1b506fc7c24eec5a3693cdcbed40d9c226cfc6a1
+- package: github.com/coreos/pkg
+  version: v4
+- package: github.com/opencontainers/image-spec
+  version: v1.0.0-rc6-12-g372ad780f63454
+- package: github.com/opencontainers/selinux
+  version: v1.0.0-rc1-5-g4a2974bf1ee960
+- package: github.com/cloudflare/cfssl
+  version: 1.3.2-21-g56268a613adfed
+- package: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: v1.1-4-g2500245aa6110c
+- package: github.com/miekg/dns
+  version: 5d001d020961ae1c184f9f8152fdc73810481677
+- package: github.com/godbus/dbus
+  version: v3
+- package: github.com/spf13/pflag
+  version: v1.0.1
+- package: github.com/docker/go-connections
+  version: v0.3.0
+- package: golang.org/x/oauth2
+  version: a6bd8cefa1811bd24b86f8902872e4e8225f74c4
+- package: github.com/gogo/protobuf
+  version: v0.5
+- package: github.com/mxk/go-flowrate
+  version: cca7078d478f8520f85629ad7c68962d31ed7682
+- package: github.com/coreos/go-systemd
+  version: v17
+- package: github.com/opencontainers/go-digest
+  version: a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb
+- package: gopkg.in/yaml.v2
+  version: v2.2.1
+- package: github.com/coreos/etcd
+  version: v3.3.10

types/apis/k3s.cattle.io/v1/schema.go

@@ -0,0 +1,17 @@
+package v1
+
+import (
+	"github.com/rancher/norman/types"
+	"github.com/rancher/norman/types/factory"
+)
+
+var (
+	APIVersion = types.APIVersion{
+		Version: "v1",
+		Group:   "k3s.cattle.io",
+		Path:    "/v1-k3s",
+	}
+
+	Schemas = factory.Schemas(&APIVersion).
+		MustImport(&APIVersion, ListenerConfig{})
+)

types/apis/k3s.cattle.io/v1/types.go

@@ -0,0 +1,16 @@
+package v1
+
+import (
+	"github.com/rancher/norman/pkg/dynamiclistener"
+	"github.com/rancher/norman/types"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type ListenerConfig struct {
+	types.Namespaced
+
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Status dynamiclistener.ListenerStatus `json:"status,omitempty"`
+}

types/apis/k3s.cattle.io/v1/zz_generated_deepcopy.go

@@ -0,0 +1,66 @@
+package v1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ListenerConfig) DeepCopyInto(out *ListenerConfig) {
+	*out = *in
+	out.Namespaced = in.Namespaced
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Status.DeepCopyInto(&out.Status)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListenerConfig.
+func (in *ListenerConfig) DeepCopy() *ListenerConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(ListenerConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ListenerConfig) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ListenerConfigList) DeepCopyInto(out *ListenerConfigList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	out.ListMeta = in.ListMeta
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ListenerConfig, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListenerConfigList.
+func (in *ListenerConfigList) DeepCopy() *ListenerConfigList {
+	if in == nil {
+		return nil
+	}
+	out := new(ListenerConfigList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ListenerConfigList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}

types/apis/k3s.cattle.io/v1/zz_generated_k8s_client.go

@@ -0,0 +1,119 @@
+package v1
+
+import (
+	"context"
+	"sync"
+
+	"github.com/rancher/norman/controller"
+	"github.com/rancher/norman/objectclient"
+	"github.com/rancher/norman/objectclient/dynamic"
+	"github.com/rancher/norman/restwatch"
+	"k8s.io/client-go/rest"
+)
+
+type (
+	contextKeyType        struct{}
+	contextClientsKeyType struct{}
+)
+
+type Interface interface {
+	RESTClient() rest.Interface
+	controller.Starter
+
+	ListenerConfigsGetter
+}
+
+type Clients struct {
+	Interface Interface
+
+	ListenerConfig ListenerConfigClient
+}
+
+type Client struct {
+	sync.Mutex
+	restClient rest.Interface
+	starters   []controller.Starter
+
+	listenerConfigControllers map[string]ListenerConfigController
+}
+
+func Factory(ctx context.Context, config rest.Config) (context.Context, controller.Starter, error) {
+	c, err := NewForConfig(config)
+	if err != nil {
+		return ctx, nil, err
+	}
+
+	cs := NewClientsFromInterface(c)
+
+	ctx = context.WithValue(ctx, contextKeyType{}, c)
+	ctx = context.WithValue(ctx, contextClientsKeyType{}, cs)
+	return ctx, c, nil
+}
+
+func ClientsFrom(ctx context.Context) *Clients {
+	return ctx.Value(contextClientsKeyType{}).(*Clients)
+}
+
+func From(ctx context.Context) Interface {
+	return ctx.Value(contextKeyType{}).(Interface)
+}
+
+func NewClients(config rest.Config) (*Clients, error) {
+	iface, err := NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	return NewClientsFromInterface(iface), nil
+}
+
+func NewClientsFromInterface(iface Interface) *Clients {
+	return &Clients{
+		Interface: iface,
+
+		ListenerConfig: &listenerConfigClient2{
+			iface: iface.ListenerConfigs(""),
+		},
+	}
+}
+
+func NewForConfig(config rest.Config) (Interface, error) {
+	if config.NegotiatedSerializer == nil {
+		config.NegotiatedSerializer = dynamic.NegotiatedSerializer
+	}
+
+	restClient, err := restwatch.UnversionedRESTClientFor(&config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Client{
+		restClient: restClient,
+
+		listenerConfigControllers: map[string]ListenerConfigController{},
+	}, nil
+}
+
+func (c *Client) RESTClient() rest.Interface {
+	return c.restClient
+}
+
+func (c *Client) Sync(ctx context.Context) error {
+	return controller.Sync(ctx, c.starters...)
+}
+
+func (c *Client) Start(ctx context.Context, threadiness int) error {
+	return controller.Start(ctx, threadiness, c.starters...)
+}
+
+type ListenerConfigsGetter interface {
+	ListenerConfigs(namespace string) ListenerConfigInterface
+}
+
+func (c *Client) ListenerConfigs(namespace string) ListenerConfigInterface {
+	objectClient := objectclient.NewObjectClient(namespace, c.restClient, &ListenerConfigResource, ListenerConfigGroupVersionKind, listenerConfigFactory{})
+	return &listenerConfigClient{
+		ns:           namespace,
+		client:       c,
+		objectClient: objectClient,
+	}
+}

types/apis/k3s.cattle.io/v1/zz_generated_listener_config_controller.go

@@ -0,0 +1,440 @@
+package v1
+
+import (
+	"context"
+
+	"github.com/rancher/norman/controller"
+	"github.com/rancher/norman/objectclient"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/tools/cache"
+)
+
+var (
+	ListenerConfigGroupVersionKind = schema.GroupVersionKind{
+		Version: Version,
+		Group:   GroupName,
+		Kind:    "ListenerConfig",
+	}
+	ListenerConfigResource = metav1.APIResource{
+		Name:         "listenerconfigs",
+		SingularName: "listenerconfig",
+		Namespaced:   true,
+
+		Kind: ListenerConfigGroupVersionKind.Kind,
+	}
+)
+
+func NewListenerConfig(namespace, name string, obj ListenerConfig) *ListenerConfig {
+	obj.APIVersion, obj.Kind = ListenerConfigGroupVersionKind.ToAPIVersionAndKind()
+	obj.Name = name
+	obj.Namespace = namespace
+	return &obj
+}
+
+type ListenerConfigList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ListenerConfig
+}
+
+type ListenerConfigHandlerFunc func(key string, obj *ListenerConfig) (runtime.Object, error)
+
+type ListenerConfigChangeHandlerFunc func(obj *ListenerConfig) (runtime.Object, error)
+
+type ListenerConfigLister interface {
+	List(namespace string, selector labels.Selector) (ret []*ListenerConfig, err error)
+	Get(namespace, name string) (*ListenerConfig, error)
+}
+
+type ListenerConfigController interface {
+	Generic() controller.GenericController
+	Informer() cache.SharedIndexInformer
+	Lister() ListenerConfigLister
+	AddHandler(ctx context.Context, name string, handler ListenerConfigHandlerFunc)
+	AddClusterScopedHandler(ctx context.Context, name, clusterName string, handler ListenerConfigHandlerFunc)
+	Enqueue(namespace, name string)
+	Sync(ctx context.Context) error
+	Start(ctx context.Context, threadiness int) error
+}
+
+type ListenerConfigInterface interface {
+	ObjectClient() *objectclient.ObjectClient
+	Create(*ListenerConfig) (*ListenerConfig, error)
+	GetNamespaced(namespace, name string, opts metav1.GetOptions) (*ListenerConfig, error)
+	Get(name string, opts metav1.GetOptions) (*ListenerConfig, error)
+	Update(*ListenerConfig) (*ListenerConfig, error)
+	Delete(name string, options *metav1.DeleteOptions) error
+	DeleteNamespaced(namespace, name string, options *metav1.DeleteOptions) error
+	List(opts metav1.ListOptions) (*ListenerConfigList, error)
+	Watch(opts metav1.ListOptions) (watch.Interface, error)
+	DeleteCollection(deleteOpts *metav1.DeleteOptions, listOpts metav1.ListOptions) error
+	Controller() ListenerConfigController
+	AddHandler(ctx context.Context, name string, sync ListenerConfigHandlerFunc)
+	AddLifecycle(ctx context.Context, name string, lifecycle ListenerConfigLifecycle)
+	AddClusterScopedHandler(ctx context.Context, name, clusterName string, sync ListenerConfigHandlerFunc)
+	AddClusterScopedLifecycle(ctx context.Context, name, clusterName string, lifecycle ListenerConfigLifecycle)
+}
+
+type listenerConfigLister struct {
+	controller *listenerConfigController
+}
+
+func (l *listenerConfigLister) List(namespace string, selector labels.Selector) (ret []*ListenerConfig, err error) {
+	err = cache.ListAllByNamespace(l.controller.Informer().GetIndexer(), namespace, selector, func(obj interface{}) {
+		ret = append(ret, obj.(*ListenerConfig))
+	})
+	return
+}
+
+func (l *listenerConfigLister) Get(namespace, name string) (*ListenerConfig, error) {
+	var key string
+	if namespace != "" {
+		key = namespace + "/" + name
+	} else {
+		key = name
+	}
+	obj, exists, err := l.controller.Informer().GetIndexer().GetByKey(key)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(schema.GroupResource{
+			Group:    ListenerConfigGroupVersionKind.Group,
+			Resource: "listenerConfig",
+		}, key)
+	}
+	return obj.(*ListenerConfig), nil
+}
+
+type listenerConfigController struct {
+	controller.GenericController
+}
+
+func (c *listenerConfigController) Generic() controller.GenericController {
+	return c.GenericController
+}
+
+func (c *listenerConfigController) Lister() ListenerConfigLister {
+	return &listenerConfigLister{
+		controller: c,
+	}
+}
+
+func (c *listenerConfigController) AddHandler(ctx context.Context, name string, handler ListenerConfigHandlerFunc) {
+	c.GenericController.AddHandler(ctx, name, func(key string, obj interface{}) (interface{}, error) {
+		if obj == nil {
+			return handler(key, nil)
+		} else if v, ok := obj.(*ListenerConfig); ok {
+			return handler(key, v)
+		} else {
+			return nil, nil
+		}
+	})
+}
+
+func (c *listenerConfigController) AddClusterScopedHandler(ctx context.Context, name, cluster string, handler ListenerConfigHandlerFunc) {
+	c.GenericController.AddHandler(ctx, name, func(key string, obj interface{}) (interface{}, error) {
+		if obj == nil {
+			return handler(key, nil)
+		} else if v, ok := obj.(*ListenerConfig); ok && controller.ObjectInCluster(cluster, obj) {
+			return handler(key, v)
+		} else {
+			return nil, nil
+		}
+	})
+}
+
+type listenerConfigFactory struct {
+}
+
+func (c listenerConfigFactory) Object() runtime.Object {
+	return &ListenerConfig{}
+}
+
+func (c listenerConfigFactory) List() runtime.Object {
+	return &ListenerConfigList{}
+}
+
+func (s *listenerConfigClient) Controller() ListenerConfigController {
+	s.client.Lock()
+	defer s.client.Unlock()
+
+	c, ok := s.client.listenerConfigControllers[s.ns]
+	if ok {
+		return c
+	}
+
+	genericController := controller.NewGenericController(ListenerConfigGroupVersionKind.Kind+"Controller",
+		s.objectClient)
+
+	c = &listenerConfigController{
+		GenericController: genericController,
+	}
+
+	s.client.listenerConfigControllers[s.ns] = c
+	s.client.starters = append(s.client.starters, c)
+
+	return c
+}
+
+type listenerConfigClient struct {
+	client       *Client
+	ns           string
+	objectClient *objectclient.ObjectClient
+	controller   ListenerConfigController
+}
+
+func (s *listenerConfigClient) ObjectClient() *objectclient.ObjectClient {
+	return s.objectClient
+}
+
+func (s *listenerConfigClient) Create(o *ListenerConfig) (*ListenerConfig, error) {
+	obj, err := s.objectClient.Create(o)
+	return obj.(*ListenerConfig), err
+}
+
+func (s *listenerConfigClient) Get(name string, opts metav1.GetOptions) (*ListenerConfig, error) {
+	obj, err := s.objectClient.Get(name, opts)
+	return obj.(*ListenerConfig), err
+}
+
+func (s *listenerConfigClient) GetNamespaced(namespace, name string, opts metav1.GetOptions) (*ListenerConfig, error) {
+	obj, err := s.objectClient.GetNamespaced(namespace, name, opts)
+	return obj.(*ListenerConfig), err
+}
+
+func (s *listenerConfigClient) Update(o *ListenerConfig) (*ListenerConfig, error) {
+	obj, err := s.objectClient.Update(o.Name, o)
+	return obj.(*ListenerConfig), err
+}
+
+func (s *listenerConfigClient) Delete(name string, options *metav1.DeleteOptions) error {
+	return s.objectClient.Delete(name, options)
+}
+
+func (s *listenerConfigClient) DeleteNamespaced(namespace, name string, options *metav1.DeleteOptions) error {
+	return s.objectClient.DeleteNamespaced(namespace, name, options)
+}
+
+func (s *listenerConfigClient) List(opts metav1.ListOptions) (*ListenerConfigList, error) {
+	obj, err := s.objectClient.List(opts)
+	return obj.(*ListenerConfigList), err
+}
+
+func (s *listenerConfigClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return s.objectClient.Watch(opts)
+}
+
+// Patch applies the patch and returns the patched deployment.
+func (s *listenerConfigClient) Patch(o *ListenerConfig, patchType types.PatchType, data []byte, subresources ...string) (*ListenerConfig, error) {
+	obj, err := s.objectClient.Patch(o.Name, o, patchType, data, subresources...)
+	return obj.(*ListenerConfig), err
+}
+
+func (s *listenerConfigClient) DeleteCollection(deleteOpts *metav1.DeleteOptions, listOpts metav1.ListOptions) error {
+	return s.objectClient.DeleteCollection(deleteOpts, listOpts)
+}
+
+func (s *listenerConfigClient) AddHandler(ctx context.Context, name string, sync ListenerConfigHandlerFunc) {
+	s.Controller().AddHandler(ctx, name, sync)
+}
+
+func (s *listenerConfigClient) AddLifecycle(ctx context.Context, name string, lifecycle ListenerConfigLifecycle) {
+	sync := NewListenerConfigLifecycleAdapter(name, false, s, lifecycle)
+	s.Controller().AddHandler(ctx, name, sync)
+}
+
+func (s *listenerConfigClient) AddClusterScopedHandler(ctx context.Context, name, clusterName string, sync ListenerConfigHandlerFunc) {
+	s.Controller().AddClusterScopedHandler(ctx, name, clusterName, sync)
+}
+
+func (s *listenerConfigClient) AddClusterScopedLifecycle(ctx context.Context, name, clusterName string, lifecycle ListenerConfigLifecycle) {
+	sync := NewListenerConfigLifecycleAdapter(name+"_"+clusterName, true, s, lifecycle)
+	s.Controller().AddClusterScopedHandler(ctx, name, clusterName, sync)
+}
+
+type ListenerConfigIndexer func(obj *ListenerConfig) ([]string, error)
+
+type ListenerConfigClientCache interface {
+	Get(namespace, name string) (*ListenerConfig, error)
+	List(namespace string, selector labels.Selector) ([]*ListenerConfig, error)
+
+	Index(name string, indexer ListenerConfigIndexer)
+	GetIndexed(name, key string) ([]*ListenerConfig, error)
+}
+
+type ListenerConfigClient interface {
+	Create(*ListenerConfig) (*ListenerConfig, error)
+	Get(namespace, name string, opts metav1.GetOptions) (*ListenerConfig, error)
+	Update(*ListenerConfig) (*ListenerConfig, error)
+	Delete(namespace, name string, options *metav1.DeleteOptions) error
+	List(namespace string, opts metav1.ListOptions) (*ListenerConfigList, error)
+	Watch(opts metav1.ListOptions) (watch.Interface, error)
+
+	Cache() ListenerConfigClientCache
+
+	OnCreate(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc)
+	OnChange(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc)
+	OnRemove(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc)
+	Enqueue(namespace, name string)
+
+	Generic() controller.GenericController
+	ObjectClient() *objectclient.ObjectClient
+	Interface() ListenerConfigInterface
+}
+
+type listenerConfigClientCache struct {
+	client *listenerConfigClient2
+}
+
+type listenerConfigClient2 struct {
+	iface      ListenerConfigInterface
+	controller ListenerConfigController
+}
+
+func (n *listenerConfigClient2) Interface() ListenerConfigInterface {
+	return n.iface
+}
+
+func (n *listenerConfigClient2) Generic() controller.GenericController {
+	return n.iface.Controller().Generic()
+}
+
+func (n *listenerConfigClient2) ObjectClient() *objectclient.ObjectClient {
+	return n.Interface().ObjectClient()
+}
+
+func (n *listenerConfigClient2) Enqueue(namespace, name string) {
+	n.iface.Controller().Enqueue(namespace, name)
+}
+
+func (n *listenerConfigClient2) Create(obj *ListenerConfig) (*ListenerConfig, error) {
+	return n.iface.Create(obj)
+}
+
+func (n *listenerConfigClient2) Get(namespace, name string, opts metav1.GetOptions) (*ListenerConfig, error) {
+	return n.iface.GetNamespaced(namespace, name, opts)
+}
+
+func (n *listenerConfigClient2) Update(obj *ListenerConfig) (*ListenerConfig, error) {
+	return n.iface.Update(obj)
+}
+
+func (n *listenerConfigClient2) Delete(namespace, name string, options *metav1.DeleteOptions) error {
+	return n.iface.DeleteNamespaced(namespace, name, options)
+}
+
+func (n *listenerConfigClient2) List(namespace string, opts metav1.ListOptions) (*ListenerConfigList, error) {
+	return n.iface.List(opts)
+}
+
+func (n *listenerConfigClient2) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return n.iface.Watch(opts)
+}
+
+func (n *listenerConfigClientCache) Get(namespace, name string) (*ListenerConfig, error) {
+	return n.client.controller.Lister().Get(namespace, name)
+}
+
+func (n *listenerConfigClientCache) List(namespace string, selector labels.Selector) ([]*ListenerConfig, error) {
+	return n.client.controller.Lister().List(namespace, selector)
+}
+
+func (n *listenerConfigClient2) Cache() ListenerConfigClientCache {
+	n.loadController()
+	return &listenerConfigClientCache{
+		client: n,
+	}
+}
+
+func (n *listenerConfigClient2) OnCreate(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc) {
+	n.loadController()
+	n.iface.AddLifecycle(ctx, name+"-create", &listenerConfigLifecycleDelegate{create: sync})
+}
+
+func (n *listenerConfigClient2) OnChange(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc) {
+	n.loadController()
+	n.iface.AddLifecycle(ctx, name+"-change", &listenerConfigLifecycleDelegate{update: sync})
+}
+
+func (n *listenerConfigClient2) OnRemove(ctx context.Context, name string, sync ListenerConfigChangeHandlerFunc) {
+	n.loadController()
+	n.iface.AddLifecycle(ctx, name, &listenerConfigLifecycleDelegate{remove: sync})
+}
+
+func (n *listenerConfigClientCache) Index(name string, indexer ListenerConfigIndexer) {
+	err := n.client.controller.Informer().GetIndexer().AddIndexers(map[string]cache.IndexFunc{
+		name: func(obj interface{}) ([]string, error) {
+			if v, ok := obj.(*ListenerConfig); ok {
+				return indexer(v)
+			}
+			return nil, nil
+		},
+	})
+
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (n *listenerConfigClientCache) GetIndexed(name, key string) ([]*ListenerConfig, error) {
+	var result []*ListenerConfig
+	objs, err := n.client.controller.Informer().GetIndexer().ByIndex(name, key)
+	if err != nil {
+		return nil, err
+	}
+	for _, obj := range objs {
+		if v, ok := obj.(*ListenerConfig); ok {
+			result = append(result, v)
+		}
+	}
+
+	return result, nil
+}
+
+func (n *listenerConfigClient2) loadController() {
+	if n.controller == nil {
+		n.controller = n.iface.Controller()
+	}
+}
+
+type listenerConfigLifecycleDelegate struct {
+	create ListenerConfigChangeHandlerFunc
+	update ListenerConfigChangeHandlerFunc
+	remove ListenerConfigChangeHandlerFunc
+}
+
+func (n *listenerConfigLifecycleDelegate) HasCreate() bool {
+	return n.create != nil
+}
+
+func (n *listenerConfigLifecycleDelegate) Create(obj *ListenerConfig) (runtime.Object, error) {
+	if n.create == nil {
+		return obj, nil
+	}
+	return n.create(obj)
+}
+
+func (n *listenerConfigLifecycleDelegate) HasFinalize() bool {
+	return n.remove != nil
+}
+
+func (n *listenerConfigLifecycleDelegate) Remove(obj *ListenerConfig) (runtime.Object, error) {
+	if n.remove == nil {
+		return obj, nil
+	}
+	return n.remove(obj)
+}
+
+func (n *listenerConfigLifecycleDelegate) Updated(obj *ListenerConfig) (runtime.Object, error) {
+	if n.update == nil {
+		return obj, nil
+	}
+	return n.update(obj)
+}

types/apis/k3s.cattle.io/v1/zz_generated_listener_config_lifecycle_adapter.go

@@ -0,0 +1,62 @@
+package v1
+
+import (
+	"github.com/rancher/norman/lifecycle"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type ListenerConfigLifecycle interface {
+	Create(obj *ListenerConfig) (runtime.Object, error)
+	Remove(obj *ListenerConfig) (runtime.Object, error)
+	Updated(obj *ListenerConfig) (runtime.Object, error)
+}
+
+type listenerConfigLifecycleAdapter struct {
+	lifecycle ListenerConfigLifecycle
+}
+
+func (w *listenerConfigLifecycleAdapter) HasCreate() bool {
+	o, ok := w.lifecycle.(lifecycle.ObjectLifecycleCondition)
+	return !ok || o.HasCreate()
+}
+
+func (w *listenerConfigLifecycleAdapter) HasFinalize() bool {
+	o, ok := w.lifecycle.(lifecycle.ObjectLifecycleCondition)
+	return !ok || o.HasFinalize()
+}
+
+func (w *listenerConfigLifecycleAdapter) Create(obj runtime.Object) (runtime.Object, error) {
+	o, err := w.lifecycle.Create(obj.(*ListenerConfig))
+	if o == nil {
+		return nil, err
+	}
+	return o, err
+}
+
+func (w *listenerConfigLifecycleAdapter) Finalize(obj runtime.Object) (runtime.Object, error) {
+	o, err := w.lifecycle.Remove(obj.(*ListenerConfig))
+	if o == nil {
+		return nil, err
+	}
+	return o, err
+}
+
+func (w *listenerConfigLifecycleAdapter) Updated(obj runtime.Object) (runtime.Object, error) {
+	o, err := w.lifecycle.Updated(obj.(*ListenerConfig))
+	if o == nil {
+		return nil, err
+	}
+	return o, err
+}
+
+func NewListenerConfigLifecycleAdapter(name string, clusterScoped bool, client ListenerConfigInterface, l ListenerConfigLifecycle) ListenerConfigHandlerFunc {
+	adapter := &listenerConfigLifecycleAdapter{lifecycle: l}
+	syncFn := lifecycle.NewObjectLifecycleAdapter(name, clusterScoped, adapter, client.ObjectClient())
+	return func(key string, obj *ListenerConfig) (runtime.Object, error) {
+		newObj, err := syncFn(key, obj)
+		if o, ok := newObj.(runtime.Object); ok {
+			return o, err
+		}
+		return nil, err
+	}
+}

types/apis/k3s.cattle.io/v1/zz_generated_scheme.go

@@ -0,0 +1,40 @@
+package v1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+const (
+	GroupName = "k3s.cattle.io"
+	Version   = "v1"
+)
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: Version}
+
+// Kind takes an unqualified kind and returns a Group qualified GroupKind
+func Kind(kind string) schema.GroupKind {
+	return SchemeGroupVersion.WithKind(kind).GroupKind()
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = SchemeBuilder.AddToScheme
+)
+
+// Adds the list of known types to api.Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	// TODO this gets cleaned up when the types are fixed
+	scheme.AddKnownTypes(SchemeGroupVersion,
+
+		&ListenerConfig{},
+		&ListenerConfigList{},
+	)
+	return nil
+}

types/codegen/cleanup/main.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"github.com/rancher/norman/generator/cleanup"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	if err := cleanup.Cleanup("./types"); err != nil {
+		logrus.Fatal(err)
+	}
+}

types/codegen/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"github.com/rancher/norman/generator"
+	v1 "github.com/rancher/rio/types/apis/k3s.cattle.io/v1"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	basePackage = "github.com/rancher/rio/types"
+)
+
+func main() {
+	if err := generator.DefaultGenerate(v1.Schemas, basePackage, false, nil); err != nil {
+		logrus.Fatal(err)
+	}
+}

vendor.conf

@@ -0,0 +1,17 @@
+package=github.com/rancher/rio
+package=github.com/jteeuwen/go-bindata
+package=github.com/jteeuwen/go-bindata/go-bindata
+
+k8s.io/kubernetes                   a1d7d1b140f43b6503311ffc1dd80017c553bf8e  file:///home/darren/src/kuberlite/.git            transitive=true,staging=true
+
+github.com/rancher/norman           5726ebfba191eef161af9638e85bf4fcaa58e008  https://github.com/ibuildthecloud/norman.git
+github.com/coreos/flannel           39af3d7e46f2efa156644e247bdcf3b5bc5f1394
+github.com/natefinch/lumberjack     aee4629129445bbdfb69aa565537dcfa16544311
+github.com/gorilla/mux              v1.6.2
+github.com/gorilla/websocket        v1.2.0
+github.com/mattn/go-sqlite3         v1.9.0
+github.com/patrickmn/go-cache       v2.1.0
+golang.org/x/crypto                 a49355c7e3f8fe157a85be2f77e6e269a0f89602
+gopkg.in/freddierice/go-losetup.v1  fc9adea44124401d8bfef3a97eaf61b5d44cc2c6
+github.com/urfave/cli               8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff
+golang.org/x/sync                   450f422ab23cf9881c94e2db30cac0eb1b7cf80c

version/version.go

@@ -0,0 +1,6 @@
+package version
+
+var (
+	Version   = "dev"
+	GitCommit = "HEAD"
+)