From 2c9444399b427ffb706818f5bf3892a8880673bf Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 29 May 2019 11:53:51 -0700 Subject: [PATCH 1/8] Refactor certs --- manifests/rolebindings.yaml | 12 ++ pkg/agent/config/config.go | 141 ++++++++----- pkg/agent/flannel/setup.go | 4 +- pkg/agent/proxy/proxy.go | 27 +-- pkg/agent/tunnel/tunnel.go | 2 +- pkg/cli/cmds/server.go | 17 +- pkg/cli/server/server.go | 26 ++- pkg/daemons/agent/agent.go | 14 +- pkg/daemons/config/types.go | 99 ++++++---- pkg/daemons/control/server.go | 358 +++++++++++++++++++++------------- pkg/server/router.go | 211 ++++++++++++-------- pkg/server/server.go | 14 +- 12 files changed, 594 insertions(+), 331 deletions(-) create mode 100644 manifests/rolebindings.yaml diff --git a/manifests/rolebindings.yaml b/manifests/rolebindings.yaml new file mode 100644 index 0000000000..d048eee998 --- /dev/null +++ b/manifests/rolebindings.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-apiserver-kubelet-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kubelet-api-admin +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: kube-apiserver diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index a32bc5cf69..1559785b90 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -23,6 +23,7 @@ import ( "github.com/rancher/k3s/pkg/cli/cmds" "github.com/rancher/k3s/pkg/clientaccess" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/net" @@ -82,6 +83,10 @@ func getNodeNamedCrt(nodeName, nodePasswordFile string) HTTPRequester { } defer resp.Body.Close() + if resp.StatusCode == http.StatusForbidden { + return nil, fmt.Errorf("Node password rejected, contents of '%s' may not match server passwd entry", nodePasswordFile) + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("%s: %s", u, resp.Status) } @@ -101,45 +106,55 @@ func ensureNodePassword(nodePasswordFile string) (string, error) { return "", err } nodePassword := hex.EncodeToString(password) - return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword), 0600) + return nodePassword, ioutil.WriteFile(nodePasswordFile, []byte(nodePassword+"\n"), 0600) } -func getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { - nodeCert, err := Request("/v1-k3s/node.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) +func getServingCert(nodeName, servingCertFile, servingKeyFile, nodePasswordFile string, info *clientaccess.Info) (*tls.Certificate, error) { + servingCert, err := Request("/v1-k3s/serving-kubelet.crt", info, getNodeNamedCrt(nodeName, nodePasswordFile)) if err != nil { return nil, err } - if err := ioutil.WriteFile(nodeCertFile, nodeCert, 0600); err != nil { + if err := ioutil.WriteFile(servingCertFile, servingCert, 0600); err != nil { return nil, errors.Wrapf(err, "failed to write node cert") } - nodeKey, err := clientaccess.Get("/v1-k3s/node.key", info) + servingKey, err := clientaccess.Get("/v1-k3s/serving-kubelet.key", info) if err != nil { return nil, err } - if err := ioutil.WriteFile(nodeKeyFile, nodeKey, 0600); err != nil { + if err := ioutil.WriteFile(servingKeyFile, servingKey, 0600); err != nil { return nil, errors.Wrapf(err, "failed to write node key") } - cert, err := tls.X509KeyPair(nodeCert, nodeKey) + cert, err := tls.X509KeyPair(servingCert, servingKey) if err != nil { return nil, err } return &cert, nil } -func writeNodeCA(dataDir string, nodeCert *tls.Certificate) (string, error) { - clientCABytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Bytes: nodeCert.Certificate[1], - }) - - clientCA := filepath.Join(dataDir, "client-ca.pem") - if err := ioutil.WriteFile(clientCA, clientCABytes, 0600); err != nil { - return "", errors.Wrapf(err, "failed to write client CA") +func getHostFile(filename string, info *clientaccess.Info) error { + basename := filepath.Base(filename) + fileBytes, err := clientaccess.Get("/v1-k3s/"+basename, info) + if err != nil { + return err } + if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", filename) + } + return nil +} - return clientCA, nil +func getNodeNamedHostFile(filename, nodeName, nodePasswordFile string, info *clientaccess.Info) error { + basename := filepath.Base(filename) + fileBytes, err := Request("/v1-k3s/"+basename, info, getNodeNamedCrt(nodeName, nodePasswordFile)) + if err != nil { + return err + } + if err := ioutil.WriteFile(filename, fileBytes, 0600); err != nil { + return errors.Wrapf(err, "failed to write cert %s", filename) + } + return nil } func getHostnameAndIP(info cmds.Agent) (string, string, error) { @@ -169,17 +184,17 @@ func getHostnameAndIP(info cmds.Agent) (string, string, error) { } func localAddress(controlConfig *config.Control) string { - return fmt.Sprintf("127.0.0.1:%d", controlConfig.AdvertisePort) + return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort) } -func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, nodeCert *tls.Certificate) (string, error) { +func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) { os.MkdirAll(envInfo.DataDir, 0700) kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml") info.URL = "https://" + localAddress(controlConfig) info.CACerts = pem.EncodeToMemory(&pem.Block{ Type: cert.CertificateBlockType, - Bytes: nodeCert.Certificate[1], + Bytes: tlsCert.Certificate[1], }) return kubeConfigPath, info.WriteKubeConfig(kubeConfigPath) @@ -253,25 +268,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - nodeCertFile := filepath.Join(envInfo.DataDir, "token-node.crt") - nodeKeyFile := filepath.Join(envInfo.DataDir, "token-node.key") - nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") - - nodeCert, err := getNodeCert(nodeName, nodeCertFile, nodeKeyFile, nodePasswordFile, info) - if err != nil { - return nil, err - } - - clientCA, err := writeNodeCA(envInfo.DataDir, nodeCert) - if err != nil { - return nil, err - } - - kubeConfig, err := writeKubeConfig(envInfo, *info, controlConfig, nodeCert) - if err != nil { - return nil, err - } - hostLocal, err := exec.LookPath("host-local") if err != nil { return nil, errors.Wrapf(err, "failed to find host-local") @@ -285,6 +281,61 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { } } + proxyURL := "https://" + localAddress(controlConfig) + + clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt") + if err := getHostFile(clientCAFile, info); err != nil { + return nil, err + } + + serverCAFile := filepath.Join(envInfo.DataDir, "server-ca.crt") + if err := getHostFile(serverCAFile, info); err != nil { + return nil, err + } + + servingKubeletCert := filepath.Join(envInfo.DataDir, "serving-kubelet.crt") + servingKubeletKey := filepath.Join(envInfo.DataDir, "serving-kubelet.key") + nodePasswordFile := filepath.Join(envInfo.DataDir, "node-password.txt") + servingCert, err := getServingCert(nodeName, servingKubeletCert, servingKubeletKey, nodePasswordFile, info) + if err != nil { + return nil, err + } + + kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert) + if err != nil { + return nil, err + } + + clientKubeletCert := filepath.Join(envInfo.DataDir, "client-kubelet.crt") + if err := getNodeNamedHostFile(clientKubeletCert, nodeName, nodePasswordFile, info); err != nil { + return nil, err + } + + clientKubeletKey := filepath.Join(envInfo.DataDir, "client-kubelet.key") + if err := getHostFile(clientKubeletKey, info); err != nil { + return nil, err + } + + kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig") + if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { + return nil, err + } + + clientKubeProxyCert := filepath.Join(envInfo.DataDir, "client-kube-proxy.crt") + if err := getHostFile(clientKubeProxyCert, info); err != nil { + return nil, err + } + + clientKubeProxyKey := filepath.Join(envInfo.DataDir, "client-kube-proxy.key") + if err := getHostFile(clientKubeProxyKey, info); err != nil { + return nil, err + } + + kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig") + if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { + return nil, err + } + nodeConfig := &config.Node{ Docker: envInfo.Docker, NoFlannel: envInfo.NoFlannel, @@ -295,14 +346,16 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Images = filepath.Join(envInfo.DataDir, "images") nodeConfig.AgentConfig.NodeIP = nodeIP nodeConfig.AgentConfig.NodeName = nodeName - nodeConfig.AgentConfig.NodeCertFile = nodeCertFile - nodeConfig.AgentConfig.NodeKeyFile = nodeKeyFile + nodeConfig.AgentConfig.ServingKubeletCert = servingKubeletCert + nodeConfig.AgentConfig.ServingKubeletKey = servingKubeletKey nodeConfig.AgentConfig.ClusterDNS = controlConfig.ClusterDNS nodeConfig.AgentConfig.ClusterDomain = controlConfig.ClusterDomain nodeConfig.AgentConfig.ResolvConf = locateOrGenerateResolvConf(envInfo) - nodeConfig.AgentConfig.CACertPath = clientCA + nodeConfig.AgentConfig.ClientCA = clientCAFile nodeConfig.AgentConfig.ListenAddress = "0.0.0.0" - nodeConfig.AgentConfig.KubeConfig = kubeConfig + nodeConfig.AgentConfig.KubeConfigNode = kubeconfigNode + nodeConfig.AgentConfig.KubeConfigKubelet = kubeconfigKubelet + nodeConfig.AgentConfig.KubeConfigKubeProxy = kubeconfigKubeproxy nodeConfig.AgentConfig.RootDir = filepath.Join(envInfo.DataDir, "kubelet") nodeConfig.AgentConfig.PauseImage = envInfo.PauseImage nodeConfig.CACerts = info.CACerts @@ -316,7 +369,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { nodeConfig.Containerd.Address = filepath.Join(nodeConfig.Containerd.State, "containerd.sock") nodeConfig.Containerd.Template = filepath.Join(envInfo.DataDir, "etc/containerd/config.toml.tmpl") nodeConfig.ServerAddress = serverURLParsed.Host - nodeConfig.Certificate = nodeCert + nodeConfig.Certificate = servingCert if !nodeConfig.NoFlannel { nodeConfig.FlannelConf = filepath.Join(envInfo.DataDir, "etc/flannel/net-conf.json") nodeConfig.AgentConfig.CNIBinDir = filepath.Dir(hostLocal) diff --git a/pkg/agent/flannel/setup.go b/pkg/agent/flannel/setup.go index e90158174e..c2da4f347f 100644 --- a/pkg/agent/flannel/setup.go +++ b/pkg/agent/flannel/setup.go @@ -55,7 +55,7 @@ func Prepare(ctx context.Context, config *config.Node) error { func Run(ctx context.Context, config *config.Node) error { nodeName := config.AgentConfig.NodeName - restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) + restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode) if err != nil { return err } @@ -79,7 +79,7 @@ func Run(ctx context.Context, config *config.Node) error { } go func() { - err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfig) + err := flannel(ctx, config.FlannelIface, config.FlannelConf, config.AgentConfig.KubeConfigNode) logrus.Fatalf("flannel exited: %v", err) }() diff --git a/pkg/agent/proxy/proxy.go b/pkg/agent/proxy/proxy.go index 21d6dc91a9..e93a7be39c 100644 --- a/pkg/agent/proxy/proxy.go +++ b/pkg/agent/proxy/proxy.go @@ -1,35 +1,18 @@ package proxy import ( - "crypto/tls" - "net/http" - - "github.com/pkg/errors" + "github.com/google/tcpproxy" "github.com/rancher/k3s/pkg/daemons/config" - "github.com/rancher/k3s/pkg/proxy" "github.com/sirupsen/logrus" ) func Run(config *config.Node) error { - proxy, err := proxy.NewSimpleProxy(config.ServerAddress, config.CACerts, true) - if err != nil { - return err - } - - listener, err := tls.Listen("tcp", config.LocalAddress, &tls.Config{ - Certificates: []tls.Certificate{ - *config.Certificate, - }, - }) - - if err != nil { - return errors.Wrap(err, "Failed to start tls listener") - } - + logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress) + var proxy tcpproxy.Proxy + proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress)) go func() { - err := http.Serve(listener, proxy) + err := proxy.Run() logrus.Fatalf("TLS proxy stopped: %v", err) }() - return nil } diff --git a/pkg/agent/tunnel/tunnel.go b/pkg/agent/tunnel/tunnel.go index 6ce3a9938c..43ec741adc 100644 --- a/pkg/agent/tunnel/tunnel.go +++ b/pkg/agent/tunnel/tunnel.go @@ -26,7 +26,7 @@ var ( ) func Setup(config *config.Node) error { - restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfig) + restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode) if err != nil { return err } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index d4c65e4b31..0a2d2b2b63 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -17,7 +17,7 @@ type Server struct { DisableAgent bool KubeConfigOutput string KubeConfigMode string - KnownIPs cli.StringSlice + TLSSan cli.StringSlice BindAddress string ExtraAPIArgs cli.StringSlice ExtraSchedulerArgs cli.StringSlice @@ -28,6 +28,8 @@ type Server struct { StorageCAFile string StorageCertFile string StorageKeyFile string + AdvertiseIP string + AdvertisePort int } var ServerConfig Server @@ -120,7 +122,7 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { cli.StringSliceFlag{ Name: "tls-san", Usage: "Add additional hostname or IP as a Subject Alternative Name in the TLS cert", - Value: &ServerConfig.KnownIPs, + Value: &ServerConfig.TLSSan, }, cli.StringSliceFlag{ Name: "kube-apiserver-arg", @@ -172,6 +174,17 @@ func NewServerCommand(action func(*cli.Context) error) cli.Command { Destination: &ServerConfig.StorageKeyFile, EnvVar: "K3S_STORAGE_KEYFILE", }, + cli.StringFlag{ + Name: "advertise-address", + Usage: "IP address that apiserver uses to advertise to members of the cluster", + Destination: &ServerConfig.AdvertiseIP, + }, + cli.IntFlag{ + Name: "advertise-port", + Usage: "Port that apiserver uses to advertise to members of the cluster", + Value: 0, + Destination: &ServerConfig.AdvertisePort, + }, NodeIPFlag, NodeNameFlag, DockerFlag, diff --git a/pkg/cli/server/server.go b/pkg/cli/server/server.go index 66a8a93938..fa4f8f1141 100644 --- a/pkg/cli/server/server.go +++ b/pkg/cli/server/server.go @@ -23,6 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli" "k8s.io/apimachinery/pkg/util/net" + "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/pkg/volume/csi" _ "github.com/go-sql-driver/mysql" // ensure we have mysql @@ -102,8 +103,16 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.Rootless = cfg.Rootless serverConfig.TLSConfig.HTTPSPort = cfg.HTTPSPort serverConfig.TLSConfig.HTTPPort = cfg.HTTPPort - serverConfig.TLSConfig.KnownIPs = knownIPs(cfg.KnownIPs) + for _, san := range knownIPs(cfg.TLSSan) { + addr := net2.ParseIP(san) + if addr != nil { + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, san) + } else { + serverConfig.TLSConfig.Domains = append(serverConfig.TLSConfig.Domains, san) + } + } serverConfig.TLSConfig.BindAddress = cfg.BindAddress + serverConfig.ControlConfig.HTTPSPort = cfg.HTTPSPort serverConfig.ControlConfig.ExtraAPIArgs = cfg.ExtraAPIArgs serverConfig.ControlConfig.ExtraControllerArgs = cfg.ExtraControllerArgs serverConfig.ControlConfig.ExtraSchedulerAPIArgs = cfg.ExtraSchedulerArgs @@ -113,6 +122,15 @@ func run(app *cli.Context, cfg *cmds.Server) error { serverConfig.ControlConfig.StorageCAFile = cfg.StorageCAFile serverConfig.ControlConfig.StorageCertFile = cfg.StorageCertFile serverConfig.ControlConfig.StorageKeyFile = cfg.StorageKeyFile + serverConfig.ControlConfig.AdvertiseIP = cfg.AdvertiseIP + serverConfig.ControlConfig.AdvertisePort = cfg.AdvertisePort + + if serverConfig.ControlConfig.AdvertiseIP == "" && cmds.AgentConfig.NodeIP != "" { + serverConfig.ControlConfig.AdvertiseIP = cmds.AgentConfig.NodeIP + } + if serverConfig.ControlConfig.AdvertiseIP != "" { + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, serverConfig.ControlConfig.AdvertiseIP) + } _, serverConfig.ControlConfig.ClusterIPRange, err = net2.ParseCIDR(cfg.ClusterCIDR) if err != nil { @@ -123,6 +141,12 @@ func run(app *cli.Context, cfg *cmds.Server) error { return errors.Wrapf(err, "Invalid CIDR %s: %v", cfg.ServiceCIDR, err) } + _, apiServerServiceIP, err := master.DefaultServiceIPRange(*serverConfig.ControlConfig.ServiceIPRange) + if err != nil { + return err + } + serverConfig.TLSConfig.KnownIPs = append(serverConfig.TLSConfig.KnownIPs, apiServerServiceIP.String()) + // If cluster-dns CLI arg is not set, we set ClusterDNS address to be ServiceCIDR network + 10, // i.e. when you set service-cidr to 192.168.0.0/16 and don't provide cluster-dns, it will be set to 192.168.0.10 if cfg.ClusterDNS == "" { diff --git a/pkg/daemons/agent/agent.go b/pkg/daemons/agent/agent.go index 7702e0e56e..1bd83a7e0d 100644 --- a/pkg/daemons/agent/agent.go +++ b/pkg/daemons/agent/agent.go @@ -35,7 +35,7 @@ func kubeProxy(cfg *config.Agent) { argsMap := map[string]string{ "proxy-mode": "iptables", "healthz-bind-address": "127.0.0.1", - "kubeconfig": cfg.KubeConfig, + "kubeconfig": cfg.KubeConfigKubeProxy, "cluster-cidr": cfg.ClusterCIDR.String(), } args := config.GetArgsList(argsMap, cfg.ExtraKubeProxyArgs) @@ -58,7 +58,7 @@ func kubelet(cfg *config.Agent) { "read-only-port": "0", "allow-privileged": "true", "cluster-domain": cfg.ClusterDomain, - "kubeconfig": cfg.KubeConfig, + "kubeconfig": cfg.KubeConfigKubelet, "eviction-hard": "imagefs.available<5%,nodefs.available<5%", "eviction-minimum-reclaim": "imagefs.available=10%,nodefs.available=10%", "fail-swap-on": "false", @@ -95,13 +95,13 @@ func kubelet(cfg *config.Agent) { if cfg.ListenAddress != "" { argsMap["address"] = cfg.ListenAddress } - if cfg.CACertPath != "" { + if cfg.ClientCA != "" { argsMap["anonymous-auth"] = "false" - argsMap["client-ca-file"] = cfg.CACertPath + argsMap["client-ca-file"] = cfg.ClientCA } - if cfg.NodeCertFile != "" && cfg.NodeKeyFile != "" { - argsMap["tls-cert-file"] = cfg.NodeCertFile - argsMap["tls-private-key-file"] = cfg.NodeKeyFile + if cfg.ServingKubeletCert != "" && cfg.ServingKubeletKey != "" { + argsMap["tls-cert-file"] = cfg.ServingKubeletCert + argsMap["tls-private-key-file"] = cfg.ServingKubeletKey } if cfg.NodeName != "" { argsMap["hostname-override"] = cfg.NodeName diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 92fc1641b4..489684562e 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -36,32 +36,41 @@ type Containerd struct { } type Agent struct { - NodeName string - NodeCertFile string - NodeKeyFile string - ClusterCIDR net.IPNet - ClusterDNS net.IP - ClusterDomain string - ResolvConf string - RootDir string - KubeConfig string - NodeIP string - RuntimeSocket string - ListenAddress string - CACertPath string - CNIBinDir string - CNIConfDir string - ExtraKubeletArgs []string - ExtraKubeProxyArgs []string - PauseImage string - CNIPlugin bool - NodeTaints []string - NodeLabels []string + NodeName string + ClientKubeletCert string + ClientKubeletKey string + ClientKubeProxyCert string + ClientKubeProxyKey string + ServingKubeletCert string + ServingKubeletKey string + ClusterCIDR net.IPNet + ClusterDNS net.IP + ClusterDomain string + ResolvConf string + RootDir string + KubeConfigNode string + KubeConfigKubelet string + KubeConfigKubeProxy string + NodeIP string + RuntimeSocket string + ListenAddress string + ClientCA string + CNIBinDir string + CNIConfDir string + ExtraKubeletArgs []string + ExtraKubeProxyArgs []string + PauseImage string + CNIPlugin bool + NodeTaints []string + NodeLabels []string } type Control struct { AdvertisePort int + AdvertiseIP string ListenPort int + HTTPSPort int + ProxyPort int ClusterSecret string ClusterIPRange *net.IPNet ServiceIPRange *net.IPNet @@ -87,28 +96,44 @@ type Control struct { } type ControlRuntime struct { - TLSCert string - TLSKey string - TLSCA string - TLSCAKey string - TokenCA string - TokenCAKey string - ServiceKey string - PasswdFile string - KubeConfigSystem string + ClientKubeAPICert string + ClientKubeAPIKey string + ClientCA string + ClientCAKey string + ServerCA string + ServerCAKey string + ServiceKey string + PasswdFile string - NodeCert string - NodeKey string - ClientToken string - NodeToken string - Handler http.Handler - Tunnel http.Handler - Authenticator authenticator.Request + KubeConfigAdmin string + KubeConfigController string + KubeConfigScheduler string + KubeConfigAPIServer string + + ServingKubeAPICert string + ServingKubeAPIKey string + ClientToken string + NodeToken string + Handler http.Handler + Tunnel http.Handler + Authenticator authenticator.Request RequestHeaderCA string RequestHeaderCAKey string ClientAuthProxyCert string ClientAuthProxyKey string + + ClientAdminCert string + ClientAdminKey string + ClientControllerCert string + ClientControllerKey string + ClientSchedulerCert string + ClientSchedulerKey string + ClientKubeProxyCert string + ClientKubeProxyKey string + + ServingKubeletKey string + ClientKubeletKey string } type ArgString []string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index c9cedc834b..fa9299d71a 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -1,12 +1,10 @@ package control import ( - "bufio" "context" + "crypto" cryptorand "crypto/rand" - "crypto/rsa" "crypto/x509" - "encoding/base64" "encoding/csv" "encoding/hex" "fmt" @@ -41,14 +39,12 @@ import ( var ( localhostIP = net.ParseIP("127.0.0.1") - x509KeyServerOnly = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} - x509KeyClientUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} - requestHeaderCN = "kubernetes-proxy" + requestHeaderCN = "system:auth-proxy" kubeconfigTemplate = template.Must(template.New("kubeconfig").Parse(`apiVersion: v1 clusters: - cluster: server: {{.URL}} - certificate-authority-data: {{.CACert}} + certificate-authority: {{.CACert}} name: local contexts: - context: @@ -62,8 +58,8 @@ preferences: {} users: - name: user user: - username: {{.User}} - password: {{.Password}} + client-certificate: {{.ClientCert}} + client-key: {{.ClientKey}} `)) ) @@ -99,14 +95,17 @@ func Server(ctx context.Context, cfg *config.Control) error { func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { argsMap := map[string]string{ - "kubeconfig": runtime.KubeConfigSystem, + "kubeconfig": runtime.KubeConfigController, "service-account-private-key-file": runtime.ServiceKey, "allocate-node-cidrs": "true", "cluster-cidr": cfg.ClusterIPRange.String(), - "root-ca-file": runtime.TokenCA, + "root-ca-file": runtime.ServerCA, "port": "10252", - "bind-address": "127.0.0.1", + "bind-address": localhostIP.String(), "secure-port": "0", + "use-service-account-credentials": "true", + "cluster-signing-cert-file": runtime.ServerCA, + "cluster-signing-key-file": runtime.ServerCAKey, } if cfg.NoLeaderElect { argsMap["leader-elect"] = "false" @@ -125,7 +124,7 @@ func controllerManager(cfg *config.Control, runtime *config.ControlRuntime) { func scheduler(cfg *config.Control, runtime *config.ControlRuntime) { argsMap := map[string]string{ - "kubeconfig": runtime.KubeConfigSystem, + "kubeconfig": runtime.KubeConfigScheduler, "port": "10251", "bind-address": "127.0.0.1", "secure-port": "0", @@ -163,18 +162,20 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["service-account-signing-key-file"] = runtime.ServiceKey argsMap["service-cluster-ip-range"] = cfg.ServiceIPRange.String() argsMap["advertise-port"] = strconv.Itoa(cfg.AdvertisePort) - argsMap["advertise-address"] = localhostIP.String() + if cfg.AdvertiseIP != "" { + argsMap["advertise-address"] = cfg.AdvertiseIP + } argsMap["insecure-port"] = "0" argsMap["secure-port"] = strconv.Itoa(cfg.ListenPort) argsMap["bind-address"] = localhostIP.String() - argsMap["tls-cert-file"] = runtime.TLSCert - argsMap["tls-private-key-file"] = runtime.TLSKey + argsMap["tls-cert-file"] = runtime.ServingKubeAPICert + argsMap["tls-private-key-file"] = runtime.ServingKubeAPIKey argsMap["service-account-key-file"] = runtime.ServiceKey argsMap["service-account-issuer"] = "k3s" argsMap["api-audiences"] = "unknown" argsMap["basic-auth-file"] = runtime.PasswdFile - argsMap["kubelet-client-certificate"] = runtime.NodeCert - argsMap["kubelet-client-key"] = runtime.NodeKey + argsMap["kubelet-client-certificate"] = runtime.ClientKubeAPICert + argsMap["kubelet-client-key"] = runtime.ClientKubeAPIKey argsMap["requestheader-client-ca-file"] = runtime.RequestHeaderCA argsMap["requestheader-allowed-names"] = requestHeaderCN argsMap["proxy-client-cert-file"] = runtime.ClientAuthProxyCert @@ -182,6 +183,8 @@ func apiServer(ctx context.Context, cfg *config.Control, runtime *config.Control argsMap["requestheader-extra-headers-prefix"] = "X-Remote-Extra-" argsMap["requestheader-group-headers"] = "X-Remote-Group" argsMap["requestheader-username-headers"] = "X-Remote-User" + argsMap["client-ca-file"] = runtime.ClientCA + argsMap["enable-admission-plugins"] = "NodeRestriction" args := config.GetArgsList(argsMap, cfg.ExtraAPIArgs) @@ -214,13 +217,17 @@ func defaults(config *config.Control) { } if config.AdvertisePort == 0 { - config.AdvertisePort = 6445 + config.AdvertisePort = config.HTTPSPort } if config.ListenPort == 0 { config.ListenPort = 6444 } + if config.ProxyPort == 0 { + config.ProxyPort = 6445 + } + if config.DataDir == "" { config.DataDir = "./management-state" } @@ -247,22 +254,40 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { os.MkdirAll(path.Join(config.DataDir, "tls"), 0700) os.MkdirAll(path.Join(config.DataDir, "cred"), 0700) - name := "localhost" - runtime.TLSCert = path.Join(config.DataDir, "tls", name+".crt") - runtime.TLSKey = path.Join(config.DataDir, "tls", name+".key") - runtime.TLSCA = path.Join(config.DataDir, "tls", "ca.crt") - runtime.TLSCAKey = path.Join(config.DataDir, "tls", "ca.key") - runtime.TokenCA = path.Join(config.DataDir, "tls", "token-ca.crt") - runtime.TokenCAKey = path.Join(config.DataDir, "tls", "token-ca.key") - runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key") - runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd") - runtime.KubeConfigSystem = path.Join(config.DataDir, "cred", "kubeconfig-system.yaml") - runtime.NodeKey = path.Join(config.DataDir, "tls", "token-node.key") - runtime.NodeCert = path.Join(config.DataDir, "tls", "token-node-1.crt") + runtime.ClientCA = path.Join(config.DataDir, "tls", "client-ca.crt") + runtime.ClientCAKey = path.Join(config.DataDir, "tls", "client-ca.key") + runtime.ServerCA = path.Join(config.DataDir, "tls", "server-ca.crt") + runtime.ServerCAKey = path.Join(config.DataDir, "tls", "server-ca.key") runtime.RequestHeaderCA = path.Join(config.DataDir, "tls", "request-header-ca.crt") runtime.RequestHeaderCAKey = path.Join(config.DataDir, "tls", "request-header-ca.key") - runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") + + runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key") + runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd") + + runtime.KubeConfigAdmin = path.Join(config.DataDir, "cred", "admin.kubeconfig") + runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig") + runtime.KubeConfigScheduler = path.Join(config.DataDir, "cred", "scheduler.kubeconfig") + runtime.KubeConfigAPIServer = path.Join(config.DataDir, "cred", "api-server.kubeconfig") + + runtime.ClientAdminCert = path.Join(config.DataDir, "tls", "client-admin.crt") + runtime.ClientAdminKey = path.Join(config.DataDir, "tls", "client-admin.key") + runtime.ClientControllerCert = path.Join(config.DataDir, "tls", "client-controller.crt") + runtime.ClientControllerKey = path.Join(config.DataDir, "tls", "client-controller.key") + runtime.ClientSchedulerCert = path.Join(config.DataDir, "tls", "client-scheduler.crt") + runtime.ClientSchedulerKey = path.Join(config.DataDir, "tls", "client-scheduler.key") + runtime.ClientKubeAPICert = path.Join(config.DataDir, "tls", "client-kube-apiserver.crt") + runtime.ClientKubeAPIKey = path.Join(config.DataDir, "tls", "client-kube-apiserver.key") + runtime.ClientKubeProxyCert = path.Join(config.DataDir, "tls", "client-kube-proxy.crt") + runtime.ClientKubeProxyKey = path.Join(config.DataDir, "tls", "client-kube-proxy.key") + + runtime.ServingKubeAPICert = path.Join(config.DataDir, "tls", "serving-kube-apiserver.crt") + runtime.ServingKubeAPIKey = path.Join(config.DataDir, "tls", "serving-kube-apiserver.key") + + runtime.ClientKubeletKey = path.Join(config.DataDir, "tls", "client-kubelet.key") + runtime.ServingKubeletKey = path.Join(config.DataDir, "tls", "serving-kubelet.key") + runtime.ClientAuthProxyCert = path.Join(config.DataDir, "tls", "client-auth-proxy.crt") + runtime.ClientAuthProxyKey = path.Join(config.DataDir, "tls", "client-auth-proxy.key") if err := genCerts(config, runtime); err != nil { return err @@ -279,32 +304,45 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { return readTokens(runtime) } -func readTokens(runtime *config.ControlRuntime) error { - f, err := os.Open(runtime.PasswdFile) +func readTokenFile(passwdFile string) (map[string]string, error) { + f, err := os.Open(passwdFile) if err != nil { - return err + return nil, err } + defer f.Close() + reader := csv.NewReader(f) reader.FieldsPerRecord = -1 + tokens := map[string]string{} + for { record, err := reader.Read() if err == io.EOF { break } if err != nil { - return err + return nil, err } if len(record) < 2 { continue } + tokens[record[1]] = record[0] + } + return tokens, nil +} - switch record[1] { - case "node": - runtime.NodeToken = "node:" + record[0] - case "admin": - runtime.ClientToken = "admin:" + record[0] - } +func readTokens(runtime *config.ControlRuntime) error { + tokens, err := readTokenFile(runtime.PasswdFile) + if err != nil { + return err + } + + if nodeToken, ok := tokens["node"]; ok { + runtime.NodeToken = "node:" + nodeToken + } + if clientToken, ok := tokens["admin"]; ok { + runtime.ClientToken = "admin:" + clientToken } return nil @@ -321,31 +359,48 @@ func ensureNodeToken(config *config.Control, runtime *config.ControlRuntime) err } defer f.Close() - buf := &strings.Builder{} - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - parts := strings.Split(line, ",") - if len(parts) < 4 { - continue + records := [][]string{} + reader := csv.NewReader(f) + for { + record, err := reader.Read() + if err == io.EOF { + break } - if parts[1] == "node" { - if parts[0] == config.ClusterSecret { + if err != nil { + return err + } + if len(record) < 3 { + return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", runtime.PasswdFile, len(record)) + } + if record[1] == "node" { + if record[0] == config.ClusterSecret { return nil } - parts[0] = config.ClusterSecret - line = strings.Join(parts, ",") + record[0] = config.ClusterSecret } - buf.WriteString(line) - buf.WriteString("\n") - } - - if scan.Err() != nil { - return scan.Err() + records = append(records, record) } f.Close() - return ioutil.WriteFile(runtime.PasswdFile, []byte(buf.String()), 0600) + return WritePasswords(runtime.PasswdFile, records) +} + +func WritePasswords(passwdFile string, records [][]string) error { + out, err := os.Create(passwdFile + ".tmp") + if err != nil { + return err + } + defer out.Close() + + if err := out.Chmod(0600); err != nil { + return err + } + + if err := csv.NewWriter(out).WriteAll(records); err != nil { + return err + } + + return os.Rename(passwdFile+".tmp", passwdFile) } func genUsers(config *config.Control, runtime *config.ControlRuntime) error { @@ -370,24 +425,11 @@ func genUsers(config *config.Control, runtime *config.ControlRuntime) error { nodeToken = config.ClusterSecret } - passwd := fmt.Sprintf(`%s,admin,admin,system:masters -%s,system,system,system:masters -%s,node,node,system:masters -`, adminToken, systemToken, nodeToken) - - caCertBytes, err := ioutil.ReadFile(runtime.TLSCA) - if err != nil { - return err - } - - caCert := base64.StdEncoding.EncodeToString(caCertBytes) - - if err := kubeConfig(runtime.KubeConfigSystem, fmt.Sprintf("https://localhost:%d", config.ListenPort), caCert, - "system", systemToken); err != nil { - return err - } - - return ioutil.WriteFile(runtime.PasswdFile, []byte(passwd), 0600) + return WritePasswords(runtime.PasswdFile, [][]string{ + {adminToken, "admin", "admin", "system:masters"}, + {systemToken, "system", "system", "system:masters"}, + {nodeToken, "node", "node", "system:masters"}, + }) } func getToken() (string, error) { @@ -400,10 +442,10 @@ func getToken() (string, error) { } func genCerts(config *config.Control, runtime *config.ControlRuntime) error { - if err := genTLSCerts(config, runtime); err != nil { + if err := genClientCerts(config, runtime); err != nil { return err } - if err := genTokenCerts(config, runtime); err != nil { + if err := genServerCerts(config, runtime); err != nil { return err } if err := genRequestHeaderCerts(config, runtime); err != nil { @@ -412,32 +454,78 @@ func genCerts(config *config.Control, runtime *config.ControlRuntime) error { return nil } -func genTLSCerts(config *config.Control, runtime *config.ControlRuntime) error { - regen, err := createSigningCertKey("k3s-tls", runtime.TLSCA, runtime.TLSCAKey) +type signedCertFactory = func(commonName string, organization []string, certFile, keyFile string) (bool, error) + +func getSigningCertFactory(regen bool, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile string) signedCertFactory { + return func(commonName string, organization []string, certFile, keyFile string) (bool, error) { + return createClientCertKey(regen, commonName, organization, altNames, extKeyUsage, caCertFile, caKeyFile, certFile, keyFile) + } +} + +func genClientCerts(config *config.Control, runtime *config.ControlRuntime) error { + regen, err := createSigningCertKey("k3s-client", runtime.ClientCA, runtime.ClientCAKey) if err != nil { return err } - _, apiServerServiceIP, err := master.DefaultServiceIPRange(*config.ServiceIPRange) + factory := getSigningCertFactory(regen, nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.ClientCA, runtime.ClientCAKey) + + var certGen bool + apiEndpoint := fmt.Sprintf("https://localhost:%d", config.ListenPort) + + certGen, err = factory("system:admin", []string{"system:masters"}, runtime.ClientAdminCert, runtime.ClientAdminKey) if err != nil { return err } + if certGen { + if err := KubeConfig(runtime.KubeConfigAdmin, apiEndpoint, runtime.ServerCA, runtime.ClientAdminCert, runtime.ClientAdminKey); err != nil { + return err + } + } - if err := createClientCertKey(regen, "localhost", - nil, &certutil.AltNames{ - DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}, - IPs: []net.IP{apiServerServiceIP, localhostIP}, - }, x509KeyServerOnly, - runtime.TLSCA, runtime.TLSCAKey, - runtime.TLSCert, runtime.TLSKey); err != nil { + certGen, err = factory("system:kube-controller-manager", nil, runtime.ClientControllerCert, runtime.ClientControllerKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigController, apiEndpoint, runtime.ServerCA, runtime.ClientControllerCert, runtime.ClientControllerKey); err != nil { + return err + } + } + + certGen, err = factory("system:kube-scheduler", nil, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigScheduler, apiEndpoint, runtime.ServerCA, runtime.ClientSchedulerCert, runtime.ClientSchedulerKey); err != nil { + return err + } + } + + certGen, err = factory("kube-apiserver", nil, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey) + if err != nil { + return err + } + if certGen { + if err := KubeConfig(runtime.KubeConfigAPIServer, apiEndpoint, runtime.ServerCA, runtime.ClientKubeAPICert, runtime.ClientKubeAPIKey); err != nil { + return err + } + } + + if _, err = factory("system:kube-proxy", []string{"system:nodes"}, runtime.ClientKubeProxyCert, runtime.ClientKubeProxyKey); err != nil { + return err + } + + if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ClientKubeletKey); err != nil { return err } return nil } -func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error { - regen, err := createSigningCertKey("k3s-token", runtime.TokenCA, runtime.TokenCAKey) +func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error { + regen, err := createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey) if err != nil { return err } @@ -447,13 +535,17 @@ func genTokenCerts(config *config.Control, runtime *config.ControlRuntime) error return err } - if err := createClientCertKey(regen, "kubernetes", []string{"system:masters"}, + if _, err := createClientCertKey(regen, "kube-apiserver", nil, &certutil.AltNames{ DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"}, IPs: []net.IP{apiServerServiceIP, localhostIP}, - }, x509KeyClientUsage, - runtime.TokenCA, runtime.TokenCAKey, - runtime.NodeCert, runtime.NodeKey); err != nil { + }, []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + runtime.ServerCA, runtime.ServerCAKey, + runtime.ServingKubeAPICert, runtime.ServingKubeAPIKey); err != nil { + return err + } + + if _, _, err := certutil.LoadOrGenerateKeyFile(runtime.ServingKubeletKey); err != nil { return err } @@ -466,8 +558,8 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim return err } - if err := createClientCertKey(regen, requestHeaderCN, - nil, nil, x509KeyClientUsage, + if _, err := createClientCertKey(regen, requestHeaderCN, nil, + nil, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, runtime.RequestHeaderCA, runtime.RequestHeaderCAKey, runtime.ClientAuthProxyCert, runtime.ClientAuthProxyKey); err != nil { return err @@ -476,36 +568,41 @@ func genRequestHeaderCerts(config *config.Control, runtime *config.ControlRuntim return nil } -func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) error { +func createClientCertKey(regen bool, commonName string, organization []string, altNames *certutil.AltNames, extKeyUsage []x509.ExtKeyUsage, caCertFile, caKeyFile, certFile, keyFile string) (bool, error) { if !regen { if exists(certFile, keyFile) { - return nil + return false, nil } } caKeyBytes, err := ioutil.ReadFile(caKeyFile) if err != nil { - return err - } - - caBytes, err := ioutil.ReadFile(caCertFile) - if err != nil { - return err + return false, err } caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) if err != nil { - return err + return false, err + } + + caBytes, err := ioutil.ReadFile(caCertFile) + if err != nil { + return false, err } caCert, err := certutil.ParseCertsPEM(caBytes) if err != nil { - return err + return false, err } - key, err := certutil.NewPrivateKey() + keyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile) if err != nil { - return err + return false, err + } + + key, err := certutil.ParsePrivateKeyPEM(keyBytes) + if err != nil { + return false, err } cfg := certutil.Config{ @@ -516,16 +613,12 @@ func createClientCertKey(regen bool, commonName string, organization []string, a if altNames != nil { cfg.AltNames = *altNames } - cert, err := certutil.NewSignedCert(cfg, key, caCert[0], caKey.(*rsa.PrivateKey)) + cert, err := certutil.NewSignedCert(cfg, key.(crypto.Signer), caCert[0], caKey.(crypto.Signer)) if err != nil { - return err + return false, err } - if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(key)); err != nil { - return err - } - - return certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) + return true, certutil.WriteCert(certFile, append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) } func exists(files ...string) bool { @@ -556,7 +649,12 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) { return false, nil } - caKey, err := certutil.NewPrivateKey() + caKeyBytes, _, err := certutil.LoadOrGenerateKeyFile(keyFile) + if err != nil { + return false, err + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) if err != nil { return false, err } @@ -565,32 +663,28 @@ func createSigningCertKey(prefix, certFile, keyFile string) (bool, error) { CommonName: fmt.Sprintf("%s-ca@%d", prefix, time.Now().Unix()), } - cert, err := certutil.NewSelfSignedCACert(cfg, caKey) + cert, err := certutil.NewSelfSignedCACert(cfg, caKey.(crypto.Signer)) if err != nil { return false, err } - if err := certutil.WriteKey(keyFile, certutil.EncodePrivateKeyPEM(caKey)); err != nil { - return false, err - } - if err := certutil.WriteCert(certFile, certutil.EncodeCertPEM(cert)); err != nil { return false, err } return true, nil } -func kubeConfig(dest, url, cert, user, password string) error { +func KubeConfig(dest, url, caCert, clientCert, clientKey string) error { data := struct { - URL string - CACert string - User string - Password string + URL string + CACert string + ClientCert string + ClientKey string }{ - URL: url, - CACert: cert, - User: user, - Password: password, + URL: url, + CACert: caCert, + ClientCert: clientCert, + ClientKey: clientKey, } output, err := os.Create(dest) diff --git a/pkg/server/router.go b/pkg/server/router.go index c5a630423b..06adb05b89 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -1,11 +1,12 @@ package server import ( - "bufio" - "crypto/rsa" + "crypto" "crypto/x509" + "encoding/csv" "errors" "fmt" + "io" "io/ioutil" "net" "net/http" @@ -17,10 +18,10 @@ import ( "github.com/gorilla/mux" certutil "github.com/rancher/dynamiclistener/cert" "github.com/rancher/k3s/pkg/daemons/config" + "github.com/rancher/k3s/pkg/daemons/control" "github.com/rancher/k3s/pkg/openapi" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/kubernetes/pkg/master" ) const ( @@ -38,8 +39,14 @@ func router(serverConfig *config.Control, tunnel http.Handler, cacertsGetter CAC authed.Use(authMiddleware(serverConfig)) authed.NotFoundHandler = serverConfig.Runtime.Handler authed.Path("/v1-k3s/connect").Handler(tunnel) - authed.Path("/v1-k3s/node.crt").Handler(nodeCrt(serverConfig)) - authed.Path("/v1-k3s/node.key").Handler(nodeKey(serverConfig)) + authed.Path("/v1-k3s/serving-kubelet.crt").Handler(servingKubeletCert(serverConfig)) + authed.Path("/v1-k3s/serving-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ServingKubeletKey)) + authed.Path("/v1-k3s/client-kubelet.crt").Handler(clientKubeletCert(serverConfig)) + authed.Path("/v1-k3s/client-kubelet.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeletKey)) + authed.Path("/v1-k3s/client-kube-proxy.crt").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyCert)) + authed.Path("/v1-k3s/client-kube-proxy.key").Handler(fileHandler(serverConfig.Runtime.ClientKubeProxyKey)) + authed.Path("/v1-k3s/client-ca.crt").Handler(fileHandler(serverConfig.Runtime.ClientCA)) + authed.Path("/v1-k3s/server-ca.crt").Handler(fileHandler(serverConfig.Runtime.ServerCA)) authed.Path("/v1-k3s/config").Handler(configHandler(serverConfig)) staticDir := filepath.Join(serverConfig.DataDir, "static") @@ -65,82 +72,85 @@ func cacerts(getter CACertsGetter) http.Handler { }) } -func nodeCrt(server *config.Control) http.Handler { +func getNodeInfo(req *http.Request) (string, string, error) { + nodeNames := req.Header["K3s-Node-Name"] + if len(nodeNames) != 1 || nodeNames[0] == "" { + return "", "", errors.New("node name not set") + } + + nodePasswords := req.Header["K3s-Node-Password"] + if len(nodePasswords) != 1 || nodePasswords[0] == "" { + return "", "", errors.New("node password not set") + } + + return nodeNames[0], nodePasswords[0], nil +} + +func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) { + keyBytes, err := ioutil.ReadFile(signingKeyFile) + if err != nil { + return nil, nil, nil, err + } + + key, err := certutil.ParsePrivateKeyPEM(keyBytes) + if err != nil { + return nil, nil, nil, err + } + + caKeyBytes, err := ioutil.ReadFile(caKeyFile) + if err != nil { + return nil, nil, nil, err + } + + caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) + if err != nil { + return nil, nil, nil, err + } + + caBytes, err := ioutil.ReadFile(caCertFile) + if err != nil { + return nil, nil, nil, err + } + + caCert, err := certutil.ParseCertsPEM(caBytes) + if err != nil { + return nil, nil, nil, err + } + + return caCert, caKey.(crypto.Signer), key.(crypto.Signer), nil +} + +func servingKubeletCert(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - nodeNames := req.Header["K3s-Node-Name"] - if len(nodeNames) != 1 || nodeNames[0] == "" { - sendError(errors.New("node name not set"), resp) - return + nodeName, nodePassword, err := getNodeInfo(req) + if err != nil { + sendError(err, resp) } - nodePasswords := req.Header["K3s-Node-Password"] - if len(nodePasswords) != 1 || nodePasswords[0] == "" { - sendError(errors.New("node password not set"), resp) - return - } - - if err := ensureNodePassword(server.Runtime.PasswdFile, nodeNames[0], nodePasswords[0]); err != nil { + if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { sendError(err, resp, http.StatusForbidden) return } - nodeKey, err := ioutil.ReadFile(server.Runtime.NodeKey) + caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ServerCA, server.Runtime.ServerCAKey, server.Runtime.ServingKubeletKey) if err != nil { sendError(err, resp) return } - key, err := certutil.ParsePrivateKeyPEM(nodeKey) - if err != nil { - sendError(err, resp) - return - } - - caKeyBytes, err := ioutil.ReadFile(server.Runtime.TokenCAKey) - if err != nil { - sendError(err, resp) - return - } - - caBytes, err := ioutil.ReadFile(server.Runtime.TokenCA) - if err != nil { - sendError(err, resp) - return - } - - caKey, err := certutil.ParsePrivateKeyPEM(caKeyBytes) - if err != nil { - sendError(err, resp) - return - } - - caCert, err := certutil.ParseCertsPEM(caBytes) - if err != nil { - sendError(err, resp) - return - } - - _, apiServerServiceIP, err := master.DefaultServiceIPRange(*server.ServiceIPRange) - if err != nil { - sendError(err, resp) - return - } - - cfg := certutil.Config{ - CommonName: "kubernetes", - Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + cert, err := certutil.NewSignedCert(certutil.Config{ + CommonName: nodeName, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, AltNames: certutil.AltNames{ - DNSNames: []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost", nodeNames[0]}, - IPs: []net.IP{apiServerServiceIP, net.ParseIP("127.0.0.1")}, + DNSNames: []string{nodeName, "localhost"}, + IPs: []net.IP{net.ParseIP("127.0.0.1")}, }, - } - - cert, err := certutil.NewSignedCert(cfg, key.(*rsa.PrivateKey), caCert[0], caKey.(*rsa.PrivateKey)) + }, key, caCert[0], caKey) if err != nil { sendError(err, resp) return @@ -150,13 +160,50 @@ func nodeCrt(server *config.Control) http.Handler { }) } -func nodeKey(server *config.Control) http.Handler { +func clientKubeletCert(server *config.Control) http.Handler { return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { if req.TLS == nil { resp.WriteHeader(http.StatusNotFound) return } - http.ServeFile(resp, req, server.Runtime.NodeKey) + + nodeName, nodePassword, err := getNodeInfo(req) + if err != nil { + sendError(err, resp) + } + + if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { + sendError(err, resp, http.StatusForbidden) + return + } + + caCert, caKey, key, err := getCACertAndKeys(server.Runtime.ClientCA, server.Runtime.ClientCAKey, server.Runtime.ClientKubeletKey) + if err != nil { + sendError(err, resp) + return + } + + cert, err := certutil.NewSignedCert(certutil.Config{ + CommonName: "system:node:" + nodeName, + Organization: []string{"system:nodes"}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + }, key, caCert[0], caKey) + if err != nil { + sendError(err, resp) + return + } + + resp.Write(append(certutil.EncodeCertPEM(cert), certutil.EncodeCertPEM(caCert[0])...)) + }) +} + +func fileHandler(fileName string) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + if req.TLS == nil { + resp.WriteHeader(http.StatusNotFound) + return + } + http.ServeFile(resp, req, fileName) }) } @@ -225,29 +272,29 @@ func ensureNodePassword(passwdFile, nodeName, passwd string) error { defer f.Close() user := strings.ToLower("node:" + nodeName) - buf := &strings.Builder{} - scan := bufio.NewScanner(f) - for scan.Scan() { - line := scan.Text() - parts := strings.Split(line, ",") - if len(parts) < 4 { - continue + records := [][]string{} + reader := csv.NewReader(f) + for { + record, err := reader.Read() + if err == io.EOF { + break } - if parts[1] == user { - if parts[0] == passwd { + if err != nil { + return err + } + if len(record) < 3 { + return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record)) + } + if record[1] == user { + if record[0] == passwd { return nil } - return fmt.Errorf("Node password validation failed for [%s]", nodeName) + return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) } - buf.WriteString(line) - buf.WriteString("\n") - } - buf.WriteString(fmt.Sprintf("%s,%s,%s,system:masters\n", passwd, user, user)) - - if scan.Err() != nil { - return scan.Err() + records = append(records, record) } + records = append(records, []string{passwd, user, user, "system:node:" + nodeName}) f.Close() - return ioutil.WriteFile(passwdFile, []byte(buf.String()), 0600) + return control.WritePasswords(passwdFile, records) } diff --git a/pkg/server/server.go b/pkg/server/server.go index 8ab93a9ff5..7c9be7d208 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -77,6 +77,18 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { controlConfig = &config.ControlConfig ) + caBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCA) + if err != nil { + return "", err + } + caKeyBytes, err := ioutil.ReadFile(controlConfig.Runtime.ServerCAKey) + if err != nil { + return "", err + } + + tlsConfig.CACerts = string(caBytes) + tlsConfig.CAKey = string(caKeyBytes) + tlsConfig.Handler = router(controlConfig, controlConfig.Runtime.Tunnel, func() (string, error) { if tlsServer == nil { return "", nil @@ -84,7 +96,7 @@ func startWrangler(ctx context.Context, config *Config) (string, error) { return tlsServer.CACert() }) - sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigSystem) + sc, err := newContext(ctx, controlConfig.Runtime.KubeConfigAdmin) if err != nil { return "", err } From f580a32422b4a0bc370b8f25e9943f5bd063856d Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 29 May 2019 11:52:42 -0700 Subject: [PATCH 2/8] Update vendor --- trash.lock | 7 +- vendor.conf | 4 +- vendor/github.com/google/tcpproxy/.gitignore | 2 + vendor/github.com/google/tcpproxy/.travis.yml | 49 ++ .../google/tcpproxy/CONTRIBUTING.md | 8 + vendor/github.com/google/tcpproxy/LICENSE | 202 ++++++++ vendor/github.com/google/tcpproxy/README.md | 5 + vendor/github.com/google/tcpproxy/http.go | 125 +++++ vendor/github.com/google/tcpproxy/listener.go | 108 ++++ vendor/github.com/google/tcpproxy/sni.go | 192 +++++++ vendor/github.com/google/tcpproxy/tcpproxy.go | 474 ++++++++++++++++++ .../rancher/dynamiclistener/server.go | 104 +++- .../rancher/dynamiclistener/types.go | 1 + vendor/k8s.io/client-go/pkg/version/base.go | 6 +- .../kubernetes/pkg/master/controller.go | 4 + vendor/k8s.io/kubernetes/pkg/version/base.go | 6 +- 16 files changed, 1267 insertions(+), 30 deletions(-) create mode 100644 vendor/github.com/google/tcpproxy/.gitignore create mode 100644 vendor/github.com/google/tcpproxy/.travis.yml create mode 100644 vendor/github.com/google/tcpproxy/CONTRIBUTING.md create mode 100644 vendor/github.com/google/tcpproxy/LICENSE create mode 100644 vendor/github.com/google/tcpproxy/README.md create mode 100644 vendor/github.com/google/tcpproxy/http.go create mode 100644 vendor/github.com/google/tcpproxy/listener.go create mode 100644 vendor/github.com/google/tcpproxy/sni.go create mode 100644 vendor/github.com/google/tcpproxy/tcpproxy.go diff --git a/trash.lock b/trash.lock index 74d1e0dc47..79543f1566 100755 --- a/trash.lock +++ b/trash.lock @@ -125,6 +125,8 @@ import: version: v1.0.21 - package: github.com/google/gofuzz version: 44d81051d367757e1c7c6a5a86423ece9afcf63c +- package: github.com/google/tcpproxy + version: dfa16c61dad2b18a385dfb351adf71566720535b - package: github.com/googleapis/gnostic version: 0c5108395e2debce0d731cf0287ddf7242066aba - package: github.com/gorilla/mux @@ -225,7 +227,8 @@ import: - package: github.com/prometheus/procfs version: 65c1f6f8f0fc1e2185eb9863a3bc751496404259 - package: github.com/rancher/dynamiclistener - version: 077eb13a904f2c62496f31b158135d9743526f82 + version: 03208cf106d553d58d3a73267aa473a45af63120 + repo: https://github.com/erikwilson/rancher-dynamiclistener.git - package: github.com/rancher/helm-controller version: d5f5c830231110722f14d446d3b2038e5cdf1532 - package: github.com/rancher/remotedialer @@ -310,7 +313,7 @@ import: - package: k8s.io/klog version: v0.2.0-14-g8e90cee79f8237 - package: k8s.io/kubernetes - version: v1.14.3-k3s.1 + version: v1.14.3-k3s.2 repo: https://github.com/rancher/k3s.git transitive: true staging: true diff --git a/vendor.conf b/vendor.conf index 67d07ff993..34ac3014f8 100644 --- a/vendor.conf +++ b/vendor.conf @@ -9,7 +9,8 @@ package=github.com/opencontainers/runc/libcontainer/nsenter package=github.com/opencontainers/runc/libcontainer/specconv package=github.com/opencontainers/runc/contrib/cmd/recvtty -k8s.io/kubernetes v1.14.3-k3s.1 https://github.com/rancher/k3s.git transitive=true,staging=true +k8s.io/kubernetes v1.14.3-k3s.2 https://github.com/rancher/k3s.git transitive=true,staging=true +github.com/rancher/dynamiclistener 03208cf106d553d58d3a73267aa473a45af63120 https://github.com/erikwilson/rancher-dynamiclistener.git github.com/rancher/wrangler 4202dbfa88013c19238bb004d82e013f0593493d github.com/rancher/wrangler-api efe26ac6a9d720e1bfa5a8cc5f8dce5ad598ce26 @@ -29,6 +30,7 @@ golang.org/x/crypto a49355c7e3f8fe157a85be2f77e6e269a0f89602 gopkg.in/freddierice/go-losetup.v1 fc9adea44124401d8bfef3a97eaf61b5d44cc2c6 github.com/urfave/cli 8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c +github.com/google/tcpproxy dfa16c61dad2b18a385dfb351adf71566720535b # flannel github.com/golang/glog 23def4e6c14b4da8ac2ed8007337bc5eb5007998 diff --git a/vendor/github.com/google/tcpproxy/.gitignore b/vendor/github.com/google/tcpproxy/.gitignore new file mode 100644 index 0000000000..ab78466b90 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/.gitignore @@ -0,0 +1,2 @@ +tlsrouter +tlsrouter.test diff --git a/vendor/github.com/google/tcpproxy/.travis.yml b/vendor/github.com/google/tcpproxy/.travis.yml new file mode 100644 index 0000000000..9c94088fa3 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/.travis.yml @@ -0,0 +1,49 @@ +language: go +go: +- 1.8 +- tip +os: +- linux +install: +- go get github.com/golang/lint/golint +before_script: +script: +- go get -t ./... +- go build ./... +- go test ./... +- go vet ./... +- golint -set_exit_status . + +jobs: + include: + - stage: deploy + go: 1.8 + install: + - gem install fpm + script: + - go build ./cmd/tlsrouter + - fpm -s dir -t deb -n tlsrouter -v $(date '+%Y%m%d%H%M%S') + --license Apache2 + --vendor "David Anderson " + --maintainer "David Anderson " + --description "TLS SNI router" + --url "https://github.com/google/tlsrouter" + ./tlsrouter=/usr/bin/tlsrouter + ./systemd/tlsrouter.service=/lib/systemd/system/tlsrouter.service + deploy: + - provider: packagecloud + repository: tlsrouter + username: danderson + dist: debian/stretch + skip_cleanup: true + on: + branch: master + token: + secure: gNU3o70EU4oYeIS6pr0K5oLMGqqxrcf41EOv6c/YoHPVdV6Cx4j9NW0/ISgu6a1/Xf2NgWKT5BWwLpAuhmGdALuOz1Ah//YBWd9N8mGHGaC6RpOPDU8/9NkQdBEmjEH9sgX4PNOh1KQ7d7O0OH0g8RqJlJa0MkUYbTtN6KJ29oiUXxKmZM4D/iWB8VonKOnrtx1NwQL8jL8imZyEV/1fknhDwumz2iKeU1le4Neq9zkxwICMLUonmgphlrp+SDb1EOoHxT6cn51bqBQtQUplfC4dN4OQU/CPqE9E1N1noibvN29YA93qfcrjD3I95KT9wzq+3B6he33+kb0Gz+Cj5ypGy4P85l7TuX4CtQg0U3NAlJCk32IfsdjK+o47pdmADij9IIb9yKt+g99FMERkJJY5EInqEsxHlW/vNF5OqQCmpiHstZL4R2XaHEsWh6j77npnjjC1Aea8xZTWr8PTsbSzVkbG7bTmFpZoPH8eEmr4GNuw5gnbi6D1AJDjcA+UdY9s5qZNpzuWOqfhOFxL+zUW+8sHBvcoFw3R+pwHECs2LCL1c0xAC1LtNUnmW/gnwHavtvKkzErjR1P8Xl7obCbeChJjp+b/BcFYlNACldZcuzBAPyPwIdlWVyUonL4bm63upfMEEShiAIDDJ21y7fjsQK7CfPA7g25bpyo+hV8= + - provider: script + on: + branch: master + script: go run scripts/prune_old_versions.go -user=danderson -repo=tlsrouter -distro=debian -version=stretch -package=tlsrouter -arch=amd64 -limit=2 + env: + # Packagecloud API key, for prune_old_versions.go + - secure: "SRcNwt+45QyPS1w9aGxMg9905Y6d9w4mBM29G6iTTnUB5nD7cAk4m+tf834knGSobVXlWcRnTDW8zrHdQ9yX22dPqCpH5qE+qzTmIvxRHrVJRMmPeYvligJ/9jYfHgQbvuRT8cUpIcpCQAla6rw8nXfKTOE3h8XqMP2hdc3DTVOu2HCfKCNco1tJ7is+AIAnFV2Wpsbb3ZsdKFvHvi2RKUfFaX61J1GNt2/XJIlZs8jC6Y1IAC+ftjql9UsAE/WjZ9fL0Ww1b9/LBIIGHXWI3HpVv9WvlhhIxIlJgOVjmU2lbSuj2w/EBDJ9cd1Qe+wJkT3yKzE1NRsNScVjGg+Ku5igJu/XXuaHkIX01+15BqgPduBYRL0atiNQDhqgBiSyVhXZBX9vsgsp0bgpKaBSF++CV18Q9dara8aljqqS33M3imO3I8JmXU10944QA9Wvu7pCYuIzXxhINcDXRvqxBqz5LnFJGwnGqngTrOCSVS2xn7Y+sjmhe1n5cPCEISlozfa9mPYPvMPp8zg3TbATOOM8CVfcpaNscLqa/+SExN3zMwSanjNKrBgoaQcBzGW5mIgSPxhXkWikBgapiEN7+2Y032Lhqdb9dYjH+EuwcnofspDjjMabWxnuJaln+E3/9vZi2ooQrBEtvymUTy4VMSnqwIX5bU7nPdIuQycdWhk=" diff --git a/vendor/github.com/google/tcpproxy/CONTRIBUTING.md b/vendor/github.com/google/tcpproxy/CONTRIBUTING.md new file mode 100644 index 0000000000..188ad870fc --- /dev/null +++ b/vendor/github.com/google/tcpproxy/CONTRIBUTING.md @@ -0,0 +1,8 @@ +Contributions are welcome by pull request. + +You need to sign the Google Contributor License Agreement before your +contributions can be accepted. You can find the individual and organization +level CLAs here: + +Individual: https://cla.developers.google.com/about/google-individual +Organization: https://cla.developers.google.com/about/google-corporate diff --git a/vendor/github.com/google/tcpproxy/LICENSE b/vendor/github.com/google/tcpproxy/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/tcpproxy/README.md b/vendor/github.com/google/tcpproxy/README.md new file mode 100644 index 0000000000..374f52c9ee --- /dev/null +++ b/vendor/github.com/google/tcpproxy/README.md @@ -0,0 +1,5 @@ +# tcpproxy + +For library usage, see https://godoc.org/github.com/google/tcpproxy/ + +For CLI usage, see https://github.com/google/tcpproxy/blob/master/cmd/tlsrouter/README.md diff --git a/vendor/github.com/google/tcpproxy/http.go b/vendor/github.com/google/tcpproxy/http.go new file mode 100644 index 0000000000..d28c66fa88 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/http.go @@ -0,0 +1,125 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpproxy + +import ( + "bufio" + "bytes" + "context" + "net/http" +) + +// AddHTTPHostRoute appends a route to the ipPort listener that +// routes to dest if the incoming HTTP/1.x Host header name is +// httpHost. If it doesn't match, rule processing continues for any +// additional routes on ipPort. +// +// The ipPort is any valid net.Listen TCP address. +func (p *Proxy) AddHTTPHostRoute(ipPort, httpHost string, dest Target) { + p.AddHTTPHostMatchRoute(ipPort, equals(httpHost), dest) +} + +// AddHTTPHostMatchRoute appends a route to the ipPort listener that +// routes to dest if the incoming HTTP/1.x Host header name is +// accepted by matcher. If it doesn't match, rule processing continues +// for any additional routes on ipPort. +// +// The ipPort is any valid net.Listen TCP address. +func (p *Proxy) AddHTTPHostMatchRoute(ipPort string, match Matcher, dest Target) { + p.addRoute(ipPort, httpHostMatch{match, dest}) +} + +type httpHostMatch struct { + matcher Matcher + target Target +} + +func (m httpHostMatch) match(br *bufio.Reader) (Target, string) { + hh := httpHostHeader(br) + if m.matcher(context.TODO(), hh) { + return m.target, hh + } + return nil, "" +} + +// httpHostHeader returns the HTTP Host header from br without +// consuming any of its bytes. It returns "" if it can't find one. +func httpHostHeader(br *bufio.Reader) string { + const maxPeek = 4 << 10 + peekSize := 0 + for { + peekSize++ + if peekSize > maxPeek { + b, _ := br.Peek(br.Buffered()) + return httpHostHeaderFromBytes(b) + } + b, err := br.Peek(peekSize) + if n := br.Buffered(); n > peekSize { + b, _ = br.Peek(n) + peekSize = n + } + if len(b) > 0 { + if b[0] < 'A' || b[0] > 'Z' { + // Doesn't look like an HTTP verb + // (GET, POST, etc). + return "" + } + if bytes.Index(b, crlfcrlf) != -1 || bytes.Index(b, lflf) != -1 { + req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(b))) + if err != nil { + return "" + } + if len(req.Header["Host"]) > 1 { + // TODO(bradfitz): what does + // ReadRequest do if there are + // multiple Host headers? + return "" + } + return req.Host + } + } + if err != nil { + return httpHostHeaderFromBytes(b) + } + } +} + +var ( + lfHostColon = []byte("\nHost:") + lfhostColon = []byte("\nhost:") + crlf = []byte("\r\n") + lf = []byte("\n") + crlfcrlf = []byte("\r\n\r\n") + lflf = []byte("\n\n") +) + +func httpHostHeaderFromBytes(b []byte) string { + if i := bytes.Index(b, lfHostColon); i != -1 { + return string(bytes.TrimSpace(untilEOL(b[i+len(lfHostColon):]))) + } + if i := bytes.Index(b, lfhostColon); i != -1 { + return string(bytes.TrimSpace(untilEOL(b[i+len(lfhostColon):]))) + } + return "" +} + +// untilEOL returns v, truncated before the first '\n' byte, if any. +// The returned slice may include a '\r' at the end. +func untilEOL(v []byte) []byte { + if i := bytes.IndexByte(v, '\n'); i != -1 { + return v[:i] + } + return v +} diff --git a/vendor/github.com/google/tcpproxy/listener.go b/vendor/github.com/google/tcpproxy/listener.go new file mode 100644 index 0000000000..1ddc48ee21 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/listener.go @@ -0,0 +1,108 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpproxy + +import ( + "io" + "net" + "sync" +) + +// TargetListener implements both net.Listener and Target. +// Matched Targets become accepted connections. +type TargetListener struct { + Address string // Address is the string reported by TargetListener.Addr().String(). + + mu sync.Mutex + cond *sync.Cond + closed bool + nextConn net.Conn +} + +var ( + _ net.Listener = (*TargetListener)(nil) + _ Target = (*TargetListener)(nil) +) + +func (tl *TargetListener) lock() { + tl.mu.Lock() + if tl.cond == nil { + tl.cond = sync.NewCond(&tl.mu) + } +} + +type tcpAddr string + +func (a tcpAddr) Network() string { return "tcp" } +func (a tcpAddr) String() string { return string(a) } + +// Addr returns the listener's Address field as a net.Addr. +func (tl *TargetListener) Addr() net.Addr { return tcpAddr(tl.Address) } + +// Close stops listening for new connections. All new connections +// routed to this listener will be closed. Already accepted +// connections are not closed. +func (tl *TargetListener) Close() error { + tl.lock() + if tl.closed { + tl.mu.Unlock() + return nil + } + tl.closed = true + tl.mu.Unlock() + tl.cond.Broadcast() + return nil +} + +// HandleConn implements the Target interface. It blocks until tl is +// closed or another goroutine has called Accept and received c. +func (tl *TargetListener) HandleConn(c net.Conn) { + tl.lock() + defer tl.mu.Unlock() + for tl.nextConn != nil && !tl.closed { + tl.cond.Wait() + } + if tl.closed { + c.Close() + return + } + tl.nextConn = c + tl.cond.Broadcast() // Signal might be sufficient; verify. + for tl.nextConn == c && !tl.closed { + tl.cond.Wait() + } + if tl.closed { + c.Close() + return + } +} + +// Accept implements the Accept method in the net.Listener interface. +func (tl *TargetListener) Accept() (net.Conn, error) { + tl.lock() + for tl.nextConn == nil && !tl.closed { + tl.cond.Wait() + } + if tl.closed { + tl.mu.Unlock() + return nil, io.EOF + } + c := tl.nextConn + tl.nextConn = nil + tl.mu.Unlock() + tl.cond.Broadcast() // Signal might be sufficient; verify. + + return c, nil +} diff --git a/vendor/github.com/google/tcpproxy/sni.go b/vendor/github.com/google/tcpproxy/sni.go new file mode 100644 index 0000000000..53b53c24d7 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/sni.go @@ -0,0 +1,192 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tcpproxy + +import ( + "bufio" + "bytes" + "context" + "crypto/tls" + "io" + "net" + "strings" +) + +// AddSNIRoute appends a route to the ipPort listener that routes to +// dest if the incoming TLS SNI server name is sni. If it doesn't +// match, rule processing continues for any additional routes on +// ipPort. +// +// By default, the proxy will route all ACME tls-sni-01 challenges +// received on ipPort to all SNI dests. You can disable ACME routing +// with AddStopACMESearch. +// +// The ipPort is any valid net.Listen TCP address. +func (p *Proxy) AddSNIRoute(ipPort, sni string, dest Target) { + p.AddSNIMatchRoute(ipPort, equals(sni), dest) +} + +// AddSNIMatchRoute appends a route to the ipPort listener that routes +// to dest if the incoming TLS SNI server name is accepted by +// matcher. If it doesn't match, rule processing continues for any +// additional routes on ipPort. +// +// By default, the proxy will route all ACME tls-sni-01 challenges +// received on ipPort to all SNI dests. You can disable ACME routing +// with AddStopACMESearch. +// +// The ipPort is any valid net.Listen TCP address. +func (p *Proxy) AddSNIMatchRoute(ipPort string, matcher Matcher, dest Target) { + cfg := p.configFor(ipPort) + if !cfg.stopACME { + if len(cfg.acmeTargets) == 0 { + p.addRoute(ipPort, &acmeMatch{cfg}) + } + cfg.acmeTargets = append(cfg.acmeTargets, dest) + } + + p.addRoute(ipPort, sniMatch{matcher, dest}) +} + +// AddStopACMESearch prevents ACME probing of subsequent SNI routes. +// Any ACME challenges on ipPort for SNI routes previously added +// before this call will still be proxied to all possible SNI +// backends. +func (p *Proxy) AddStopACMESearch(ipPort string) { + p.configFor(ipPort).stopACME = true +} + +type sniMatch struct { + matcher Matcher + target Target +} + +func (m sniMatch) match(br *bufio.Reader) (Target, string) { + sni := clientHelloServerName(br) + if m.matcher(context.TODO(), sni) { + return m.target, sni + } + return nil, "" +} + +// acmeMatch matches "*.acme.invalid" ACME tls-sni-01 challenges and +// searches for a Target in cfg.acmeTargets that has the challenge +// response. +type acmeMatch struct { + cfg *config +} + +func (m *acmeMatch) match(br *bufio.Reader) (Target, string) { + sni := clientHelloServerName(br) + if !strings.HasSuffix(sni, ".acme.invalid") { + return nil, "" + } + + // TODO: cache. ACME issuers will hit multiple times in a short + // burst for each issuance event. A short TTL cache + singleflight + // should have an excellent hit rate. + // TODO: maybe an acme-specific timeout as well? + // TODO: plumb context upwards? + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ch := make(chan Target, len(m.cfg.acmeTargets)) + for _, target := range m.cfg.acmeTargets { + go tryACME(ctx, ch, target, sni) + } + for range m.cfg.acmeTargets { + if target := <-ch; target != nil { + return target, sni + } + } + + // No target was happy with the provided challenge. + return nil, "" +} + +func tryACME(ctx context.Context, ch chan<- Target, dest Target, sni string) { + var ret Target + defer func() { ch <- ret }() + + conn, targetConn := net.Pipe() + defer conn.Close() + go dest.HandleConn(targetConn) + + deadline, ok := ctx.Deadline() + if ok { + conn.SetDeadline(deadline) + } + + client := tls.Client(conn, &tls.Config{ + ServerName: sni, + InsecureSkipVerify: true, + }) + if err := client.Handshake(); err != nil { + // TODO: log? + return + } + certs := client.ConnectionState().PeerCertificates + if len(certs) == 0 { + // TODO: log? + return + } + // acme says the first cert offered by the server must match the + // challenge hostname. + if err := certs[0].VerifyHostname(sni); err != nil { + // TODO: log? + return + } + + // Target presented what looks like a valid challenge + // response, send it back to the matcher. + ret = dest +} + +// clientHelloServerName returns the SNI server name inside the TLS ClientHello, +// without consuming any bytes from br. +// On any error, the empty string is returned. +func clientHelloServerName(br *bufio.Reader) (sni string) { + const recordHeaderLen = 5 + hdr, err := br.Peek(recordHeaderLen) + if err != nil { + return "" + } + const recordTypeHandshake = 0x16 + if hdr[0] != recordTypeHandshake { + return "" // Not TLS. + } + recLen := int(hdr[3])<<8 | int(hdr[4]) // ignoring version in hdr[1:3] + helloBytes, err := br.Peek(recordHeaderLen + recLen) + if err != nil { + return "" + } + tls.Server(sniSniffConn{r: bytes.NewReader(helloBytes)}, &tls.Config{ + GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { + sni = hello.ServerName + return nil, nil + }, + }).Handshake() + return +} + +// sniSniffConn is a net.Conn that reads from r, fails on Writes, +// and crashes otherwise. +type sniSniffConn struct { + r io.Reader + net.Conn // nil; crash on any unexpected use +} + +func (c sniSniffConn) Read(p []byte) (int, error) { return c.r.Read(p) } +func (sniSniffConn) Write(p []byte) (int, error) { return 0, io.EOF } diff --git a/vendor/github.com/google/tcpproxy/tcpproxy.go b/vendor/github.com/google/tcpproxy/tcpproxy.go new file mode 100644 index 0000000000..9826d94226 --- /dev/null +++ b/vendor/github.com/google/tcpproxy/tcpproxy.go @@ -0,0 +1,474 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tcpproxy lets users build TCP proxies, optionally making +// routing decisions based on HTTP/1 Host headers and the SNI hostname +// in TLS connections. +// +// Typical usage: +// +// var p tcpproxy.Proxy +// p.AddHTTPHostRoute(":80", "foo.com", tcpproxy.To("10.0.0.1:8081")) +// p.AddHTTPHostRoute(":80", "bar.com", tcpproxy.To("10.0.0.2:8082")) +// p.AddRoute(":80", tcpproxy.To("10.0.0.1:8081")) // fallback +// p.AddSNIRoute(":443", "foo.com", tcpproxy.To("10.0.0.1:4431")) +// p.AddSNIRoute(":443", "bar.com", tcpproxy.To("10.0.0.2:4432")) +// p.AddRoute(":443", tcpproxy.To("10.0.0.1:4431")) // fallback +// log.Fatal(p.Run()) +// +// Calling Run (or Start) on a proxy also starts all the necessary +// listeners. +// +// For each accepted connection, the rules for that ipPort are +// matched, in order. If one matches (currently HTTP Host, SNI, or +// always), then the connection is handed to the target. +// +// The two predefined Target implementations are: +// +// 1) DialProxy, proxying to another address (use the To func to return a +// DialProxy value), +// +// 2) TargetListener, making the matched connection available via a +// net.Listener.Accept call. +// +// But Target is an interface, so you can also write your own. +// +// Note that tcpproxy does not do any TLS encryption or decryption. It +// only (via DialProxy) copies bytes around. The SNI hostname in the TLS +// header is unencrypted, for better or worse. +// +// This package makes no API stability promises. If you depend on it, +// vendor it. +package tcpproxy + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "net" + "time" +) + +// Proxy is a proxy. Its zero value is a valid proxy that does +// nothing. Call methods to add routes before calling Start or Run. +// +// The order that routes are added in matters; each is matched in the order +// registered. +type Proxy struct { + configs map[string]*config // ip:port => config + + lns []net.Listener + donec chan struct{} // closed before err + err error // any error from listening + + // ListenFunc optionally specifies an alternate listen + // function. If nil, net.Dial is used. + // The provided net is always "tcp". + ListenFunc func(net, laddr string) (net.Listener, error) +} + +// Matcher reports whether hostname matches the Matcher's criteria. +type Matcher func(ctx context.Context, hostname string) bool + +// equals is a trivial Matcher that implements string equality. +func equals(want string) Matcher { + return func(_ context.Context, got string) bool { + return want == got + } +} + +// config contains the proxying state for one listener. +type config struct { + routes []route + acmeTargets []Target // accumulates targets that should be probed for acme. + stopACME bool // if true, AddSNIRoute doesn't add targets to acmeTargets. +} + +// A route matches a connection to a target. +type route interface { + // match examines the initial bytes of a connection, looking for a + // match. If a match is found, match returns a non-nil Target to + // which the stream should be proxied. match returns nil if the + // connection doesn't match. + // + // match must not consume bytes from the given bufio.Reader, it + // can only Peek. + // + // If an sni or host header was parsed successfully, that will be + // returned as the second parameter. + match(*bufio.Reader) (Target, string) +} + +func (p *Proxy) netListen() func(net, laddr string) (net.Listener, error) { + if p.ListenFunc != nil { + return p.ListenFunc + } + return net.Listen +} + +func (p *Proxy) configFor(ipPort string) *config { + if p.configs == nil { + p.configs = make(map[string]*config) + } + if p.configs[ipPort] == nil { + p.configs[ipPort] = &config{} + } + return p.configs[ipPort] +} + +func (p *Proxy) addRoute(ipPort string, r route) { + cfg := p.configFor(ipPort) + cfg.routes = append(cfg.routes, r) +} + +// AddRoute appends an always-matching route to the ipPort listener, +// directing any connection to dest. +// +// This is generally used as either the only rule (for simple TCP +// proxies), or as the final fallback rule for an ipPort. +// +// The ipPort is any valid net.Listen TCP address. +func (p *Proxy) AddRoute(ipPort string, dest Target) { + p.addRoute(ipPort, fixedTarget{dest}) +} + +type fixedTarget struct { + t Target +} + +func (m fixedTarget) match(*bufio.Reader) (Target, string) { return m.t, "" } + +// Run is calls Start, and then Wait. +// +// It blocks until there's an error. The return value is always +// non-nil. +func (p *Proxy) Run() error { + if err := p.Start(); err != nil { + return err + } + return p.Wait() +} + +// Wait waits for the Proxy to finish running. Currently this can only +// happen if a Listener is closed, or Close is called on the proxy. +// +// It is only valid to call Wait after a successful call to Start. +func (p *Proxy) Wait() error { + <-p.donec + return p.err +} + +// Close closes all the proxy's self-opened listeners. +func (p *Proxy) Close() error { + for _, c := range p.lns { + c.Close() + } + return nil +} + +// Start creates a TCP listener for each unique ipPort from the +// previously created routes and starts the proxy. It returns any +// error from starting listeners. +// +// If it returns a non-nil error, any successfully opened listeners +// are closed. +func (p *Proxy) Start() error { + if p.donec != nil { + return errors.New("already started") + } + p.donec = make(chan struct{}) + errc := make(chan error, len(p.configs)) + p.lns = make([]net.Listener, 0, len(p.configs)) + for ipPort, config := range p.configs { + ln, err := p.netListen()("tcp", ipPort) + if err != nil { + p.Close() + return err + } + p.lns = append(p.lns, ln) + go p.serveListener(errc, ln, config.routes) + } + go p.awaitFirstError(errc) + return nil +} + +func (p *Proxy) awaitFirstError(errc <-chan error) { + p.err = <-errc + close(p.donec) +} + +func (p *Proxy) serveListener(ret chan<- error, ln net.Listener, routes []route) { + for { + c, err := ln.Accept() + if err != nil { + ret <- err + return + } + go p.serveConn(c, routes) + } +} + +// serveConn runs in its own goroutine and matches c against routes. +// It returns whether it matched purely for testing. +func (p *Proxy) serveConn(c net.Conn, routes []route) bool { + br := bufio.NewReader(c) + for _, route := range routes { + if target, hostName := route.match(br); target != nil { + if n := br.Buffered(); n > 0 { + peeked, _ := br.Peek(br.Buffered()) + c = &Conn{ + HostName: hostName, + Peeked: peeked, + Conn: c, + } + } + target.HandleConn(c) + return true + } + } + // TODO: hook for this? + log.Printf("tcpproxy: no routes matched conn %v/%v; closing", c.RemoteAddr().String(), c.LocalAddr().String()) + c.Close() + return false +} + +// Conn is an incoming connection that has had some bytes read from it +// to determine how to route the connection. The Read method stitches +// the peeked bytes and unread bytes back together. +type Conn struct { + // HostName is the hostname field that was sent to the request router. + // In the case of TLS, this is the SNI header, in the case of HTTPHost + // route, it will be the host header. In the case of a fixed + // route, i.e. those created with AddRoute(), this will always be + // empty. This can be useful in the case where further routing decisions + // need to be made in the Target impementation. + HostName string + + // Peeked are the bytes that have been read from Conn for the + // purposes of route matching, but have not yet been consumed + // by Read calls. It set to nil by Read when fully consumed. + Peeked []byte + + // Conn is the underlying connection. + // It can be type asserted against *net.TCPConn or other types + // as needed. It should not be read from directly unless + // Peeked is nil. + net.Conn +} + +func (c *Conn) Read(p []byte) (n int, err error) { + if len(c.Peeked) > 0 { + n = copy(p, c.Peeked) + c.Peeked = c.Peeked[n:] + if len(c.Peeked) == 0 { + c.Peeked = nil + } + return n, nil + } + return c.Conn.Read(p) +} + +// Target is what an incoming matched connection is sent to. +type Target interface { + // HandleConn is called when an incoming connection is + // matched. After the call to HandleConn, the tcpproxy + // package never touches the conn again. Implementations are + // responsible for closing the connection when needed. + // + // The concrete type of conn will be of type *Conn if any + // bytes have been consumed for the purposes of route + // matching. + HandleConn(net.Conn) +} + +// To is shorthand way of writing &tlsproxy.DialProxy{Addr: addr}. +func To(addr string) *DialProxy { + return &DialProxy{Addr: addr} +} + +// DialProxy implements Target by dialing a new connection to Addr +// and then proxying data back and forth. +// +// The To func is a shorthand way of creating a DialProxy. +type DialProxy struct { + // Addr is the TCP address to proxy to. + Addr string + + // KeepAlivePeriod sets the period between TCP keep alives. + // If zero, a default is used. To disable, use a negative number. + // The keep-alive is used for both the client connection and + KeepAlivePeriod time.Duration + + // DialTimeout optionally specifies a dial timeout. + // If zero, a default is used. + // If negative, the timeout is disabled. + DialTimeout time.Duration + + // DialContext optionally specifies an alternate dial function + // for TCP targets. If nil, the standard + // net.Dialer.DialContext method is used. + DialContext func(ctx context.Context, network, address string) (net.Conn, error) + + // OnDialError optionally specifies an alternate way to handle errors dialing Addr. + // If nil, the error is logged and src is closed. + // If non-nil, src is not closed automatically. + OnDialError func(src net.Conn, dstDialErr error) + + // ProxyProtocolVersion optionally specifies the version of + // HAProxy's PROXY protocol to use. The PROXY protocol provides + // connection metadata to the DialProxy target, via a header + // inserted ahead of the client's traffic. The DialProxy target + // must explicitly support and expect the PROXY header; there is + // no graceful downgrade. + // If zero, no PROXY header is sent. Currently, version 1 is supported. + ProxyProtocolVersion int +} + +// UnderlyingConn returns c.Conn if c of type *Conn, +// otherwise it returns c. +func UnderlyingConn(c net.Conn) net.Conn { + if wrap, ok := c.(*Conn); ok { + return wrap.Conn + } + return c +} + +func goCloseConn(c net.Conn) { go c.Close() } + +// HandleConn implements the Target interface. +func (dp *DialProxy) HandleConn(src net.Conn) { + ctx := context.Background() + var cancel context.CancelFunc + if dp.DialTimeout >= 0 { + ctx, cancel = context.WithTimeout(ctx, dp.dialTimeout()) + } + dst, err := dp.dialContext()(ctx, "tcp", dp.Addr) + if cancel != nil { + cancel() + } + if err != nil { + dp.onDialError()(src, err) + return + } + defer goCloseConn(dst) + + if err = dp.sendProxyHeader(dst, src); err != nil { + dp.onDialError()(src, err) + return + } + defer goCloseConn(src) + + if ka := dp.keepAlivePeriod(); ka > 0 { + if c, ok := UnderlyingConn(src).(*net.TCPConn); ok { + c.SetKeepAlive(true) + c.SetKeepAlivePeriod(ka) + } + if c, ok := dst.(*net.TCPConn); ok { + c.SetKeepAlive(true) + c.SetKeepAlivePeriod(ka) + } + } + + errc := make(chan error, 1) + go proxyCopy(errc, src, dst) + go proxyCopy(errc, dst, src) + <-errc +} + +func (dp *DialProxy) sendProxyHeader(w io.Writer, src net.Conn) error { + switch dp.ProxyProtocolVersion { + case 0: + return nil + case 1: + var srcAddr, dstAddr *net.TCPAddr + if a, ok := src.RemoteAddr().(*net.TCPAddr); ok { + srcAddr = a + } + if a, ok := src.LocalAddr().(*net.TCPAddr); ok { + dstAddr = a + } + + if srcAddr == nil || dstAddr == nil { + _, err := io.WriteString(w, "PROXY UNKNOWN\r\n") + return err + } + + family := "TCP4" + if srcAddr.IP.To4() == nil { + family = "TCP6" + } + _, err := fmt.Fprintf(w, "PROXY %s %s %d %s %d\r\n", family, srcAddr.IP, srcAddr.Port, dstAddr.IP, dstAddr.Port) + return err + default: + return fmt.Errorf("PROXY protocol version %d not supported", dp.ProxyProtocolVersion) + } +} + +// proxyCopy is the function that copies bytes around. +// It's a named function instead of a func literal so users get +// named goroutines in debug goroutine stack dumps. +func proxyCopy(errc chan<- error, dst, src net.Conn) { + // Before we unwrap src and/or dst, copy any buffered data. + if wc, ok := src.(*Conn); ok && len(wc.Peeked) > 0 { + if _, err := dst.Write(wc.Peeked); err != nil { + errc <- err + return + } + wc.Peeked = nil + } + + // Unwrap the src and dst from *Conn to *net.TCPConn so Go + // 1.11's splice optimization kicks in. + src = UnderlyingConn(src) + dst = UnderlyingConn(dst) + + _, err := io.Copy(dst, src) + errc <- err +} + +func (dp *DialProxy) keepAlivePeriod() time.Duration { + if dp.KeepAlivePeriod != 0 { + return dp.KeepAlivePeriod + } + return time.Minute +} + +func (dp *DialProxy) dialTimeout() time.Duration { + if dp.DialTimeout > 0 { + return dp.DialTimeout + } + return 10 * time.Second +} + +var defaultDialer = new(net.Dialer) + +func (dp *DialProxy) dialContext() func(ctx context.Context, network, address string) (net.Conn, error) { + if dp.DialContext != nil { + return dp.DialContext + } + return defaultDialer.DialContext +} + +func (dp *DialProxy) onDialError() func(src net.Conn, dstDialErr error) { + if dp.OnDialError != nil { + return dp.OnDialError + } + return func(src net.Conn, dstDialErr error) { + log.Printf("tcpproxy: for incoming conn %v, error dialing %q: %v", src.RemoteAddr().String(), dp.Addr, dstDialErr) + src.Close() + } +} diff --git a/vendor/github.com/rancher/dynamiclistener/server.go b/vendor/github.com/rancher/dynamiclistener/server.go index 4a7877ff51..c839da5bd9 100644 --- a/vendor/github.com/rancher/dynamiclistener/server.go +++ b/vendor/github.com/rancher/dynamiclistener/server.go @@ -3,6 +3,8 @@ package dynamiclistener import ( "bytes" "context" + "crypto" + "crypto/ecdsa" "crypto/md5" "crypto/rsa" "crypto/tls" @@ -46,7 +48,7 @@ type server struct { // dynamic config change on refresh activeCert *tls.Certificate activeCA *x509.Certificate - activeCAKey *rsa.PrivateKey + activeCAKey crypto.Signer activeCAKeyString string domains map[string]bool } @@ -91,6 +93,40 @@ func (s *server) CACert() (string, error) { return status.CACert, nil } +func marshalPrivateKey(privateKey crypto.Signer) (string, []byte, error) { + var ( + keyType string + bytes []byte + err error + ) + if key, ok := privateKey.(*ecdsa.PrivateKey); ok { + keyType = cert.ECPrivateKeyBlockType + bytes, err = x509.MarshalECPrivateKey(key) + } else if key, ok := privateKey.(*rsa.PrivateKey); ok { + keyType = cert.RSAPrivateKeyBlockType + bytes = x509.MarshalPKCS1PrivateKey(key) + } else { + keyType = cert.PrivateKeyBlockType + bytes, err = x509.MarshalPKCS8PrivateKey(privateKey) + } + if err != nil { + logrus.Errorf("Unable to marshal private key: %v", err) + } + return keyType, bytes, err +} + +func newPrivateKey() (crypto.Signer, error) { + caKeyBytes, err := cert.MakeEllipticPrivateKeyPEM() + if err != nil { + return nil, err + } + caKeyIFace, err := cert.ParsePrivateKeyPEM(caKeyBytes) + if err != nil { + return nil, err + } + return caKeyIFace.(crypto.Signer), nil +} + func (s *server) save() { if s.activeCert != nil { return @@ -114,7 +150,10 @@ func (s *server) save() { } for key, cert := range s.certs { - certStr := certToString(cert) + certStr, err := certToString(cert) + if err != nil { + continue + } if cfg.GeneratedCerts[key] != certStr { cfg.GeneratedCerts[key] = certStr changed = true @@ -139,9 +178,14 @@ func (s *server) save() { } caKeyBuffer := bytes.Buffer{} + keyType, keyBytes, err := marshalPrivateKey(s.activeCAKey) + if err != nil { + return + } + if err := pem.Encode(&caKeyBuffer, &pem.Block{ - Type: cert.RSAPrivateKeyBlockType, - Bytes: x509.MarshalPKCS1PrivateKey(s.activeCAKey), + Type: keyType, + Bytes: keyBytes, }); err != nil { return } @@ -198,8 +242,8 @@ func (s *server) userConfigure() error { return nil } -func genCA() (*x509.Certificate, *rsa.PrivateKey, error) { - caKey, err := cert.NewPrivateKey() +func genCA() (*x509.Certificate, crypto.Signer, error) { + caKey, err := newPrivateKey() if err != nil { return nil, nil, err } @@ -225,7 +269,7 @@ func (s *server) Update(status *ListenerStatus) error { s.Unlock() return err } - s.activeCAKey = cert.PrivateKey.(*rsa.PrivateKey) + s.activeCAKey = cert.PrivateKey.(crypto.Signer) s.activeCAKeyString = status.CAKey x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) @@ -380,12 +424,25 @@ func (s *server) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, e changed = true if s.activeCA == nil { - ca, key, err := genCA() - if err != nil { - return nil, err + if s.userConfig.CACerts != "" && s.userConfig.CAKey != "" { + ca, err := cert.ParseCertsPEM([]byte(s.userConfig.CACerts)) + if err != nil { + return nil, err + } + key, err := cert.ParsePrivateKeyPEM([]byte(s.userConfig.CAKey)) + if err != nil { + return nil, err + } + s.activeCA = ca[0] + s.activeCAKey = key.(crypto.Signer) + } else { + ca, key, err := genCA() + if err != nil { + return nil, err + } + s.activeCA = ca + s.activeCAKey = key } - s.activeCA = ca - s.activeCAKey = key } cfg := cert.Config{ @@ -398,7 +455,7 @@ func (s *server) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, e }, } - key, err := cert.NewPrivateKey() + key, err := newPrivateKey() if err != nil { return nil, err } @@ -439,6 +496,7 @@ func (s *server) cacheIPHandler(handler http.Handler) http.Handler { func (s *server) serveHTTPS() error { conf := &tls.Config{ + ClientAuth: tls.RequestClientCert, GetCertificate: s.getCertificate, PreferServerCipherSuites: true, } @@ -602,32 +660,36 @@ func stringToCert(certString string) *tls.Certificate { return nil } - cert, key := parts[0], parts[1] - keyBytes, err := base64.StdEncoding.DecodeString(key) + certPart, keyPart := parts[0], parts[1] + keyBytes, err := base64.StdEncoding.DecodeString(keyPart) if err != nil { return nil } - rsaKey, err := x509.ParsePKCS1PrivateKey(keyBytes) + key, err := cert.ParsePrivateKeyPEM(keyBytes) if err != nil { return nil } - certBytes, err := base64.StdEncoding.DecodeString(cert) + certBytes, err := base64.StdEncoding.DecodeString(certPart) if err != nil { return nil } return &tls.Certificate{ Certificate: [][]byte{certBytes}, - PrivateKey: rsaKey, + PrivateKey: key, } } -func certToString(cert *tls.Certificate) string { +func certToString(cert *tls.Certificate) (string, error) { + _, keyBytes, err := marshalPrivateKey(cert.PrivateKey.(crypto.Signer)) + if err != nil { + return "", err + } certString := base64.StdEncoding.EncodeToString(cert.Certificate[0]) - keyString := base64.StdEncoding.EncodeToString(x509.MarshalPKCS1PrivateKey(cert.PrivateKey.(*rsa.PrivateKey))) - return certString + "#" + keyString + keyString := base64.StdEncoding.EncodeToString(keyBytes) + return certString + "#" + keyString, nil } type tcpKeepAliveListener struct { diff --git a/vendor/github.com/rancher/dynamiclistener/types.go b/vendor/github.com/rancher/dynamiclistener/types.go index 87527bdc68..7fabf1c950 100644 --- a/vendor/github.com/rancher/dynamiclistener/types.go +++ b/vendor/github.com/rancher/dynamiclistener/types.go @@ -29,6 +29,7 @@ type UserConfig struct { Mode string NoCACerts bool CACerts string + CAKey string Cert string Key string BindAddress string diff --git a/vendor/k8s.io/client-go/pkg/version/base.go b/vendor/k8s.io/client-go/pkg/version/base.go index d1062c8795..0243ffea96 100644 --- a/vendor/k8s.io/client-go/pkg/version/base.go +++ b/vendor/k8s.io/client-go/pkg/version/base.go @@ -3,8 +3,8 @@ package version var ( gitMajor = "1" gitMinor = "14" - gitVersion = "v1.14.3-k3s.1" - gitCommit = "8343999292c55c807be4406fcaa9f047e8751ffd" + gitVersion = "v1.14.3-k3s.2" + gitCommit = "6174f1fed28fd19300038f6578bf48e3920fa7ba" gitTreeState = "clean" - buildDate = "2019-06-12T04:56+00:00Z" + buildDate = "2019-06-21T08:17+00:00Z" ) diff --git a/vendor/k8s.io/kubernetes/pkg/master/controller.go b/vendor/k8s.io/kubernetes/pkg/master/controller.go index 3e90b84f69..517f8db3df 100644 --- a/vendor/k8s.io/kubernetes/pkg/master/controller.go +++ b/vendor/k8s.io/kubernetes/pkg/master/controller.go @@ -142,6 +142,10 @@ func (c *Controller) Start() { return } + // Service definition is reconciled during first run to correct port and type per expectations. + if err := c.UpdateKubernetesService(true); err != nil { + klog.Errorf("Unable to perform initial Kubernetes service initialization: %v", err) + } // Reconcile during first run removing itself until server is ready. endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https", c.ExtraEndpointPorts) if err := c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts); err != nil { diff --git a/vendor/k8s.io/kubernetes/pkg/version/base.go b/vendor/k8s.io/kubernetes/pkg/version/base.go index d1062c8795..0243ffea96 100644 --- a/vendor/k8s.io/kubernetes/pkg/version/base.go +++ b/vendor/k8s.io/kubernetes/pkg/version/base.go @@ -3,8 +3,8 @@ package version var ( gitMajor = "1" gitMinor = "14" - gitVersion = "v1.14.3-k3s.1" - gitCommit = "8343999292c55c807be4406fcaa9f047e8751ffd" + gitVersion = "v1.14.3-k3s.2" + gitCommit = "6174f1fed28fd19300038f6578bf48e3920fa7ba" gitTreeState = "clean" - buildDate = "2019-06-12T04:56+00:00Z" + buildDate = "2019-06-21T08:17+00:00Z" ) From 1e035820bf2dc9ffa16a872aa808a3e559a40e8b Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Wed, 29 May 2019 11:54:07 -0700 Subject: [PATCH 3/8] Generated data --- pkg/deploy/zz_generated_bindata.go | 31 ++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/pkg/deploy/zz_generated_bindata.go b/pkg/deploy/zz_generated_bindata.go index b5a77ddaec..0e6046d180 100644 --- a/pkg/deploy/zz_generated_bindata.go +++ b/pkg/deploy/zz_generated_bindata.go @@ -1,6 +1,7 @@ // Code generated by go-bindata. // sources: // manifests/coredns.yaml +// manifests/rolebindings.yaml // manifests/traefik.yaml // DO NOT EDIT! @@ -89,6 +90,26 @@ func corednsYaml() (*asset, error) { return a, nil } +var _rolebindingsYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\xcf\xbd\x0a\xc2\x40\x10\x04\xe0\xfe\x9e\xe2\x5e\xe0\x22\x76\x72\xa5\x16\xf6\x01\xed\x37\xb9\x55\xd7\xdc\x1f\xbb\x7b\x01\x7d\x7a\x09\x48\x1a\x51\xb0\x1c\x18\xe6\x63\xa0\xd2\x19\x59\xa8\x64\x6f\x79\x80\xb1\x83\xa6\xb7\xc2\xf4\x04\xa5\x92\xbb\x69\x27\x1d\x95\xcd\xbc\x35\x13\xe5\xe0\xed\x21\x36\x51\xe4\xbe\x44\xdc\x53\x0e\x94\xaf\x26\xa1\x42\x00\x05\x6f\xac\xcd\x90\xd0\xdb\xa9\x0d\xe8\xa0\x92\x20\xcf\xc8\x6e\x89\x11\xd5\x41\x48\x94\x0d\x97\x88\x3d\x5e\x96\x36\x54\x3a\x72\x69\xf5\x87\x6c\xac\xfd\x80\x57\x47\x1e\xa2\x98\xfc\xba\x5f\xe9\x6d\x48\x1b\xee\x38\xaa\x78\xe3\xfe\x42\x4e\x82\xfc\xe5\x85\x79\x05\x00\x00\xff\xff\x54\xf2\x55\xe2\x29\x01\x00\x00") + +func rolebindingsYamlBytes() ([]byte, error) { + return bindataRead( + _rolebindingsYaml, + "rolebindings.yaml", + ) +} + +func rolebindingsYaml() (*asset, error) { + bytes, err := rolebindingsYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "rolebindings.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _traefikYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x6c\xcf\x4f\x4b\xc3\x40\x10\x05\xf0\x7b\x3e\xc5\x50\xe8\xb1\xbb\x16\xc4\xc3\xde\xfc\x13\x50\x04\x29\x56\xbd\xca\x64\xf3\xda\x0c\xdd\x6c\xc2\xce\xa4\xa0\xe2\x77\x97\x94\x1c\x3d\xce\xcc\xe3\xc7\x3c\x1e\xe5\x03\x45\x65\xc8\x81\x3a\xa4\xde\x45\x36\x4b\x70\x32\xf8\xf3\xb6\x3a\x49\x6e\x03\x3d\x22\xf5\xf7\x1d\x17\xab\x7a\x18\xb7\x6c\x1c\x2a\xa2\xcc\x3d\x02\x59\x61\x1c\xe4\xb4\xcc\x3a\x72\x44\xa0\xd3\xd4\x60\xa3\x5f\x6a\xe8\x2b\x1d\x11\xe7\x78\x9c\x81\x40\x9d\xd9\xa8\xc1\xfb\xf5\xcf\xf3\xfb\x5d\xfd\xfa\x52\xbf\xd5\xfb\xcf\xdb\xdd\xd3\xef\xda\xab\xb1\x49\xf4\x97\xa0\xfa\x05\xde\x6c\xdd\xcd\xb5\xbb\x72\x76\xfc\xae\x88\x14\x36\x5b\x44\xa5\xe1\xe8\x90\xb9\x49\x68\x03\xad\xac\x4c\x58\x5d\x0e\xaa\xe9\xdf\xfd\xfc\x52\xc9\x30\xa8\x93\x7c\x2c\x50\xad\x73\x3b\x0e\x92\xcd\x4d\x8a\x07\x1c\x78\x4a\xb6\x9b\x9a\x24\xda\xa1\xdd\xa3\x9c\x65\x6e\xb2\x08\x7f\x01\x00\x00\xff\xff\x90\xbb\x64\x2c\x26\x01\x00\x00") func traefikYamlBytes() ([]byte, error) { @@ -161,8 +182,9 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "coredns.yaml": corednsYaml, - "traefik.yaml": traefikYaml, + "coredns.yaml": corednsYaml, + "rolebindings.yaml": rolebindingsYaml, + "traefik.yaml": traefikYaml, } // AssetDir returns the file names below a certain @@ -206,8 +228,9 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "coredns.yaml": &bintree{corednsYaml, map[string]*bintree{}}, - "traefik.yaml": &bintree{traefikYaml, map[string]*bintree{}}, + "coredns.yaml": &bintree{corednsYaml, map[string]*bintree{}}, + "rolebindings.yaml": &bintree{rolebindingsYaml, map[string]*bintree{}}, + "traefik.yaml": &bintree{traefikYaml, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory From 93f6690f261e72c7321ce806dfdb20977e882c8b Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Fri, 21 Jun 2019 10:45:31 -0700 Subject: [PATCH 4/8] Graceful upgrade token to server CA --- pkg/daemons/control/server.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index fa9299d71a..027173d78d 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -524,8 +524,25 @@ func genClientCerts(config *config.Control, runtime *config.ControlRuntime) erro return nil } +func createServerSigningCertKey(config *config.Control, runtime *config.ControlRuntime) (bool, error) { + TokenCA := path.Join(config.DataDir, "tls", "token-ca.crt") + TokenCAKey := path.Join(config.DataDir, "tls", "token-ca.key") + + if exists(TokenCA, TokenCAKey) && !exists(runtime.ServerCA) && !exists(runtime.ServerCAKey) { + logrus.Infof("Upgrading token-ca files to server-ca") + if err := os.Link(TokenCA, runtime.ServerCA); err != nil { + return false, err + } + if err := os.Link(TokenCAKey, runtime.ServerCAKey); err != nil { + return false, err + } + return true, nil + } + return createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey) +} + func genServerCerts(config *config.Control, runtime *config.ControlRuntime) error { - regen, err := createSigningCertKey("k3s-server", runtime.ServerCA, runtime.ServerCAKey) + regen, err := createServerSigningCertKey(config, runtime) if err != nil { return err } From c9b62c9a90ac2fb47699ff8d02d11e9270587fab Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Fri, 21 Jun 2019 11:54:23 -0700 Subject: [PATCH 5/8] Remove CA Certs/Key from listenerconfig storage --- pkg/tls/storage.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pkg/tls/storage.go b/pkg/tls/storage.go index 4a1407ab55..eaacbc6fdf 100644 --- a/pkg/tls/storage.go +++ b/pkg/tls/storage.go @@ -19,6 +19,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont storage := &listenerConfigStorage{ client: listenerConfigs, cache: listenerConfigs.Cache(), + config: config, } server, err := dynamiclistener.NewServer(storage, config) @@ -30,7 +31,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont if obj == nil { return nil, nil } - return obj, server.Update(fromStorage(obj)) + return obj, server.Update(storage.fromStorage(obj)) }) return server, err @@ -39,6 +40,7 @@ func NewServer(ctx context.Context, listenerConfigs k3sclient.ListenerConfigCont type listenerConfigStorage struct { cache k3sclient.ListenerConfigCache client k3sclient.ListenerConfigClient + config dynamiclistener.UserConfig } func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dynamiclistener.ListenerStatus, error) { @@ -53,7 +55,7 @@ func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dy }) ls, err := l.client.Create(ls) - return fromStorage(ls), err + return l.fromStorage(ls), err } else if err != nil { return nil, err } @@ -63,8 +65,13 @@ func (l *listenerConfigStorage) Set(config *dynamiclistener.ListenerStatus) (*dy obj.Status = *config obj.Status.Revision = "" + if l.config.CACerts != "" && l.config.CAKey != "" { + obj.Status.CACert = "" + obj.Status.CAKey = "" + } + obj, err = l.client.Update(obj) - return fromStorage(obj), err + return l.fromStorage(obj), err } func (l *listenerConfigStorage) Get() (*dynamiclistener.ListenerStatus, error) { @@ -75,15 +82,21 @@ func (l *listenerConfigStorage) Get() (*dynamiclistener.ListenerStatus, error) { if errors.IsNotFound(err) { return &dynamiclistener.ListenerStatus{}, nil } - return fromStorage(obj), err + return l.fromStorage(obj), err } -func fromStorage(obj *v1.ListenerConfig) *dynamiclistener.ListenerStatus { +func (l *listenerConfigStorage) fromStorage(obj *v1.ListenerConfig) *dynamiclistener.ListenerStatus { if obj == nil { return nil } copy := obj.DeepCopy() copy.Status.Revision = obj.ResourceVersion + + if l.config.CACerts != "" && l.config.CAKey != "" { + copy.Status.CACert = l.config.CACerts + copy.Status.CAKey = l.config.CAKey + } + return ©.Status } From 29865fd9c986fc0a290b558efec6cdf482fba11c Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Mon, 24 Jun 2019 10:20:43 -0700 Subject: [PATCH 6/8] Remove agent proxy --- pkg/agent/config/config.go | 12 ++++-------- pkg/agent/proxy/proxy.go | 18 ------------------ pkg/agent/run.go | 5 ----- 3 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 pkg/agent/proxy/proxy.go diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index 1559785b90..443b94092c 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -187,11 +187,9 @@ func localAddress(controlConfig *config.Control) string { return fmt.Sprintf("127.0.0.1:%d", controlConfig.ProxyPort) } -func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, controlConfig *config.Control, tlsCert *tls.Certificate) (string, error) { +func writeKubeConfig(envInfo *cmds.Agent, info clientaccess.Info, tlsCert *tls.Certificate) (string, error) { os.MkdirAll(envInfo.DataDir, 0700) kubeConfigPath := filepath.Join(envInfo.DataDir, "kubeconfig.yaml") - - info.URL = "https://" + localAddress(controlConfig) info.CACerts = pem.EncodeToMemory(&pem.Block{ Type: cert.CertificateBlockType, Bytes: tlsCert.Certificate[1], @@ -281,8 +279,6 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { } } - proxyURL := "https://" + localAddress(controlConfig) - clientCAFile := filepath.Join(envInfo.DataDir, "client-ca.crt") if err := getHostFile(clientCAFile, info); err != nil { return nil, err @@ -301,7 +297,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { return nil, err } - kubeconfigNode, err := writeKubeConfig(envInfo, *info, controlConfig, servingCert) + kubeconfigNode, err := writeKubeConfig(envInfo, *info, servingCert) if err != nil { return nil, err } @@ -317,7 +313,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { } kubeconfigKubelet := filepath.Join(envInfo.DataDir, "kubelet.kubeconfig") - if err := control.KubeConfig(kubeconfigKubelet, proxyURL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { + if err := control.KubeConfig(kubeconfigKubelet, info.URL, serverCAFile, clientKubeletCert, clientKubeletKey); err != nil { return nil, err } @@ -332,7 +328,7 @@ func get(envInfo *cmds.Agent) (*config.Node, error) { } kubeconfigKubeproxy := filepath.Join(envInfo.DataDir, "kubeproxy.kubeconfig") - if err := control.KubeConfig(kubeconfigKubeproxy, proxyURL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { + if err := control.KubeConfig(kubeconfigKubeproxy, info.URL, serverCAFile, clientKubeProxyCert, clientKubeProxyKey); err != nil { return nil, err } diff --git a/pkg/agent/proxy/proxy.go b/pkg/agent/proxy/proxy.go deleted file mode 100644 index e93a7be39c..0000000000 --- a/pkg/agent/proxy/proxy.go +++ /dev/null @@ -1,18 +0,0 @@ -package proxy - -import ( - "github.com/google/tcpproxy" - "github.com/rancher/k3s/pkg/daemons/config" - "github.com/sirupsen/logrus" -) - -func Run(config *config.Node) error { - logrus.Infof("Starting proxy %s -> %s", config.LocalAddress, config.ServerAddress) - var proxy tcpproxy.Proxy - proxy.AddRoute(config.LocalAddress, tcpproxy.To(config.ServerAddress)) - go func() { - err := proxy.Run() - logrus.Fatalf("TLS proxy stopped: %v", err) - }() - return nil -} diff --git a/pkg/agent/run.go b/pkg/agent/run.go index cc18e3203d..640e19c26a 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -12,7 +12,6 @@ import ( "github.com/rancher/k3s/pkg/agent/config" "github.com/rancher/k3s/pkg/agent/containerd" "github.com/rancher/k3s/pkg/agent/flannel" - "github.com/rancher/k3s/pkg/agent/proxy" "github.com/rancher/k3s/pkg/agent/syssetup" "github.com/rancher/k3s/pkg/agent/tunnel" "github.com/rancher/k3s/pkg/cli/cmds" @@ -52,10 +51,6 @@ func run(ctx context.Context, cfg cmds.Agent) error { return err } - if err := proxy.Run(nodeConfig); err != nil { - return err - } - if err := agent.Agent(&nodeConfig.AgentConfig); err != nil { return err } From 7090a7d551c9210708d88ba4f1c7f87efd2fe1d3 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Tue, 25 Jun 2019 10:42:15 -0700 Subject: [PATCH 7/8] Move node password to separate file --- pkg/daemons/config/types.go | 1 + pkg/daemons/control/server.go | 1 + pkg/server/router.go | 53 ++++++++++++++++++----------------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index 489684562e..afcd6d7fdf 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -104,6 +104,7 @@ type ControlRuntime struct { ServerCAKey string ServiceKey string PasswdFile string + NodePasswdFile string KubeConfigAdmin string KubeConfigController string diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index 027173d78d..88e1ed269d 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -263,6 +263,7 @@ func prepare(config *config.Control, runtime *config.ControlRuntime) error { runtime.ServiceKey = path.Join(config.DataDir, "tls", "service.key") runtime.PasswdFile = path.Join(config.DataDir, "cred", "passwd") + runtime.NodePasswdFile = path.Join(config.DataDir, "cred", "node-passwd") runtime.KubeConfigAdmin = path.Join(config.DataDir, "cred", "admin.kubeconfig") runtime.KubeConfigController = path.Join(config.DataDir, "cred", "controller.kubeconfig") diff --git a/pkg/server/router.go b/pkg/server/router.go index 06adb05b89..a0cc4bde57 100644 --- a/pkg/server/router.go +++ b/pkg/server/router.go @@ -83,7 +83,7 @@ func getNodeInfo(req *http.Request) (string, string, error) { return "", "", errors.New("node password not set") } - return nodeNames[0], nodePasswords[0], nil + return strings.ToLower(nodeNames[0]), nodePasswords[0], nil } func getCACertAndKeys(caCertFile, caKeyFile, signingKeyFile string) ([]*x509.Certificate, crypto.Signer, crypto.Signer, error) { @@ -132,7 +132,7 @@ func servingKubeletCert(server *config.Control) http.Handler { sendError(err, resp) } - if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { + if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil { sendError(err, resp, http.StatusForbidden) return } @@ -172,7 +172,7 @@ func clientKubeletCert(server *config.Control) http.Handler { sendError(err, resp) } - if err := ensureNodePassword(server.Runtime.PasswdFile, nodeName, nodePassword); err != nil { + if err := ensureNodePassword(server.Runtime.NodePasswdFile, nodeName, nodePassword); err != nil { sendError(err, resp, http.StatusForbidden) return } @@ -265,36 +265,37 @@ func sendError(err error, resp http.ResponseWriter, status ...int) { } func ensureNodePassword(passwdFile, nodeName, passwd string) error { - f, err := os.Open(passwdFile) - if err != nil { - return err - } - defer f.Close() - user := strings.ToLower("node:" + nodeName) - records := [][]string{} - reader := csv.NewReader(f) - for { - record, err := reader.Read() - if err == io.EOF { - break - } + + if _, err := os.Stat(passwdFile); !os.IsNotExist(err) { + f, err := os.Open(passwdFile) if err != nil { return err } - if len(record) < 3 { - return fmt.Errorf("password file '%s' must have at least 3 columns (password, user name, user uid), found %d", passwdFile, len(record)) - } - if record[1] == user { - if record[0] == passwd { - return nil + defer f.Close() + reader := csv.NewReader(f) + for { + record, err := reader.Read() + if err == io.EOF { + break } - return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) + if err != nil { + return err + } + if len(record) < 2 { + return fmt.Errorf("password file '%s' must have at least 2 columns (password, nodeName), found %d", passwdFile, len(record)) + } + if record[1] == nodeName { + if record[0] == passwd { + return nil + } + return fmt.Errorf("Node password validation failed for '%s', using passwd file '%s'", nodeName, passwdFile) + } + records = append(records, record) } - records = append(records, record) + f.Close() } - records = append(records, []string{passwd, user, user, "system:node:" + nodeName}) - f.Close() + records = append(records, []string{passwd, nodeName}) return control.WritePasswords(passwdFile, records) } From d7590bc13f7e00abf8eb3fd47017f11735f08b96 Mon Sep 17 00:00:00 2001 From: Erik Wilson Date: Tue, 25 Jun 2019 13:54:59 -0700 Subject: [PATCH 8/8] Update Vendor --- trash.lock | 2 - vendor.conf | 1 - vendor/github.com/google/tcpproxy/.gitignore | 2 - vendor/github.com/google/tcpproxy/.travis.yml | 49 -- .../google/tcpproxy/CONTRIBUTING.md | 8 - vendor/github.com/google/tcpproxy/LICENSE | 202 -------- vendor/github.com/google/tcpproxy/README.md | 5 - vendor/github.com/google/tcpproxy/http.go | 125 ----- vendor/github.com/google/tcpproxy/listener.go | 108 ---- vendor/github.com/google/tcpproxy/sni.go | 192 ------- vendor/github.com/google/tcpproxy/tcpproxy.go | 474 ------------------ 11 files changed, 1168 deletions(-) delete mode 100644 vendor/github.com/google/tcpproxy/.gitignore delete mode 100644 vendor/github.com/google/tcpproxy/.travis.yml delete mode 100644 vendor/github.com/google/tcpproxy/CONTRIBUTING.md delete mode 100644 vendor/github.com/google/tcpproxy/LICENSE delete mode 100644 vendor/github.com/google/tcpproxy/README.md delete mode 100644 vendor/github.com/google/tcpproxy/http.go delete mode 100644 vendor/github.com/google/tcpproxy/listener.go delete mode 100644 vendor/github.com/google/tcpproxy/sni.go delete mode 100644 vendor/github.com/google/tcpproxy/tcpproxy.go diff --git a/trash.lock b/trash.lock index 79543f1566..5316383d5d 100755 --- a/trash.lock +++ b/trash.lock @@ -125,8 +125,6 @@ import: version: v1.0.21 - package: github.com/google/gofuzz version: 44d81051d367757e1c7c6a5a86423ece9afcf63c -- package: github.com/google/tcpproxy - version: dfa16c61dad2b18a385dfb351adf71566720535b - package: github.com/googleapis/gnostic version: 0c5108395e2debce0d731cf0287ddf7242066aba - package: github.com/gorilla/mux diff --git a/vendor.conf b/vendor.conf index 34ac3014f8..64c0c471c6 100644 --- a/vendor.conf +++ b/vendor.conf @@ -30,7 +30,6 @@ golang.org/x/crypto a49355c7e3f8fe157a85be2f77e6e269a0f89602 gopkg.in/freddierice/go-losetup.v1 fc9adea44124401d8bfef3a97eaf61b5d44cc2c6 github.com/urfave/cli 8e01ec4cd3e2d84ab2fe90d8210528ffbb06d8ff golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c -github.com/google/tcpproxy dfa16c61dad2b18a385dfb351adf71566720535b # flannel github.com/golang/glog 23def4e6c14b4da8ac2ed8007337bc5eb5007998 diff --git a/vendor/github.com/google/tcpproxy/.gitignore b/vendor/github.com/google/tcpproxy/.gitignore deleted file mode 100644 index ab78466b90..0000000000 --- a/vendor/github.com/google/tcpproxy/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -tlsrouter -tlsrouter.test diff --git a/vendor/github.com/google/tcpproxy/.travis.yml b/vendor/github.com/google/tcpproxy/.travis.yml deleted file mode 100644 index 9c94088fa3..0000000000 --- a/vendor/github.com/google/tcpproxy/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -language: go -go: -- 1.8 -- tip -os: -- linux -install: -- go get github.com/golang/lint/golint -before_script: -script: -- go get -t ./... -- go build ./... -- go test ./... -- go vet ./... -- golint -set_exit_status . - -jobs: - include: - - stage: deploy - go: 1.8 - install: - - gem install fpm - script: - - go build ./cmd/tlsrouter - - fpm -s dir -t deb -n tlsrouter -v $(date '+%Y%m%d%H%M%S') - --license Apache2 - --vendor "David Anderson " - --maintainer "David Anderson " - --description "TLS SNI router" - --url "https://github.com/google/tlsrouter" - ./tlsrouter=/usr/bin/tlsrouter - ./systemd/tlsrouter.service=/lib/systemd/system/tlsrouter.service - deploy: - - provider: packagecloud - repository: tlsrouter - username: danderson - dist: debian/stretch - skip_cleanup: true - on: - branch: master - token: - secure: gNU3o70EU4oYeIS6pr0K5oLMGqqxrcf41EOv6c/YoHPVdV6Cx4j9NW0/ISgu6a1/Xf2NgWKT5BWwLpAuhmGdALuOz1Ah//YBWd9N8mGHGaC6RpOPDU8/9NkQdBEmjEH9sgX4PNOh1KQ7d7O0OH0g8RqJlJa0MkUYbTtN6KJ29oiUXxKmZM4D/iWB8VonKOnrtx1NwQL8jL8imZyEV/1fknhDwumz2iKeU1le4Neq9zkxwICMLUonmgphlrp+SDb1EOoHxT6cn51bqBQtQUplfC4dN4OQU/CPqE9E1N1noibvN29YA93qfcrjD3I95KT9wzq+3B6he33+kb0Gz+Cj5ypGy4P85l7TuX4CtQg0U3NAlJCk32IfsdjK+o47pdmADij9IIb9yKt+g99FMERkJJY5EInqEsxHlW/vNF5OqQCmpiHstZL4R2XaHEsWh6j77npnjjC1Aea8xZTWr8PTsbSzVkbG7bTmFpZoPH8eEmr4GNuw5gnbi6D1AJDjcA+UdY9s5qZNpzuWOqfhOFxL+zUW+8sHBvcoFw3R+pwHECs2LCL1c0xAC1LtNUnmW/gnwHavtvKkzErjR1P8Xl7obCbeChJjp+b/BcFYlNACldZcuzBAPyPwIdlWVyUonL4bm63upfMEEShiAIDDJ21y7fjsQK7CfPA7g25bpyo+hV8= - - provider: script - on: - branch: master - script: go run scripts/prune_old_versions.go -user=danderson -repo=tlsrouter -distro=debian -version=stretch -package=tlsrouter -arch=amd64 -limit=2 - env: - # Packagecloud API key, for prune_old_versions.go - - secure: "SRcNwt+45QyPS1w9aGxMg9905Y6d9w4mBM29G6iTTnUB5nD7cAk4m+tf834knGSobVXlWcRnTDW8zrHdQ9yX22dPqCpH5qE+qzTmIvxRHrVJRMmPeYvligJ/9jYfHgQbvuRT8cUpIcpCQAla6rw8nXfKTOE3h8XqMP2hdc3DTVOu2HCfKCNco1tJ7is+AIAnFV2Wpsbb3ZsdKFvHvi2RKUfFaX61J1GNt2/XJIlZs8jC6Y1IAC+ftjql9UsAE/WjZ9fL0Ww1b9/LBIIGHXWI3HpVv9WvlhhIxIlJgOVjmU2lbSuj2w/EBDJ9cd1Qe+wJkT3yKzE1NRsNScVjGg+Ku5igJu/XXuaHkIX01+15BqgPduBYRL0atiNQDhqgBiSyVhXZBX9vsgsp0bgpKaBSF++CV18Q9dara8aljqqS33M3imO3I8JmXU10944QA9Wvu7pCYuIzXxhINcDXRvqxBqz5LnFJGwnGqngTrOCSVS2xn7Y+sjmhe1n5cPCEISlozfa9mPYPvMPp8zg3TbATOOM8CVfcpaNscLqa/+SExN3zMwSanjNKrBgoaQcBzGW5mIgSPxhXkWikBgapiEN7+2Y032Lhqdb9dYjH+EuwcnofspDjjMabWxnuJaln+E3/9vZi2ooQrBEtvymUTy4VMSnqwIX5bU7nPdIuQycdWhk=" diff --git a/vendor/github.com/google/tcpproxy/CONTRIBUTING.md b/vendor/github.com/google/tcpproxy/CONTRIBUTING.md deleted file mode 100644 index 188ad870fc..0000000000 --- a/vendor/github.com/google/tcpproxy/CONTRIBUTING.md +++ /dev/null @@ -1,8 +0,0 @@ -Contributions are welcome by pull request. - -You need to sign the Google Contributor License Agreement before your -contributions can be accepted. You can find the individual and organization -level CLAs here: - -Individual: https://cla.developers.google.com/about/google-individual -Organization: https://cla.developers.google.com/about/google-corporate diff --git a/vendor/github.com/google/tcpproxy/LICENSE b/vendor/github.com/google/tcpproxy/LICENSE deleted file mode 100644 index d645695673..0000000000 --- a/vendor/github.com/google/tcpproxy/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/google/tcpproxy/README.md b/vendor/github.com/google/tcpproxy/README.md deleted file mode 100644 index 374f52c9ee..0000000000 --- a/vendor/github.com/google/tcpproxy/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# tcpproxy - -For library usage, see https://godoc.org/github.com/google/tcpproxy/ - -For CLI usage, see https://github.com/google/tcpproxy/blob/master/cmd/tlsrouter/README.md diff --git a/vendor/github.com/google/tcpproxy/http.go b/vendor/github.com/google/tcpproxy/http.go deleted file mode 100644 index d28c66fa88..0000000000 --- a/vendor/github.com/google/tcpproxy/http.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tcpproxy - -import ( - "bufio" - "bytes" - "context" - "net/http" -) - -// AddHTTPHostRoute appends a route to the ipPort listener that -// routes to dest if the incoming HTTP/1.x Host header name is -// httpHost. If it doesn't match, rule processing continues for any -// additional routes on ipPort. -// -// The ipPort is any valid net.Listen TCP address. -func (p *Proxy) AddHTTPHostRoute(ipPort, httpHost string, dest Target) { - p.AddHTTPHostMatchRoute(ipPort, equals(httpHost), dest) -} - -// AddHTTPHostMatchRoute appends a route to the ipPort listener that -// routes to dest if the incoming HTTP/1.x Host header name is -// accepted by matcher. If it doesn't match, rule processing continues -// for any additional routes on ipPort. -// -// The ipPort is any valid net.Listen TCP address. -func (p *Proxy) AddHTTPHostMatchRoute(ipPort string, match Matcher, dest Target) { - p.addRoute(ipPort, httpHostMatch{match, dest}) -} - -type httpHostMatch struct { - matcher Matcher - target Target -} - -func (m httpHostMatch) match(br *bufio.Reader) (Target, string) { - hh := httpHostHeader(br) - if m.matcher(context.TODO(), hh) { - return m.target, hh - } - return nil, "" -} - -// httpHostHeader returns the HTTP Host header from br without -// consuming any of its bytes. It returns "" if it can't find one. -func httpHostHeader(br *bufio.Reader) string { - const maxPeek = 4 << 10 - peekSize := 0 - for { - peekSize++ - if peekSize > maxPeek { - b, _ := br.Peek(br.Buffered()) - return httpHostHeaderFromBytes(b) - } - b, err := br.Peek(peekSize) - if n := br.Buffered(); n > peekSize { - b, _ = br.Peek(n) - peekSize = n - } - if len(b) > 0 { - if b[0] < 'A' || b[0] > 'Z' { - // Doesn't look like an HTTP verb - // (GET, POST, etc). - return "" - } - if bytes.Index(b, crlfcrlf) != -1 || bytes.Index(b, lflf) != -1 { - req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(b))) - if err != nil { - return "" - } - if len(req.Header["Host"]) > 1 { - // TODO(bradfitz): what does - // ReadRequest do if there are - // multiple Host headers? - return "" - } - return req.Host - } - } - if err != nil { - return httpHostHeaderFromBytes(b) - } - } -} - -var ( - lfHostColon = []byte("\nHost:") - lfhostColon = []byte("\nhost:") - crlf = []byte("\r\n") - lf = []byte("\n") - crlfcrlf = []byte("\r\n\r\n") - lflf = []byte("\n\n") -) - -func httpHostHeaderFromBytes(b []byte) string { - if i := bytes.Index(b, lfHostColon); i != -1 { - return string(bytes.TrimSpace(untilEOL(b[i+len(lfHostColon):]))) - } - if i := bytes.Index(b, lfhostColon); i != -1 { - return string(bytes.TrimSpace(untilEOL(b[i+len(lfhostColon):]))) - } - return "" -} - -// untilEOL returns v, truncated before the first '\n' byte, if any. -// The returned slice may include a '\r' at the end. -func untilEOL(v []byte) []byte { - if i := bytes.IndexByte(v, '\n'); i != -1 { - return v[:i] - } - return v -} diff --git a/vendor/github.com/google/tcpproxy/listener.go b/vendor/github.com/google/tcpproxy/listener.go deleted file mode 100644 index 1ddc48ee21..0000000000 --- a/vendor/github.com/google/tcpproxy/listener.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tcpproxy - -import ( - "io" - "net" - "sync" -) - -// TargetListener implements both net.Listener and Target. -// Matched Targets become accepted connections. -type TargetListener struct { - Address string // Address is the string reported by TargetListener.Addr().String(). - - mu sync.Mutex - cond *sync.Cond - closed bool - nextConn net.Conn -} - -var ( - _ net.Listener = (*TargetListener)(nil) - _ Target = (*TargetListener)(nil) -) - -func (tl *TargetListener) lock() { - tl.mu.Lock() - if tl.cond == nil { - tl.cond = sync.NewCond(&tl.mu) - } -} - -type tcpAddr string - -func (a tcpAddr) Network() string { return "tcp" } -func (a tcpAddr) String() string { return string(a) } - -// Addr returns the listener's Address field as a net.Addr. -func (tl *TargetListener) Addr() net.Addr { return tcpAddr(tl.Address) } - -// Close stops listening for new connections. All new connections -// routed to this listener will be closed. Already accepted -// connections are not closed. -func (tl *TargetListener) Close() error { - tl.lock() - if tl.closed { - tl.mu.Unlock() - return nil - } - tl.closed = true - tl.mu.Unlock() - tl.cond.Broadcast() - return nil -} - -// HandleConn implements the Target interface. It blocks until tl is -// closed or another goroutine has called Accept and received c. -func (tl *TargetListener) HandleConn(c net.Conn) { - tl.lock() - defer tl.mu.Unlock() - for tl.nextConn != nil && !tl.closed { - tl.cond.Wait() - } - if tl.closed { - c.Close() - return - } - tl.nextConn = c - tl.cond.Broadcast() // Signal might be sufficient; verify. - for tl.nextConn == c && !tl.closed { - tl.cond.Wait() - } - if tl.closed { - c.Close() - return - } -} - -// Accept implements the Accept method in the net.Listener interface. -func (tl *TargetListener) Accept() (net.Conn, error) { - tl.lock() - for tl.nextConn == nil && !tl.closed { - tl.cond.Wait() - } - if tl.closed { - tl.mu.Unlock() - return nil, io.EOF - } - c := tl.nextConn - tl.nextConn = nil - tl.mu.Unlock() - tl.cond.Broadcast() // Signal might be sufficient; verify. - - return c, nil -} diff --git a/vendor/github.com/google/tcpproxy/sni.go b/vendor/github.com/google/tcpproxy/sni.go deleted file mode 100644 index 53b53c24d7..0000000000 --- a/vendor/github.com/google/tcpproxy/sni.go +++ /dev/null @@ -1,192 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package tcpproxy - -import ( - "bufio" - "bytes" - "context" - "crypto/tls" - "io" - "net" - "strings" -) - -// AddSNIRoute appends a route to the ipPort listener that routes to -// dest if the incoming TLS SNI server name is sni. If it doesn't -// match, rule processing continues for any additional routes on -// ipPort. -// -// By default, the proxy will route all ACME tls-sni-01 challenges -// received on ipPort to all SNI dests. You can disable ACME routing -// with AddStopACMESearch. -// -// The ipPort is any valid net.Listen TCP address. -func (p *Proxy) AddSNIRoute(ipPort, sni string, dest Target) { - p.AddSNIMatchRoute(ipPort, equals(sni), dest) -} - -// AddSNIMatchRoute appends a route to the ipPort listener that routes -// to dest if the incoming TLS SNI server name is accepted by -// matcher. If it doesn't match, rule processing continues for any -// additional routes on ipPort. -// -// By default, the proxy will route all ACME tls-sni-01 challenges -// received on ipPort to all SNI dests. You can disable ACME routing -// with AddStopACMESearch. -// -// The ipPort is any valid net.Listen TCP address. -func (p *Proxy) AddSNIMatchRoute(ipPort string, matcher Matcher, dest Target) { - cfg := p.configFor(ipPort) - if !cfg.stopACME { - if len(cfg.acmeTargets) == 0 { - p.addRoute(ipPort, &acmeMatch{cfg}) - } - cfg.acmeTargets = append(cfg.acmeTargets, dest) - } - - p.addRoute(ipPort, sniMatch{matcher, dest}) -} - -// AddStopACMESearch prevents ACME probing of subsequent SNI routes. -// Any ACME challenges on ipPort for SNI routes previously added -// before this call will still be proxied to all possible SNI -// backends. -func (p *Proxy) AddStopACMESearch(ipPort string) { - p.configFor(ipPort).stopACME = true -} - -type sniMatch struct { - matcher Matcher - target Target -} - -func (m sniMatch) match(br *bufio.Reader) (Target, string) { - sni := clientHelloServerName(br) - if m.matcher(context.TODO(), sni) { - return m.target, sni - } - return nil, "" -} - -// acmeMatch matches "*.acme.invalid" ACME tls-sni-01 challenges and -// searches for a Target in cfg.acmeTargets that has the challenge -// response. -type acmeMatch struct { - cfg *config -} - -func (m *acmeMatch) match(br *bufio.Reader) (Target, string) { - sni := clientHelloServerName(br) - if !strings.HasSuffix(sni, ".acme.invalid") { - return nil, "" - } - - // TODO: cache. ACME issuers will hit multiple times in a short - // burst for each issuance event. A short TTL cache + singleflight - // should have an excellent hit rate. - // TODO: maybe an acme-specific timeout as well? - // TODO: plumb context upwards? - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ch := make(chan Target, len(m.cfg.acmeTargets)) - for _, target := range m.cfg.acmeTargets { - go tryACME(ctx, ch, target, sni) - } - for range m.cfg.acmeTargets { - if target := <-ch; target != nil { - return target, sni - } - } - - // No target was happy with the provided challenge. - return nil, "" -} - -func tryACME(ctx context.Context, ch chan<- Target, dest Target, sni string) { - var ret Target - defer func() { ch <- ret }() - - conn, targetConn := net.Pipe() - defer conn.Close() - go dest.HandleConn(targetConn) - - deadline, ok := ctx.Deadline() - if ok { - conn.SetDeadline(deadline) - } - - client := tls.Client(conn, &tls.Config{ - ServerName: sni, - InsecureSkipVerify: true, - }) - if err := client.Handshake(); err != nil { - // TODO: log? - return - } - certs := client.ConnectionState().PeerCertificates - if len(certs) == 0 { - // TODO: log? - return - } - // acme says the first cert offered by the server must match the - // challenge hostname. - if err := certs[0].VerifyHostname(sni); err != nil { - // TODO: log? - return - } - - // Target presented what looks like a valid challenge - // response, send it back to the matcher. - ret = dest -} - -// clientHelloServerName returns the SNI server name inside the TLS ClientHello, -// without consuming any bytes from br. -// On any error, the empty string is returned. -func clientHelloServerName(br *bufio.Reader) (sni string) { - const recordHeaderLen = 5 - hdr, err := br.Peek(recordHeaderLen) - if err != nil { - return "" - } - const recordTypeHandshake = 0x16 - if hdr[0] != recordTypeHandshake { - return "" // Not TLS. - } - recLen := int(hdr[3])<<8 | int(hdr[4]) // ignoring version in hdr[1:3] - helloBytes, err := br.Peek(recordHeaderLen + recLen) - if err != nil { - return "" - } - tls.Server(sniSniffConn{r: bytes.NewReader(helloBytes)}, &tls.Config{ - GetConfigForClient: func(hello *tls.ClientHelloInfo) (*tls.Config, error) { - sni = hello.ServerName - return nil, nil - }, - }).Handshake() - return -} - -// sniSniffConn is a net.Conn that reads from r, fails on Writes, -// and crashes otherwise. -type sniSniffConn struct { - r io.Reader - net.Conn // nil; crash on any unexpected use -} - -func (c sniSniffConn) Read(p []byte) (int, error) { return c.r.Read(p) } -func (sniSniffConn) Write(p []byte) (int, error) { return 0, io.EOF } diff --git a/vendor/github.com/google/tcpproxy/tcpproxy.go b/vendor/github.com/google/tcpproxy/tcpproxy.go deleted file mode 100644 index 9826d94226..0000000000 --- a/vendor/github.com/google/tcpproxy/tcpproxy.go +++ /dev/null @@ -1,474 +0,0 @@ -// Copyright 2017 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package tcpproxy lets users build TCP proxies, optionally making -// routing decisions based on HTTP/1 Host headers and the SNI hostname -// in TLS connections. -// -// Typical usage: -// -// var p tcpproxy.Proxy -// p.AddHTTPHostRoute(":80", "foo.com", tcpproxy.To("10.0.0.1:8081")) -// p.AddHTTPHostRoute(":80", "bar.com", tcpproxy.To("10.0.0.2:8082")) -// p.AddRoute(":80", tcpproxy.To("10.0.0.1:8081")) // fallback -// p.AddSNIRoute(":443", "foo.com", tcpproxy.To("10.0.0.1:4431")) -// p.AddSNIRoute(":443", "bar.com", tcpproxy.To("10.0.0.2:4432")) -// p.AddRoute(":443", tcpproxy.To("10.0.0.1:4431")) // fallback -// log.Fatal(p.Run()) -// -// Calling Run (or Start) on a proxy also starts all the necessary -// listeners. -// -// For each accepted connection, the rules for that ipPort are -// matched, in order. If one matches (currently HTTP Host, SNI, or -// always), then the connection is handed to the target. -// -// The two predefined Target implementations are: -// -// 1) DialProxy, proxying to another address (use the To func to return a -// DialProxy value), -// -// 2) TargetListener, making the matched connection available via a -// net.Listener.Accept call. -// -// But Target is an interface, so you can also write your own. -// -// Note that tcpproxy does not do any TLS encryption or decryption. It -// only (via DialProxy) copies bytes around. The SNI hostname in the TLS -// header is unencrypted, for better or worse. -// -// This package makes no API stability promises. If you depend on it, -// vendor it. -package tcpproxy - -import ( - "bufio" - "context" - "errors" - "fmt" - "io" - "log" - "net" - "time" -) - -// Proxy is a proxy. Its zero value is a valid proxy that does -// nothing. Call methods to add routes before calling Start or Run. -// -// The order that routes are added in matters; each is matched in the order -// registered. -type Proxy struct { - configs map[string]*config // ip:port => config - - lns []net.Listener - donec chan struct{} // closed before err - err error // any error from listening - - // ListenFunc optionally specifies an alternate listen - // function. If nil, net.Dial is used. - // The provided net is always "tcp". - ListenFunc func(net, laddr string) (net.Listener, error) -} - -// Matcher reports whether hostname matches the Matcher's criteria. -type Matcher func(ctx context.Context, hostname string) bool - -// equals is a trivial Matcher that implements string equality. -func equals(want string) Matcher { - return func(_ context.Context, got string) bool { - return want == got - } -} - -// config contains the proxying state for one listener. -type config struct { - routes []route - acmeTargets []Target // accumulates targets that should be probed for acme. - stopACME bool // if true, AddSNIRoute doesn't add targets to acmeTargets. -} - -// A route matches a connection to a target. -type route interface { - // match examines the initial bytes of a connection, looking for a - // match. If a match is found, match returns a non-nil Target to - // which the stream should be proxied. match returns nil if the - // connection doesn't match. - // - // match must not consume bytes from the given bufio.Reader, it - // can only Peek. - // - // If an sni or host header was parsed successfully, that will be - // returned as the second parameter. - match(*bufio.Reader) (Target, string) -} - -func (p *Proxy) netListen() func(net, laddr string) (net.Listener, error) { - if p.ListenFunc != nil { - return p.ListenFunc - } - return net.Listen -} - -func (p *Proxy) configFor(ipPort string) *config { - if p.configs == nil { - p.configs = make(map[string]*config) - } - if p.configs[ipPort] == nil { - p.configs[ipPort] = &config{} - } - return p.configs[ipPort] -} - -func (p *Proxy) addRoute(ipPort string, r route) { - cfg := p.configFor(ipPort) - cfg.routes = append(cfg.routes, r) -} - -// AddRoute appends an always-matching route to the ipPort listener, -// directing any connection to dest. -// -// This is generally used as either the only rule (for simple TCP -// proxies), or as the final fallback rule for an ipPort. -// -// The ipPort is any valid net.Listen TCP address. -func (p *Proxy) AddRoute(ipPort string, dest Target) { - p.addRoute(ipPort, fixedTarget{dest}) -} - -type fixedTarget struct { - t Target -} - -func (m fixedTarget) match(*bufio.Reader) (Target, string) { return m.t, "" } - -// Run is calls Start, and then Wait. -// -// It blocks until there's an error. The return value is always -// non-nil. -func (p *Proxy) Run() error { - if err := p.Start(); err != nil { - return err - } - return p.Wait() -} - -// Wait waits for the Proxy to finish running. Currently this can only -// happen if a Listener is closed, or Close is called on the proxy. -// -// It is only valid to call Wait after a successful call to Start. -func (p *Proxy) Wait() error { - <-p.donec - return p.err -} - -// Close closes all the proxy's self-opened listeners. -func (p *Proxy) Close() error { - for _, c := range p.lns { - c.Close() - } - return nil -} - -// Start creates a TCP listener for each unique ipPort from the -// previously created routes and starts the proxy. It returns any -// error from starting listeners. -// -// If it returns a non-nil error, any successfully opened listeners -// are closed. -func (p *Proxy) Start() error { - if p.donec != nil { - return errors.New("already started") - } - p.donec = make(chan struct{}) - errc := make(chan error, len(p.configs)) - p.lns = make([]net.Listener, 0, len(p.configs)) - for ipPort, config := range p.configs { - ln, err := p.netListen()("tcp", ipPort) - if err != nil { - p.Close() - return err - } - p.lns = append(p.lns, ln) - go p.serveListener(errc, ln, config.routes) - } - go p.awaitFirstError(errc) - return nil -} - -func (p *Proxy) awaitFirstError(errc <-chan error) { - p.err = <-errc - close(p.donec) -} - -func (p *Proxy) serveListener(ret chan<- error, ln net.Listener, routes []route) { - for { - c, err := ln.Accept() - if err != nil { - ret <- err - return - } - go p.serveConn(c, routes) - } -} - -// serveConn runs in its own goroutine and matches c against routes. -// It returns whether it matched purely for testing. -func (p *Proxy) serveConn(c net.Conn, routes []route) bool { - br := bufio.NewReader(c) - for _, route := range routes { - if target, hostName := route.match(br); target != nil { - if n := br.Buffered(); n > 0 { - peeked, _ := br.Peek(br.Buffered()) - c = &Conn{ - HostName: hostName, - Peeked: peeked, - Conn: c, - } - } - target.HandleConn(c) - return true - } - } - // TODO: hook for this? - log.Printf("tcpproxy: no routes matched conn %v/%v; closing", c.RemoteAddr().String(), c.LocalAddr().String()) - c.Close() - return false -} - -// Conn is an incoming connection that has had some bytes read from it -// to determine how to route the connection. The Read method stitches -// the peeked bytes and unread bytes back together. -type Conn struct { - // HostName is the hostname field that was sent to the request router. - // In the case of TLS, this is the SNI header, in the case of HTTPHost - // route, it will be the host header. In the case of a fixed - // route, i.e. those created with AddRoute(), this will always be - // empty. This can be useful in the case where further routing decisions - // need to be made in the Target impementation. - HostName string - - // Peeked are the bytes that have been read from Conn for the - // purposes of route matching, but have not yet been consumed - // by Read calls. It set to nil by Read when fully consumed. - Peeked []byte - - // Conn is the underlying connection. - // It can be type asserted against *net.TCPConn or other types - // as needed. It should not be read from directly unless - // Peeked is nil. - net.Conn -} - -func (c *Conn) Read(p []byte) (n int, err error) { - if len(c.Peeked) > 0 { - n = copy(p, c.Peeked) - c.Peeked = c.Peeked[n:] - if len(c.Peeked) == 0 { - c.Peeked = nil - } - return n, nil - } - return c.Conn.Read(p) -} - -// Target is what an incoming matched connection is sent to. -type Target interface { - // HandleConn is called when an incoming connection is - // matched. After the call to HandleConn, the tcpproxy - // package never touches the conn again. Implementations are - // responsible for closing the connection when needed. - // - // The concrete type of conn will be of type *Conn if any - // bytes have been consumed for the purposes of route - // matching. - HandleConn(net.Conn) -} - -// To is shorthand way of writing &tlsproxy.DialProxy{Addr: addr}. -func To(addr string) *DialProxy { - return &DialProxy{Addr: addr} -} - -// DialProxy implements Target by dialing a new connection to Addr -// and then proxying data back and forth. -// -// The To func is a shorthand way of creating a DialProxy. -type DialProxy struct { - // Addr is the TCP address to proxy to. - Addr string - - // KeepAlivePeriod sets the period between TCP keep alives. - // If zero, a default is used. To disable, use a negative number. - // The keep-alive is used for both the client connection and - KeepAlivePeriod time.Duration - - // DialTimeout optionally specifies a dial timeout. - // If zero, a default is used. - // If negative, the timeout is disabled. - DialTimeout time.Duration - - // DialContext optionally specifies an alternate dial function - // for TCP targets. If nil, the standard - // net.Dialer.DialContext method is used. - DialContext func(ctx context.Context, network, address string) (net.Conn, error) - - // OnDialError optionally specifies an alternate way to handle errors dialing Addr. - // If nil, the error is logged and src is closed. - // If non-nil, src is not closed automatically. - OnDialError func(src net.Conn, dstDialErr error) - - // ProxyProtocolVersion optionally specifies the version of - // HAProxy's PROXY protocol to use. The PROXY protocol provides - // connection metadata to the DialProxy target, via a header - // inserted ahead of the client's traffic. The DialProxy target - // must explicitly support and expect the PROXY header; there is - // no graceful downgrade. - // If zero, no PROXY header is sent. Currently, version 1 is supported. - ProxyProtocolVersion int -} - -// UnderlyingConn returns c.Conn if c of type *Conn, -// otherwise it returns c. -func UnderlyingConn(c net.Conn) net.Conn { - if wrap, ok := c.(*Conn); ok { - return wrap.Conn - } - return c -} - -func goCloseConn(c net.Conn) { go c.Close() } - -// HandleConn implements the Target interface. -func (dp *DialProxy) HandleConn(src net.Conn) { - ctx := context.Background() - var cancel context.CancelFunc - if dp.DialTimeout >= 0 { - ctx, cancel = context.WithTimeout(ctx, dp.dialTimeout()) - } - dst, err := dp.dialContext()(ctx, "tcp", dp.Addr) - if cancel != nil { - cancel() - } - if err != nil { - dp.onDialError()(src, err) - return - } - defer goCloseConn(dst) - - if err = dp.sendProxyHeader(dst, src); err != nil { - dp.onDialError()(src, err) - return - } - defer goCloseConn(src) - - if ka := dp.keepAlivePeriod(); ka > 0 { - if c, ok := UnderlyingConn(src).(*net.TCPConn); ok { - c.SetKeepAlive(true) - c.SetKeepAlivePeriod(ka) - } - if c, ok := dst.(*net.TCPConn); ok { - c.SetKeepAlive(true) - c.SetKeepAlivePeriod(ka) - } - } - - errc := make(chan error, 1) - go proxyCopy(errc, src, dst) - go proxyCopy(errc, dst, src) - <-errc -} - -func (dp *DialProxy) sendProxyHeader(w io.Writer, src net.Conn) error { - switch dp.ProxyProtocolVersion { - case 0: - return nil - case 1: - var srcAddr, dstAddr *net.TCPAddr - if a, ok := src.RemoteAddr().(*net.TCPAddr); ok { - srcAddr = a - } - if a, ok := src.LocalAddr().(*net.TCPAddr); ok { - dstAddr = a - } - - if srcAddr == nil || dstAddr == nil { - _, err := io.WriteString(w, "PROXY UNKNOWN\r\n") - return err - } - - family := "TCP4" - if srcAddr.IP.To4() == nil { - family = "TCP6" - } - _, err := fmt.Fprintf(w, "PROXY %s %s %d %s %d\r\n", family, srcAddr.IP, srcAddr.Port, dstAddr.IP, dstAddr.Port) - return err - default: - return fmt.Errorf("PROXY protocol version %d not supported", dp.ProxyProtocolVersion) - } -} - -// proxyCopy is the function that copies bytes around. -// It's a named function instead of a func literal so users get -// named goroutines in debug goroutine stack dumps. -func proxyCopy(errc chan<- error, dst, src net.Conn) { - // Before we unwrap src and/or dst, copy any buffered data. - if wc, ok := src.(*Conn); ok && len(wc.Peeked) > 0 { - if _, err := dst.Write(wc.Peeked); err != nil { - errc <- err - return - } - wc.Peeked = nil - } - - // Unwrap the src and dst from *Conn to *net.TCPConn so Go - // 1.11's splice optimization kicks in. - src = UnderlyingConn(src) - dst = UnderlyingConn(dst) - - _, err := io.Copy(dst, src) - errc <- err -} - -func (dp *DialProxy) keepAlivePeriod() time.Duration { - if dp.KeepAlivePeriod != 0 { - return dp.KeepAlivePeriod - } - return time.Minute -} - -func (dp *DialProxy) dialTimeout() time.Duration { - if dp.DialTimeout > 0 { - return dp.DialTimeout - } - return 10 * time.Second -} - -var defaultDialer = new(net.Dialer) - -func (dp *DialProxy) dialContext() func(ctx context.Context, network, address string) (net.Conn, error) { - if dp.DialContext != nil { - return dp.DialContext - } - return defaultDialer.DialContext -} - -func (dp *DialProxy) onDialError() func(src net.Conn, dstDialErr error) { - if dp.OnDialError != nil { - return dp.OnDialError - } - return func(src net.Conn, dstDialErr error) { - log.Printf("tcpproxy: for incoming conn %v, error dialing %q: %v", src.RemoteAddr().String(), dp.Addr, dstDialErr) - src.Close() - } -}