From f4fd8453ed186ee6c7b78f9515d120e06b92be0c Mon Sep 17 00:00:00 2001 From: Atin Malaviya Date: Fri, 14 Nov 2014 14:39:19 -0500 Subject: [PATCH 1/3] Added HTTPS support via a new HTTPS Port configuration option similar to the HTTP Port. --- command/agent/command.go | 32 ++--- command/agent/config.go | 91 ++++++++++++- command/agent/config_test.go | 19 ++- command/agent/http.go | 124 +++++++++++++++--- .../source/docs/agent/options.html.markdown | 1 + 5 files changed, 220 insertions(+), 47 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index 6aa51ac7c8..de800e673f 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -43,7 +43,7 @@ type Command struct { logOutput io.Writer agent *Agent rpcServer *AgentRPC - httpServer *HTTPServer + httpServers []*HTTPServer dnsServer *DNSServer } @@ -71,7 +71,7 @@ func (c *Command) readConfig() *Config { cmdFlags.BoolVar(&cmdConfig.Bootstrap, "bootstrap", false, "enable server bootstrap mode") cmdFlags.IntVar(&cmdConfig.BootstrapExpect, "bootstrap-expect", 0, "enable automatic bootstrap via expect mode") - cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, RPC)") + cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, HTTPS, RPC)") cmdFlags.StringVar(&cmdConfig.BindAddr, "bind", "", "address to bind server listeners to") cmdFlags.StringVar(&cmdConfig.AdvertiseAddr, "advertise", "", "address to advertise instead of bind addr") @@ -278,20 +278,14 @@ func (c *Command) setupAgent(config *Config, logOutput io.Writer, logWriter *log c.Ui.Output("Starting Consul agent RPC...") c.rpcServer = NewAgentRPC(agent, rpcListener, logOutput, logWriter) - if config.Ports.HTTP > 0 { - httpAddr, err := config.ClientListener(config.Addresses.HTTP, config.Ports.HTTP) - if err != nil { - c.Ui.Error(fmt.Sprintf("Invalid HTTP bind address: %s", err)) - return err - } - - server, err := NewHTTPServer(agent, config.UiDir, config.EnableDebug, logOutput, httpAddr.String()) + if config.Ports.HTTP > 0 || config.Ports.HTTPS > 0 { + servers, err := NewHTTPServers(agent, config, logOutput) if err != nil { agent.Shutdown() - c.Ui.Error(fmt.Sprintf("Error starting http server: %s", err)) + c.Ui.Error(fmt.Sprintf("Error starting http servers:", err)) return err } - c.httpServer = server + c.httpServers = servers } if config.Ports.DNS > 0 { @@ -472,8 +466,10 @@ func (c *Command) Run(args []string) int { if c.rpcServer != nil { defer c.rpcServer.Shutdown() } - if c.httpServer != nil { - defer c.httpServer.Shutdown() + if c.httpServers != nil { + for _, server := range c.httpServers { + defer server.Shutdown() + } } // Join startup nodes if specified @@ -502,7 +498,7 @@ func (c *Command) Run(args []string) int { } } - // Get the new client listener addr + // Get the new client http listener addr httpAddr, err := config.ClientListenerAddr(config.Addresses.HTTP, config.Ports.HTTP) if err != nil { c.Ui.Error(fmt.Sprintf("Failed to determine HTTP address: %v", err)) @@ -526,8 +522,8 @@ func (c *Command) Run(args []string) int { c.Ui.Info(fmt.Sprintf(" Node name: '%s'", config.NodeName)) c.Ui.Info(fmt.Sprintf(" Datacenter: '%s'", config.Datacenter)) c.Ui.Info(fmt.Sprintf(" Server: %v (bootstrap: %v)", config.Server, config.Bootstrap)) - c.Ui.Info(fmt.Sprintf(" Client Addr: %v (HTTP: %d, DNS: %d, RPC: %d)", config.ClientAddr, - config.Ports.HTTP, config.Ports.DNS, config.Ports.RPC)) + c.Ui.Info(fmt.Sprintf(" Client Addr: %v (HTTP: %d, HTTPS: %d, DNS: %d, RPC: %d)", config.ClientAddr, + config.Ports.HTTP, config.Ports.HTTPS, config.Ports.DNS, config.Ports.RPC)) c.Ui.Info(fmt.Sprintf(" Cluster Addr: %v (LAN: %d, WAN: %d)", config.AdvertiseAddr, config.Ports.SerfLan, config.Ports.SerfWan)) c.Ui.Info(fmt.Sprintf("Gossip encrypt: %v, RPC-TLS: %v, TLS-Incoming: %v", @@ -709,7 +705,7 @@ Options: -bind=0.0.0.0 Sets the bind address for cluster communication -bootstrap-expect=0 Sets server to expect bootstrap mode. -client=127.0.0.1 Sets the address to bind for client access. - This includes RPC, DNS and HTTP + This includes RPC, DNS, HTTP and HTTPS (if configured) -config-file=foo Path to a JSON file to read configuration from. This can be specified multiple times. -config-dir=foo Path to a directory to read configuration files diff --git a/command/agent/config.go b/command/agent/config.go index 3a926499f2..de9cb13d76 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -1,10 +1,13 @@ package agent import ( + "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/json" "fmt" "io" + "io/ioutil" "net" "os" "path/filepath" @@ -23,6 +26,7 @@ import ( type PortConfig struct { DNS int // DNS Query interface HTTP int // HTTP API + HTTPS int // HTTPS API RPC int // CLI RPC SerfLan int `mapstructure:"serf_lan"` // LAN gossip (Client + Server) SerfWan int `mapstructure:"serf_wan"` // WAN gossip (Server onlyg) @@ -33,9 +37,10 @@ type PortConfig struct { // for specific services. By default, either ClientAddress // or ServerAddress is used. type AddressConfig struct { - DNS string // DNS Query interface - HTTP string // HTTP API - RPC string // CLI RPC + DNS string // DNS Query interface + HTTP string // HTTP API + HTTPS string // HTTPS API + RPC string // CLI RPC } // DNSConfig is used to fine tune the DNS sub-system. @@ -122,7 +127,7 @@ type Config struct { NodeName string `mapstructure:"node_name"` // ClientAddr is used to control the address we bind to for - // client services (DNS, HTTP, RPC) + // client services (DNS, HTTP, HTTPS, RPC) ClientAddr string `mapstructure:"client_addr"` // BindAddr is used to control the address we bind to. @@ -332,6 +337,7 @@ func DefaultConfig() *Config { Ports: PortConfig{ DNS: 8600, HTTP: 8500, + HTTPS: -1, RPC: 8400, SerfLan: consul.DefaultLANSerfPort, SerfWan: consul.DefaultWANSerfPort, @@ -385,6 +391,77 @@ func (c *Config) ClientListenerAddr(override string, port int) (string, error) { return addr.String(), nil } +// AppendCA opens and parses the CA file and adds the certificates to +// the provided CertPool. +func (c *Config) AppendCA(pool *x509.CertPool) error { + if c.CAFile == "" { + return nil + } + + // Read the file + data, err := ioutil.ReadFile(c.CAFile) + if err != nil { + return fmt.Errorf("Failed to read CA file: %v", err) + } + + if !pool.AppendCertsFromPEM(data) { + return fmt.Errorf("Failed to parse any CA certificates") + } + + return nil +} + +// KeyPair is used to open and parse a certificate and key file +func (c *Config) KeyPair() (*tls.Certificate, error) { + if c.CertFile == "" || c.KeyFile == "" { + return nil, nil + } + cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) + if err != nil { + return nil, fmt.Errorf("Failed to load cert/key pair: %v", err) + } + return &cert, err +} + +// IncomingTLSConfig generates a TLS configuration for incoming requests +func (c *Config) IncomingTLSConfig() (*tls.Config, error) { + // Create the tlsConfig + tlsConfig := &tls.Config{ + ServerName: c.ServerName, + ClientCAs: x509.NewCertPool(), + ClientAuth: tls.NoClientCert, + } + if tlsConfig.ServerName == "" { + tlsConfig.ServerName = c.NodeName + } + + // Parse the CA cert if any + err := c.AppendCA(tlsConfig.ClientCAs) + if err != nil { + return nil, err + } + + // Add cert/key + cert, err := c.KeyPair() + if err != nil { + return nil, err + } else if cert != nil { + tlsConfig.Certificates = []tls.Certificate{*cert} + } + + // Check if we require verification + if c.VerifyIncoming { + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + if c.CAFile == "" { + return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!") + } + if cert == nil { + return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!") + } + } + return tlsConfig, nil +} + // DecodeConfig reads the configuration from the given reader in JSON // format and decodes it into a proper Config structure. func DecodeConfig(r io.Reader) (*Config, error) { @@ -711,6 +788,9 @@ func MergeConfig(a, b *Config) *Config { if b.Ports.HTTP != 0 { result.Ports.HTTP = b.Ports.HTTP } + if b.Ports.HTTPS != 0 { + result.Ports.HTTPS = b.Ports.HTTPS + } if b.Ports.RPC != 0 { result.Ports.RPC = b.Ports.RPC } @@ -729,6 +809,9 @@ func MergeConfig(a, b *Config) *Config { if b.Addresses.HTTP != "" { result.Addresses.HTTP = b.Addresses.HTTP } + if b.Addresses.HTTPS != "" { + result.Addresses.HTTPS = b.Addresses.HTTPS + } if b.Addresses.RPC != "" { result.Addresses.RPC = b.Addresses.RPC } diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 77e67a1728..80ca0ec5a2 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -137,7 +137,7 @@ func TestDecodeConfig(t *testing.T) { } // RPC configs - input = `{"ports": {"http": 1234, "rpc": 8100}, "client_addr": "0.0.0.0"}` + input = `{"ports": {"http": 1234, "https": 1243, "rpc": 8100}, "client_addr": "0.0.0.0"}` config, err = DecodeConfig(bytes.NewReader([]byte(input))) if err != nil { t.Fatalf("err: %s", err) @@ -151,6 +151,10 @@ func TestDecodeConfig(t *testing.T) { t.Fatalf("bad: %#v", config) } + if config.Ports.HTTPS != 1243 { + t.Fatalf("bad: %#v", config) + } + if config.Ports.RPC != 8100 { t.Fatalf("bad: %#v", config) } @@ -494,7 +498,7 @@ func TestDecodeConfig(t *testing.T) { } // Address overrides - input = `{"addresses": {"dns": "0.0.0.0", "http": "127.0.0.1", "rpc": "127.0.0.1"}}` + input = `{"addresses": {"dns": "0.0.0.0", "http": "127.0.0.1", "https": "127.0.0.1", "rpc": "127.0.0.1"}}` config, err = DecodeConfig(bytes.NewReader([]byte(input))) if err != nil { t.Fatalf("err: %s", err) @@ -506,6 +510,9 @@ func TestDecodeConfig(t *testing.T) { if config.Addresses.HTTP != "127.0.0.1" { t.Fatalf("bad: %#v", config) } + if config.Addresses.HTTPS != "127.0.0.1" { + t.Fatalf("bad: %#v", config) + } if config.Addresses.RPC != "127.0.0.1" { t.Fatalf("bad: %#v", config) } @@ -842,11 +849,13 @@ func TestMergeConfig(t *testing.T) { SerfLan: 4, SerfWan: 5, Server: 6, + HTTPS: 7, }, Addresses: AddressConfig{ - DNS: "127.0.0.1", - HTTP: "127.0.0.2", - RPC: "127.0.0.3", + DNS: "127.0.0.1", + HTTP: "127.0.0.2", + RPC: "127.0.0.3", + HTTPS: "127.0.0.4", }, Server: true, LeaveOnTerm: true, diff --git a/command/agent/http.go b/command/agent/http.go index 1954559ac5..5d07f23de6 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -1,6 +1,7 @@ package agent import ( + "crypto/tls" "encoding/json" "io" "log" @@ -23,38 +24,121 @@ type HTTPServer struct { listener net.Listener logger *log.Logger uiDir string + addr string } -// NewHTTPServer starts a new HTTP server to provide an interface to +// NewHTTPServers starts new HTTP servers to provide an interface to // the agent. -func NewHTTPServer(agent *Agent, uiDir string, enableDebug bool, logOutput io.Writer, bind string) (*HTTPServer, error) { - // Create the mux - mux := http.NewServeMux() +func NewHTTPServers(agent *Agent, config *Config, logOutput io.Writer) ([]*HTTPServer, error) { + var tlsConfig *tls.Config + var list net.Listener + var httpAddr *net.TCPAddr + var err error + var servers []*HTTPServer + + if config.Ports.HTTPS > 0 { + httpAddr, err = config.ClientListener(config.Addresses.HTTPS, config.Ports.HTTPS) + if err != nil { + return nil, err + } - // Create listener - list, err := net.Listen("tcp", bind) - if err != nil { - return nil, err + tlsConfig, err = config.IncomingTLSConfig() + if err != nil { + return nil, err + } + + ln, err := net.Listen("tcp", httpAddr.String()) + if err != nil { + return nil, err + } + + list = tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, tlsConfig) + + // Create the mux + mux := http.NewServeMux() + + // Create the server + srv := &HTTPServer{ + agent: agent, + mux: mux, + listener: list, + logger: log.New(logOutput, "", log.LstdFlags), + uiDir: config.UiDir, + addr: httpAddr.String(), + } + srv.registerHandlers(config.EnableDebug) + + // Start the server + go http.Serve(list, mux) + + servers := make([]*HTTPServer, 1) + servers[0] = srv } - // Create the server - srv := &HTTPServer{ - agent: agent, - mux: mux, - listener: list, - logger: log.New(logOutput, "", log.LstdFlags), - uiDir: uiDir, + if config.Ports.HTTP > 0 { + httpAddr, err = config.ClientListener(config.Addresses.HTTP, config.Ports.HTTP) + if err != nil { + return nil, err + } + + // Create non-TLS listener + list, err = net.Listen("tcp", httpAddr.String()) + if err != nil { + return nil, err + } + + // Create the mux + mux := http.NewServeMux() + + // Create the server + srv := &HTTPServer{ + agent: agent, + mux: mux, + listener: list, + logger: log.New(logOutput, "", log.LstdFlags), + uiDir: config.UiDir, + addr: httpAddr.String(), + } + srv.registerHandlers(config.EnableDebug) + + // Start the server + go http.Serve(list, mux) + + if servers != nil { + // we already have the https server in servers, append + servers = append(servers, srv) + } else { + servers := make([]*HTTPServer, 1) + servers[0] = srv + } } - srv.registerHandlers(enableDebug) - // Start the server - go http.Serve(list, mux) - return srv, nil + return servers, nil +} + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by NewHttpServer so +// dead TCP connections eventually go away. +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { + tc, err := ln.AcceptTCP() + if err != nil { + return + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Minute) + return tc, nil } // Shutdown is used to shutdown the HTTP server func (s *HTTPServer) Shutdown() { - s.listener.Close() + if s != nil { + s.logger.Printf("[DEBUG] http: Shutting down http server(%v)", s.addr) + s.listener.Close() + } } // registerHandlers is used to attach our handlers to the mux diff --git a/website/source/docs/agent/options.html.markdown b/website/source/docs/agent/options.html.markdown index de3bbd78a7..825eb8d6cb 100644 --- a/website/source/docs/agent/options.html.markdown +++ b/website/source/docs/agent/options.html.markdown @@ -316,6 +316,7 @@ definitions support being updated during a reload. for the following keys: * `dns` - The DNS server, -1 to disable. Default 8600. * `http` - The HTTP api, -1 to disable. Default 8500. + * `https` - The HTTPS api, -1 to disable. Default -1 (disabled). * `rpc` - The RPC endpoint. Default 8400. * `serf_lan` - The Serf LAN port. Default 8301. * `serf_wan` - The Serf WAN port. Default 8302. From f1f8c88228d6d3997dc4d01d49157eb5a750dc0f Mon Sep 17 00:00:00 2001 From: Atin Malaviya Date: Tue, 18 Nov 2014 11:03:36 -0500 Subject: [PATCH 2/3] Moved TLS Config stuff to tlsutil package --- command/agent/config.go | 74 ----------- command/agent/http.go | 17 ++- command/agent/http_test.go | 10 +- command/util_test.go | 11 +- consul/client.go | 12 +- consul/config.go | 166 ----------------------- consul/pool.go | 3 +- consul/raft_rpc.go | 3 +- consul/server.go | 14 +- tlsutil/config.go | 206 +++++++++++++++++++++++++++++ {consul => tlsutil}/config_test.go | 6 +- 11 files changed, 267 insertions(+), 255 deletions(-) create mode 100644 tlsutil/config.go rename {consul => tlsutil}/config_test.go (98%) diff --git a/command/agent/config.go b/command/agent/config.go index de9cb13d76..be5a3d4d0b 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -1,13 +1,10 @@ package agent import ( - "crypto/tls" - "crypto/x509" "encoding/base64" "encoding/json" "fmt" "io" - "io/ioutil" "net" "os" "path/filepath" @@ -391,77 +388,6 @@ func (c *Config) ClientListenerAddr(override string, port int) (string, error) { return addr.String(), nil } -// AppendCA opens and parses the CA file and adds the certificates to -// the provided CertPool. -func (c *Config) AppendCA(pool *x509.CertPool) error { - if c.CAFile == "" { - return nil - } - - // Read the file - data, err := ioutil.ReadFile(c.CAFile) - if err != nil { - return fmt.Errorf("Failed to read CA file: %v", err) - } - - if !pool.AppendCertsFromPEM(data) { - return fmt.Errorf("Failed to parse any CA certificates") - } - - return nil -} - -// KeyPair is used to open and parse a certificate and key file -func (c *Config) KeyPair() (*tls.Certificate, error) { - if c.CertFile == "" || c.KeyFile == "" { - return nil, nil - } - cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) - if err != nil { - return nil, fmt.Errorf("Failed to load cert/key pair: %v", err) - } - return &cert, err -} - -// IncomingTLSConfig generates a TLS configuration for incoming requests -func (c *Config) IncomingTLSConfig() (*tls.Config, error) { - // Create the tlsConfig - tlsConfig := &tls.Config{ - ServerName: c.ServerName, - ClientCAs: x509.NewCertPool(), - ClientAuth: tls.NoClientCert, - } - if tlsConfig.ServerName == "" { - tlsConfig.ServerName = c.NodeName - } - - // Parse the CA cert if any - err := c.AppendCA(tlsConfig.ClientCAs) - if err != nil { - return nil, err - } - - // Add cert/key - cert, err := c.KeyPair() - if err != nil { - return nil, err - } else if cert != nil { - tlsConfig.Certificates = []tls.Certificate{*cert} - } - - // Check if we require verification - if c.VerifyIncoming { - tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert - if c.CAFile == "" { - return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!") - } - if cert == nil { - return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!") - } - } - return tlsConfig, nil -} - // DecodeConfig reads the configuration from the given reader in JSON // format and decodes it into a proper Config structure. func DecodeConfig(r io.Reader) (*Config, error) { diff --git a/command/agent/http.go b/command/agent/http.go index 5d07f23de6..79b847a34a 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -3,6 +3,7 @@ package agent import ( "crypto/tls" "encoding/json" + "fmt" "io" "log" "net" @@ -13,6 +14,7 @@ import ( "time" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/tlsutil" "github.com/mitchellh/mapstructure" ) @@ -42,7 +44,16 @@ func NewHTTPServers(agent *Agent, config *Config, logOutput io.Writer) ([]*HTTPS return nil, err } - tlsConfig, err = config.IncomingTLSConfig() + tlsConf := &tlsutil.Config{ + VerifyIncoming: config.VerifyIncoming, + VerifyOutgoing: config.VerifyOutgoing, + CAFile: config.CAFile, + CertFile: config.CertFile, + KeyFile: config.KeyFile, + NodeName: config.NodeName, + ServerName: config.ServerName} + + tlsConfig, err = tlsConf.IncomingTLSConfig() if err != nil { return nil, err } @@ -78,13 +89,13 @@ func NewHTTPServers(agent *Agent, config *Config, logOutput io.Writer) ([]*HTTPS if config.Ports.HTTP > 0 { httpAddr, err = config.ClientListener(config.Addresses.HTTP, config.Ports.HTTP) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to get ClientListener address:port: %v", err) } // Create non-TLS listener list, err = net.Listen("tcp", httpAddr.String()) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to get Listen on %s: %v", httpAddr.String(), err) } // Create the mux diff --git a/command/agent/http_test.go b/command/agent/http_test.go index d2ee63e7dd..c52b822d76 100644 --- a/command/agent/http_test.go +++ b/command/agent/http_test.go @@ -25,12 +25,18 @@ func makeHTTPServer(t *testing.T) (string, *HTTPServer) { if err := os.Mkdir(uiDir, 755); err != nil { t.Fatalf("err: %v", err) } + conf.Addresses.HTTP = "" + conf.Ports.HTTP = agent.config.Ports.HTTP + conf.Ports.HTTPS = -1 addr, _ := agent.config.ClientListener("", agent.config.Ports.HTTP) - server, err := NewHTTPServer(agent, uiDir, true, agent.logOutput, addr.String()) + servers, err := NewHTTPServers(agent, conf, agent.logOutput) if err != nil { t.Fatalf("err: %v", err) } - return dir, server + if servers == nil || len(servers) == 0 { + t.Fatalf(fmt.Sprintf("Could not create HTTP server to listen on: %s", addr.String())) + } + return dir, servers[0] } func encodeReq(obj interface{}) io.ReadCloser { diff --git a/command/util_test.go b/command/util_test.go index 0366f760bb..bb09664735 100644 --- a/command/util_test.go +++ b/command/util_test.go @@ -63,19 +63,25 @@ func testAgent(t *testing.T) *agentWrapper { rpc := agent.NewAgentRPC(a, l, mult, lw) + conf.Addresses.HTTP = "127.0.0.1" httpAddr := fmt.Sprintf("127.0.0.1:%d", conf.Ports.HTTP) - http, err := agent.NewHTTPServer(a, "", false, os.Stderr, httpAddr) + http, err := agent.NewHTTPServers(a, conf, os.Stderr) if err != nil { os.RemoveAll(dir) t.Fatalf(fmt.Sprintf("err: %v", err)) } + if http == nil || len(http) == 0 { + os.RemoveAll(dir) + t.Fatalf(fmt.Sprintf("Could not create HTTP server to listen on: %s", httpAddr)) + } + return &agentWrapper{ dir: dir, config: conf, agent: a, rpc: rpc, - http: http, + http: http[0], addr: l.Addr().String(), httpAddr: httpAddr, } @@ -92,6 +98,7 @@ func nextConfig() *agent.Config { conf.Server = true conf.Ports.HTTP = 10000 + 10*idx + conf.Ports.HTTPS = 10400 + 10*idx conf.Ports.RPC = 10100 + 10*idx conf.Ports.SerfLan = 10201 + 10*idx conf.Ports.SerfWan = 10202 + 10*idx diff --git a/consul/client.go b/consul/client.go index 050d147c2b..54bd4056a5 100644 --- a/consul/client.go +++ b/consul/client.go @@ -4,6 +4,7 @@ import ( "crypto/tls" "fmt" "github.com/hashicorp/consul/consul/structs" + "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/serf/serf" "log" "math/rand" @@ -93,7 +94,16 @@ func NewClient(config *Config) (*Client, error) { // Create the tlsConfig var tlsConfig *tls.Config var err error - if tlsConfig, err = config.OutgoingTLSConfig(); err != nil { + tlsConf := &tlsutil.Config{ + VerifyIncoming: config.VerifyIncoming, + VerifyOutgoing: config.VerifyOutgoing, + CAFile: config.CAFile, + CertFile: config.CertFile, + KeyFile: config.KeyFile, + NodeName: config.NodeName, + ServerName: config.ServerName} + + if tlsConfig, err = tlsConf.OutgoingTLSConfig(); err != nil { return nil, err } diff --git a/consul/config.go b/consul/config.go index 5404e71e6c..e623dcade5 100644 --- a/consul/config.go +++ b/consul/config.go @@ -1,11 +1,8 @@ package consul import ( - "crypto/tls" - "crypto/x509" "fmt" "io" - "io/ioutil" "net" "os" "time" @@ -199,169 +196,6 @@ func (c *Config) CheckACL() error { return nil } -// AppendCA opens and parses the CA file and adds the certificates to -// the provided CertPool. -func (c *Config) AppendCA(pool *x509.CertPool) error { - if c.CAFile == "" { - return nil - } - - // Read the file - data, err := ioutil.ReadFile(c.CAFile) - if err != nil { - return fmt.Errorf("Failed to read CA file: %v", err) - } - - if !pool.AppendCertsFromPEM(data) { - return fmt.Errorf("Failed to parse any CA certificates") - } - - return nil -} - -// KeyPair is used to open and parse a certificate and key file -func (c *Config) KeyPair() (*tls.Certificate, error) { - if c.CertFile == "" || c.KeyFile == "" { - return nil, nil - } - cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) - if err != nil { - return nil, fmt.Errorf("Failed to load cert/key pair: %v", err) - } - return &cert, err -} - -// OutgoingTLSConfig generates a TLS configuration for outgoing -// requests. It will return a nil config if this configuration should -// not use TLS for outgoing connections. -func (c *Config) OutgoingTLSConfig() (*tls.Config, error) { - if !c.VerifyOutgoing { - return nil, nil - } - // Create the tlsConfig - tlsConfig := &tls.Config{ - RootCAs: x509.NewCertPool(), - InsecureSkipVerify: true, - } - if c.ServerName != "" { - tlsConfig.ServerName = c.ServerName - tlsConfig.InsecureSkipVerify = false - } - - // Ensure we have a CA if VerifyOutgoing is set - if c.VerifyOutgoing && c.CAFile == "" { - return nil, fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!") - } - - // Parse the CA cert if any - err := c.AppendCA(tlsConfig.RootCAs) - if err != nil { - return nil, err - } - - // Add cert/key - cert, err := c.KeyPair() - if err != nil { - return nil, err - } else if cert != nil { - tlsConfig.Certificates = []tls.Certificate{*cert} - } - - return tlsConfig, nil -} - -// Wrap a net.Conn into a client tls connection, performing any -// additional verification as needed. -// -// As of go 1.3, crypto/tls only supports either doing no certificate -// verification, or doing full verification including of the peer's -// DNS name. For consul, we want to validate that the certificate is -// signed by a known CA, but because consul doesn't use DNS names for -// node names, we don't verify the certificate DNS names. Since go 1.3 -// no longer supports this mode of operation, we have to do it -// manually. -func wrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) { - var err error - var tlsConn *tls.Conn - - tlsConn = tls.Client(conn, tlsConfig) - - // If crypto/tls is doing verification, there's no need to do - // our own. - if tlsConfig.InsecureSkipVerify == false { - return tlsConn, nil - } - - if err = tlsConn.Handshake(); err != nil { - tlsConn.Close() - return nil, err - } - - // The following is lightly-modified from the doFullHandshake - // method in crypto/tls's handshake_client.go. - opts := x509.VerifyOptions{ - Roots: tlsConfig.RootCAs, - CurrentTime: time.Now(), - DNSName: "", - Intermediates: x509.NewCertPool(), - } - - certs := tlsConn.ConnectionState().PeerCertificates - for i, cert := range certs { - if i == 0 { - continue - } - opts.Intermediates.AddCert(cert) - } - - _, err = certs[0].Verify(opts) - if err != nil { - tlsConn.Close() - return nil, err - } - - return tlsConn, err -} - -// IncomingTLSConfig generates a TLS configuration for incoming requests -func (c *Config) IncomingTLSConfig() (*tls.Config, error) { - // Create the tlsConfig - tlsConfig := &tls.Config{ - ServerName: c.ServerName, - ClientCAs: x509.NewCertPool(), - ClientAuth: tls.NoClientCert, - } - if tlsConfig.ServerName == "" { - tlsConfig.ServerName = c.NodeName - } - - // Parse the CA cert if any - err := c.AppendCA(tlsConfig.ClientCAs) - if err != nil { - return nil, err - } - - // Add cert/key - cert, err := c.KeyPair() - if err != nil { - return nil, err - } else if cert != nil { - tlsConfig.Certificates = []tls.Certificate{*cert} - } - - // Check if we require verification - if c.VerifyIncoming { - tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert - if c.CAFile == "" { - return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!") - } - if cert == nil { - return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!") - } - } - return tlsConfig, nil -} - // DefaultConfig is used to return a sane default configuration func DefaultConfig() *Config { hostname, err := os.Hostname() diff --git a/consul/pool.go b/consul/pool.go index 91fe035f2c..4ab75fbc6a 100644 --- a/consul/pool.go +++ b/consul/pool.go @@ -11,6 +11,7 @@ import ( "sync/atomic" "time" + "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/go-msgpack/codec" "github.com/hashicorp/yamux" "github.com/inconshreveable/muxado" @@ -222,7 +223,7 @@ func (p *ConnPool) getNewConn(addr net.Addr, version int) (*Conn, error) { } // Wrap the connection in a TLS client - tlsConn, err := wrapTLSClient(conn, p.tlsConfig) + tlsConn, err := tlsutil.WrapTLSClient(conn, p.tlsConfig) if err != nil { conn.Close() return nil, err diff --git a/consul/raft_rpc.go b/consul/raft_rpc.go index 1024cd9878..e0ee4c68e6 100644 --- a/consul/raft_rpc.go +++ b/consul/raft_rpc.go @@ -3,6 +3,7 @@ package consul import ( "crypto/tls" "fmt" + "github.com/hashicorp/consul/tlsutil" "net" "sync" "time" @@ -94,7 +95,7 @@ func (l *RaftLayer) Dial(address string, timeout time.Duration) (net.Conn, error } // Wrap the connection in a TLS client - conn, err = wrapTLSClient(conn, l.tlsConfig) + conn, err = tlsutil.WrapTLSClient(conn, l.tlsConfig) if err != nil { return nil, err } diff --git a/consul/server.go b/consul/server.go index 789db198e8..4b6aa3a93c 100644 --- a/consul/server.go +++ b/consul/server.go @@ -16,6 +16,7 @@ import ( "time" "github.com/hashicorp/consul/acl" + "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/golang-lru" "github.com/hashicorp/raft" "github.com/hashicorp/raft-mdb" @@ -168,13 +169,22 @@ func NewServer(config *Config) (*Server, error) { } // Create the tlsConfig for outgoing connections - tlsConfig, err := config.OutgoingTLSConfig() + tlsConf := &tlsutil.Config{ + VerifyIncoming: config.VerifyIncoming, + VerifyOutgoing: config.VerifyOutgoing, + CAFile: config.CAFile, + CertFile: config.CertFile, + KeyFile: config.KeyFile, + NodeName: config.NodeName, + ServerName: config.ServerName} + + tlsConfig, err := tlsConf.OutgoingTLSConfig() if err != nil { return nil, err } // Get the incoming tls config - incomingTLS, err := config.IncomingTLSConfig() + incomingTLS, err := tlsConf.IncomingTLSConfig() if err != nil { return nil, err } diff --git a/tlsutil/config.go b/tlsutil/config.go new file mode 100644 index 0000000000..ab781de13a --- /dev/null +++ b/tlsutil/config.go @@ -0,0 +1,206 @@ +package tlsutil + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "net" + "time" +) + +// Config used to create tls.Config +type Config struct { + // VerifyIncoming is used to verify the authenticity of incoming connections. + // This means that TCP requests are forbidden, only allowing for TLS. TLS connections + // must match a provided certificate authority. This can be used to force client auth. + VerifyIncoming bool + + // VerifyOutgoing is used to verify the authenticity of outgoing connections. + // This means that TLS requests are used, and TCP requests are not made. TLS connections + // must match a provided certificate authority. This is used to verify authenticity of + // server nodes. + VerifyOutgoing bool + + // CAFile is a path to a certificate authority file. This is used with VerifyIncoming + // or VerifyOutgoing to verify the TLS connection. + CAFile string + + // CertFile is used to provide a TLS certificate that is used for serving TLS connections. + // Must be provided to serve TLS connections. + CertFile string + + // KeyFile is used to provide a TLS key that is used for serving TLS connections. + // Must be provided to serve TLS connections. + KeyFile string + + // Node name is the name we use to advertise. Defaults to hostname. + NodeName string + + // ServerName is used with the TLS certificate to ensure the name we + // provide matches the certificate + ServerName string +} + +// AppendCA opens and parses the CA file and adds the certificates to +// the provided CertPool. +func (c *Config) AppendCA(pool *x509.CertPool) error { + if c.CAFile == "" { + return nil + } + + // Read the file + data, err := ioutil.ReadFile(c.CAFile) + if err != nil { + return fmt.Errorf("Failed to read CA file: %v", err) + } + + if !pool.AppendCertsFromPEM(data) { + return fmt.Errorf("Failed to parse any CA certificates") + } + + return nil +} + +// KeyPair is used to open and parse a certificate and key file +func (c *Config) KeyPair() (*tls.Certificate, error) { + if c.CertFile == "" || c.KeyFile == "" { + return nil, nil + } + cert, err := tls.LoadX509KeyPair(c.CertFile, c.KeyFile) + if err != nil { + return nil, fmt.Errorf("Failed to load cert/key pair: %v", err) + } + return &cert, err +} + +// OutgoingTLSConfig generates a TLS configuration for outgoing +// requests. It will return a nil config if this configuration should +// not use TLS for outgoing connections. +func (c *Config) OutgoingTLSConfig() (*tls.Config, error) { + if !c.VerifyOutgoing { + return nil, nil + } + // Create the tlsConfig + tlsConfig := &tls.Config{ + RootCAs: x509.NewCertPool(), + InsecureSkipVerify: true, + } + if c.ServerName != "" { + tlsConfig.ServerName = c.ServerName + tlsConfig.InsecureSkipVerify = false + } + + // Ensure we have a CA if VerifyOutgoing is set + if c.VerifyOutgoing && c.CAFile == "" { + return nil, fmt.Errorf("VerifyOutgoing set, and no CA certificate provided!") + } + + // Parse the CA cert if any + err := c.AppendCA(tlsConfig.RootCAs) + if err != nil { + return nil, err + } + + // Add cert/key + cert, err := c.KeyPair() + if err != nil { + return nil, err + } else if cert != nil { + tlsConfig.Certificates = []tls.Certificate{*cert} + } + + return tlsConfig, nil +} + +// Wrap a net.Conn into a client tls connection, performing any +// additional verification as needed. +// +// As of go 1.3, crypto/tls only supports either doing no certificate +// verification, or doing full verification including of the peer's +// DNS name. For consul, we want to validate that the certificate is +// signed by a known CA, but because consul doesn't use DNS names for +// node names, we don't verify the certificate DNS names. Since go 1.3 +// no longer supports this mode of operation, we have to do it +// manually. +func WrapTLSClient(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) { + var err error + var tlsConn *tls.Conn + + tlsConn = tls.Client(conn, tlsConfig) + + // If crypto/tls is doing verification, there's no need to do + // our own. + if tlsConfig.InsecureSkipVerify == false { + return tlsConn, nil + } + + if err = tlsConn.Handshake(); err != nil { + tlsConn.Close() + return nil, err + } + + // The following is lightly-modified from the doFullHandshake + // method in crypto/tls's handshake_client.go. + opts := x509.VerifyOptions{ + Roots: tlsConfig.RootCAs, + CurrentTime: time.Now(), + DNSName: "", + Intermediates: x509.NewCertPool(), + } + + certs := tlsConn.ConnectionState().PeerCertificates + for i, cert := range certs { + if i == 0 { + continue + } + opts.Intermediates.AddCert(cert) + } + + _, err = certs[0].Verify(opts) + if err != nil { + tlsConn.Close() + return nil, err + } + + return tlsConn, err +} + +// IncomingTLSConfig generates a TLS configuration for incoming requests +func (c *Config) IncomingTLSConfig() (*tls.Config, error) { + // Create the tlsConfig + tlsConfig := &tls.Config{ + ServerName: c.ServerName, + ClientCAs: x509.NewCertPool(), + ClientAuth: tls.NoClientCert, + } + if tlsConfig.ServerName == "" { + tlsConfig.ServerName = c.NodeName + } + + // Parse the CA cert if any + err := c.AppendCA(tlsConfig.ClientCAs) + if err != nil { + return nil, err + } + + // Add cert/key + cert, err := c.KeyPair() + if err != nil { + return nil, err + } else if cert != nil { + tlsConfig.Certificates = []tls.Certificate{*cert} + } + + // Check if we require verification + if c.VerifyIncoming { + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + if c.CAFile == "" { + return nil, fmt.Errorf("VerifyIncoming set, and no CA certificate provided!") + } + if cert == nil { + return nil, fmt.Errorf("VerifyIncoming set, and no Cert/Key pair provided!") + } + } + return tlsConfig, nil +} diff --git a/consul/config_test.go b/tlsutil/config_test.go similarity index 98% rename from consul/config_test.go rename to tlsutil/config_test.go index 1007ffba77..150fddccd3 100644 --- a/consul/config_test.go +++ b/tlsutil/config_test.go @@ -1,4 +1,4 @@ -package consul +package tlsutil import ( "crypto/tls" @@ -204,7 +204,7 @@ func TestConfig_wrapTLS_OK(t *testing.T) { t.Fatalf("OutgoingTLSConfig err: %v", err) } - tlsClient, err := wrapTLSClient(client, clientConfig) + tlsClient, err := WrapTLSClient(client, clientConfig) if err != nil { t.Fatalf("wrapTLS err: %v", err) } else { @@ -237,7 +237,7 @@ func TestConfig_wrapTLS_BadCert(t *testing.T) { t.Fatalf("OutgoingTLSConfig err: %v", err) } - tlsClient, err := wrapTLSClient(client, clientTLSConfig) + tlsClient, err := WrapTLSClient(client, clientTLSConfig) if err == nil { t.Fatalf("wrapTLS no err") } From 61f1d24f393f632739e1a432ae81bb5b0d82dadc Mon Sep 17 00:00:00 2001 From: Atin Malaviya Date: Tue, 18 Nov 2014 17:56:48 -0500 Subject: [PATCH 3/3] consul.Config() helper to generate the tlsutil.Config{} struct, 30 second keepalive, use keepalive for HTTP and HTTPS --- command/agent/command.go | 7 +++---- command/agent/http.go | 6 ++++-- command/util_test.go | 2 +- consul/client.go | 12 +----------- consul/config.go | 14 ++++++++++++++ consul/server.go | 11 +---------- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/command/agent/command.go b/command/agent/command.go index de800e673f..9236c23f2b 100644 --- a/command/agent/command.go +++ b/command/agent/command.go @@ -466,10 +466,9 @@ func (c *Command) Run(args []string) int { if c.rpcServer != nil { defer c.rpcServer.Shutdown() } - if c.httpServers != nil { - for _, server := range c.httpServers { - defer server.Shutdown() - } + + for _, server := range c.httpServers { + defer server.Shutdown() } // Join startup nodes if specified diff --git a/command/agent/http.go b/command/agent/http.go index 79b847a34a..e5804ae1dd 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -93,11 +93,13 @@ func NewHTTPServers(agent *Agent, config *Config, logOutput io.Writer) ([]*HTTPS } // Create non-TLS listener - list, err = net.Listen("tcp", httpAddr.String()) + ln, err := net.Listen("tcp", httpAddr.String()) if err != nil { return nil, fmt.Errorf("Failed to get Listen on %s: %v", httpAddr.String(), err) } + list = tcpKeepAliveListener{ln.(*net.TCPListener)} + // Create the mux mux := http.NewServeMux() @@ -140,7 +142,7 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { return } tc.SetKeepAlive(true) - tc.SetKeepAlivePeriod(3 * time.Minute) + tc.SetKeepAlivePeriod(30 * time.Second) return tc, nil } diff --git a/command/util_test.go b/command/util_test.go index bb09664735..cd201139bc 100644 --- a/command/util_test.go +++ b/command/util_test.go @@ -98,7 +98,7 @@ func nextConfig() *agent.Config { conf.Server = true conf.Ports.HTTP = 10000 + 10*idx - conf.Ports.HTTPS = 10400 + 10*idx + conf.Ports.HTTPS = 10401 + 10*idx conf.Ports.RPC = 10100 + 10*idx conf.Ports.SerfLan = 10201 + 10*idx conf.Ports.SerfWan = 10202 + 10*idx diff --git a/consul/client.go b/consul/client.go index 54bd4056a5..28838bf795 100644 --- a/consul/client.go +++ b/consul/client.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "fmt" "github.com/hashicorp/consul/consul/structs" - "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/serf/serf" "log" "math/rand" @@ -94,16 +93,7 @@ func NewClient(config *Config) (*Client, error) { // Create the tlsConfig var tlsConfig *tls.Config var err error - tlsConf := &tlsutil.Config{ - VerifyIncoming: config.VerifyIncoming, - VerifyOutgoing: config.VerifyOutgoing, - CAFile: config.CAFile, - CertFile: config.CertFile, - KeyFile: config.KeyFile, - NodeName: config.NodeName, - ServerName: config.ServerName} - - if tlsConfig, err = tlsConf.OutgoingTLSConfig(); err != nil { + if tlsConfig, err = config.tlsConfig().OutgoingTLSConfig(); err != nil { return nil, err } diff --git a/consul/config.go b/consul/config.go index e623dcade5..9cb1944cbc 100644 --- a/consul/config.go +++ b/consul/config.go @@ -7,6 +7,7 @@ import ( "os" "time" + "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/memberlist" "github.com/hashicorp/raft" "github.com/hashicorp/serf/serf" @@ -234,3 +235,16 @@ func DefaultConfig() *Config { return conf } + +func (c *Config) tlsConfig() *tlsutil.Config { + tlsConf := &tlsutil.Config{ + VerifyIncoming: c.VerifyIncoming, + VerifyOutgoing: c.VerifyOutgoing, + CAFile: c.CAFile, + CertFile: c.CertFile, + KeyFile: c.KeyFile, + NodeName: c.NodeName, + ServerName: c.ServerName} + + return tlsConf +} diff --git a/consul/server.go b/consul/server.go index 4b6aa3a93c..2adbc7edbb 100644 --- a/consul/server.go +++ b/consul/server.go @@ -16,7 +16,6 @@ import ( "time" "github.com/hashicorp/consul/acl" - "github.com/hashicorp/consul/tlsutil" "github.com/hashicorp/golang-lru" "github.com/hashicorp/raft" "github.com/hashicorp/raft-mdb" @@ -169,15 +168,7 @@ func NewServer(config *Config) (*Server, error) { } // Create the tlsConfig for outgoing connections - tlsConf := &tlsutil.Config{ - VerifyIncoming: config.VerifyIncoming, - VerifyOutgoing: config.VerifyOutgoing, - CAFile: config.CAFile, - CertFile: config.CertFile, - KeyFile: config.KeyFile, - NodeName: config.NodeName, - ServerName: config.ServerName} - + tlsConf := config.tlsConfig() tlsConfig, err := tlsConf.OutgoingTLSConfig() if err != nil { return nil, err