fix(api): refactor TLS support (#1909)

* refactor(api): refactor TLS support

* feat(api): migrate endpoint data

* refactor(api): remove unused code and rename functions

* refactor(app): remove console.log statement

api/bolt/migrate_dbversion10.go

@@ -0,0 +1,28 @@
+package bolt
+
+import "github.com/portainer/portainer"
+
+func (m *Migrator) updateEndpointsToVersion11() error {
+	legacyEndpoints, err := m.EndpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range legacyEndpoints {
+		if endpoint.Type == portainer.AgentOnDockerEnvironment {
+			endpoint.TLSConfig.TLS = true
+			endpoint.TLSConfig.TLSSkipVerify = true
+		} else {
+			if endpoint.TLSConfig.TLSSkipVerify && !endpoint.TLSConfig.TLS {
+				endpoint.TLSConfig.TLSSkipVerify = false
+			}
+		}
+
+		err = m.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

api/bolt/migrator.go

@@ -112,6 +112,14 @@ func (m *Migrator) Migrate() error {
 		}
 	}
 
+	// https://github.com/portainer/portainer/issues/1906
+	if m.CurrentDBVersion < 11 {
+		err := m.updateEndpointsToVersion11()
+		if err != nil {
+			return err
+		}
+	}
+
 	err := m.VersionService.StoreDBVersion(portainer.DBVersion)
 	if err != nil {
 		return err

api/cli/cli.go

@@ -33,11 +33,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		Addr:              kingpin.Flag("bind", "Address and port to serve Portainer").Default(defaultBindAddress).Short('p').String(),
 		Assets:            kingpin.Flag("assets", "Path to the assets").Default(defaultAssetsDirectory).Short('a').String(),
 		Data:              kingpin.Flag("data", "Path to the folder where the data is stored").Default(defaultDataDirectory).Short('d').String(),
-		Endpoint:          kingpin.Flag("host", "Dockerd endpoint").Short('H').String(),
+		EndpointURL:       kingpin.Flag("host", "Endpoint URL").Short('H').String(),
 		ExternalEndpoints: kingpin.Flag("external-endpoints", "Path to a file defining available endpoints").String(),
 		NoAuth:            kingpin.Flag("no-auth", "Disable authentication").Default(defaultNoAuth).Bool(),
 		NoAnalytics:       kingpin.Flag("no-analytics", "Disable Analytics in app").Default(defaultNoAnalytics).Bool(),
-		TLSVerify:         kingpin.Flag("tlsverify", "TLS support").Default(defaultTLSVerify).Bool(),
+		TLS:               kingpin.Flag("tlsverify", "TLS support").Default(defaultTLS).Bool(),
 		TLSSkipVerify:     kingpin.Flag("tlsskipverify", "Disable TLS server verification").Default(defaultTLSSkipVerify).Bool(),
 		TLSCacert:         kingpin.Flag("tlscacert", "Path to the CA").Default(defaultTLSCACertPath).String(),
 		TLSCert:           kingpin.Flag("tlscert", "Path to the TLS certificate file").Default(defaultTLSCertPath).String(),
@@ -69,11 +69,11 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 // ValidateFlags validates the values of the flags.
 func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 
-	if *flags.Endpoint != "" && *flags.ExternalEndpoints != "" {
+	if *flags.EndpointURL != "" && *flags.ExternalEndpoints != "" {
 		return errEndpointExcludeExternal
 	}
 
-	err := validateEndpoint(*flags.Endpoint)
+	err := validateEndpointURL(*flags.EndpointURL)
 	if err != nil {
 		return err
 	}
@@ -99,14 +99,14 @@ func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
 	return nil
 }
 
-func validateEndpoint(endpoint string) error {
+func validateEndpointURL(endpointURL string) error {
-	if endpoint != "" {
+	if endpointURL != "" {
-		if !strings.HasPrefix(endpoint, "unix://") && !strings.HasPrefix(endpoint, "tcp://") {
+		if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") {
 			return errInvalidEndpointProtocol
 		}
 
-		if strings.HasPrefix(endpoint, "unix://") {
+		if strings.HasPrefix(endpointURL, "unix://") {
-			socketPath := strings.TrimPrefix(endpoint, "unix://")
+			socketPath := strings.TrimPrefix(endpointURL, "unix://")
 			if _, err := os.Stat(socketPath); err != nil {
 				if os.IsNotExist(err) {
 					return errSocketNotFound

api/cli/defaults.go

@@ -8,7 +8,7 @@ const (
 	defaultAssetsDirectory = "./"
 	defaultNoAuth          = "false"
 	defaultNoAnalytics     = "false"
-	defaultTLSVerify       = "false"
+	defaultTLS             = "false"
 	defaultTLSSkipVerify   = "false"
 	defaultTLSCACertPath   = "/certs/ca.pem"
 	defaultTLSCertPath     = "/certs/cert.pem"

api/cli/defaults_windows.go

@@ -6,7 +6,7 @@ const (
 	defaultAssetsDirectory = "./"
 	defaultNoAuth          = "false"
 	defaultNoAnalytics     = "false"
-	defaultTLSVerify       = "false"
+	defaultTLS             = "false"
 	defaultTLSSkipVerify   = "false"
 	defaultTLSCACertPath   = "C:\\certs\\ca.pem"
 	defaultTLSCertPath     = "C:\\certs\\cert.pem"

api/cmd/portainer/main.go

@@ -1,6 +1,8 @@
 package main // import "github.com/portainer/portainer"
 
 import (
+	"strings"
+
 	"github.com/portainer/portainer"
 	"github.com/portainer/portainer/bolt"
 	"github.com/portainer/portainer/cli"
@@ -207,6 +209,93 @@ func initKeyPair(fileService portainer.FileService, signatureService portainer.D
 	return generateAndStoreKeyPair(fileService, signatureService)
 }
 
+func createTLSSecuredEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+	tlsConfiguration := portainer.TLSConfiguration{
+		TLS:           *flags.TLS,
+		TLSSkipVerify: *flags.TLSSkipVerify,
+	}
+
+	if *flags.TLS {
+		tlsConfiguration.TLSCACertPath = *flags.TLSCacert
+		tlsConfiguration.TLSCertPath = *flags.TLSCert
+		tlsConfiguration.TLSKeyPath = *flags.TLSKey
+	} else if !*flags.TLS && *flags.TLSSkipVerify {
+		tlsConfiguration.TLS = true
+	}
+
+	endpoint := &portainer.Endpoint{
+		Name:            "primary",
+		URL:             *flags.EndpointURL,
+		GroupID:         portainer.EndpointGroupID(1),
+		Type:            portainer.DockerEnvironment,
+		TLSConfig:       tlsConfiguration,
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+	}
+
+	if strings.HasPrefix(endpoint.URL, "tcp://") {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(tlsConfiguration.TLSCACertPath, tlsConfiguration.TLSCertPath, tlsConfiguration.TLSKeyPath, tlsConfiguration.TLSSkipVerify)
+		if err != nil {
+			return err
+		}
+
+		agentOnDockerEnvironment, err := client.ExecutePingOperation(endpoint.URL, tlsConfig)
+		if err != nil {
+			return err
+		}
+
+		if agentOnDockerEnvironment {
+			endpoint.Type = portainer.AgentOnDockerEnvironment
+		}
+	}
+
+	return endpointService.CreateEndpoint(endpoint)
+}
+
+func createUnsecuredEndpoint(endpointURL string, endpointService portainer.EndpointService) error {
+	if strings.HasPrefix(endpointURL, "tcp://") {
+		_, err := client.ExecutePingOperation(endpointURL, nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	endpoint := &portainer.Endpoint{
+		Name:            "primary",
+		URL:             endpointURL,
+		GroupID:         portainer.EndpointGroupID(1),
+		Type:            portainer.DockerEnvironment,
+		TLSConfig:       portainer.TLSConfiguration{},
+		AuthorizedUsers: []portainer.UserID{},
+		AuthorizedTeams: []portainer.TeamID{},
+		Extensions:      []portainer.EndpointExtension{},
+	}
+
+	return endpointService.CreateEndpoint(endpoint)
+}
+
+func initEndpoint(flags *portainer.CLIFlags, endpointService portainer.EndpointService) error {
+	if *flags.EndpointURL == "" {
+		return nil
+	}
+
+	endpoints, err := endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	if len(endpoints) > 0 {
+		log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
+		return nil
+	}
+
+	if *flags.TLS || *flags.TLSSkipVerify {
+		return createTLSSecuredEndpoint(flags, endpointService)
+	}
+	return createUnsecuredEndpoint(*flags.EndpointURL, endpointService)
+}
+
 func main() {
 	flags := initCLI()
 
@@ -249,45 +338,9 @@ func main() {
 
 	applicationStatus := initStatus(authorizeEndpointMgmt, flags)
 
-	if *flags.Endpoint != "" {
+	err = initEndpoint(flags, store.EndpointService)
-		endpoints, err := store.EndpointService.Endpoints()
+	if err != nil {
-		if err != nil {
+		log.Fatal(err)
-			log.Fatal(err)
-		}
-		if len(endpoints) == 0 {
-			endpoint := &portainer.Endpoint{
-				Name:    "primary",
-				URL:     *flags.Endpoint,
-				GroupID: portainer.EndpointGroupID(1),
-				Type:    portainer.DockerEnvironment,
-				TLSConfig: portainer.TLSConfiguration{
-					TLS:           *flags.TLSVerify,
-					TLSSkipVerify: *flags.TLSSkipVerify,
-					TLSCACertPath: *flags.TLSCacert,
-					TLSCertPath:   *flags.TLSCert,
-					TLSKeyPath:    *flags.TLSKey,
-				},
-				AuthorizedUsers: []portainer.UserID{},
-				AuthorizedTeams: []portainer.TeamID{},
-				Extensions:      []portainer.EndpointExtension{},
-			}
-
-			agentOnDockerEnvironment, err := client.ExecutePingOperationFromEndpoint(endpoint)
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			if agentOnDockerEnvironment {
-				endpoint.Type = portainer.AgentOnDockerEnvironment
-			}
-
-			err = store.EndpointService.CreateEndpoint(endpoint)
-			if err != nil {
-				log.Fatal(err)
-			}
-		} else {
-			log.Println("Instance already has defined endpoints. Skipping the endpoint defined via CLI.")
-		}
 	}
 
 	adminPasswordHash := ""

api/crypto/tls.go

@@ -4,11 +4,11 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"io/ioutil"
-
-	"github.com/portainer/portainer"
 )
 
-func CreateTLSConfig(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) {
+// CreateTLSConfigurationFromBytes initializes a tls.Config using a CA certificate, a certificate and a key
+// loaded from memory.
+func CreateTLSConfigurationFromBytes(caCert, cert, key []byte, skipClientVerification, skipServerVerification bool) (*tls.Config, error) {
 	config := &tls.Config{}
 	config.InsecureSkipVerify = skipServerVerification
 
@@ -29,32 +29,31 @@ func CreateTLSConfig(caCert, cert, key []byte, skipClientVerification, skipServe
 	return config, nil
 }
 
-// CreateTLSConfiguration initializes a tls.Config using a CA certificate, a certificate and a key
+// CreateTLSConfigurationFromDisk initializes a tls.Config using a CA certificate, a certificate and a key
-func CreateTLSConfiguration(config *portainer.TLSConfiguration) (*tls.Config, error) {
+// loaded from disk.
-	TLSConfig := &tls.Config{}
+func CreateTLSConfigurationFromDisk(caCertPath, certPath, keyPath string, skipServerVerification bool) (*tls.Config, error) {
+	config := &tls.Config{}
+	config.InsecureSkipVerify = skipServerVerification
 
-	if config.TLS && config.TLSCertPath != "" && config.TLSKeyPath != "" {
+	if certPath != "" && keyPath != "" {
-		cert, err := tls.LoadX509KeyPair(config.TLSCertPath, config.TLSKeyPath)
+		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
 		if err != nil {
 			return nil, err
 		}
 
-		TLSConfig.Certificates = []tls.Certificate{cert}
+		config.Certificates = []tls.Certificate{cert}
 	}
 
-	if config.TLS && !config.TLSSkipVerify {
+	if !skipServerVerification && caCertPath != "" {
-		caCert, err := ioutil.ReadFile(config.TLSCACertPath)
+		caCert, err := ioutil.ReadFile(caCertPath)
 		if err != nil {
 			return nil, err
 		}
 
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-
+		config.RootCAs = caCertPool
-		TLSConfig.RootCAs = caCertPool
 	}
 
-	TLSConfig.InsecureSkipVerify = config.TLSSkipVerify
+	return config, nil
-
-	return TLSConfig, nil
 }

api/exec/stack_manager.go

@@ -118,9 +118,7 @@ func prepareDockerCommandAndArgs(binaryPath, dataPath string, endpoint *portaine
 	args = append(args, "--config", dataPath)
 	args = append(args, "-H", endpoint.URL)
 
-	if !endpoint.TLSConfig.TLS && endpoint.TLSConfig.TLSSkipVerify {
+	if endpoint.TLSConfig.TLS {
-		args = append(args, "--tls")
-	} else if endpoint.TLSConfig.TLS {
 		args = append(args, "--tls")
 
 		if !endpoint.TLSConfig.TLSSkipVerify {

api/http/client/client.go

@@ -7,39 +7,8 @@ import (
 	"time"
 
 	"github.com/portainer/portainer"
-	"github.com/portainer/portainer/crypto"
 )
 
-// ExecutePingOperationFromEndpoint will send a SystemPing operation HTTP request to a Docker environment
-// using the specified endpoint configuration. It is used exclusively when
-// specifying an endpoint from the CLI via the -H flag.
-func ExecutePingOperationFromEndpoint(endpoint *portainer.Endpoint) (bool, error) {
-	if strings.HasPrefix(endpoint.URL, "unix://") {
-		return false, nil
-	}
-
-	transport := &http.Transport{}
-
-	scheme := "http"
-
-	if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
-		tlsConfig, err := crypto.CreateTLSConfiguration(&endpoint.TLSConfig)
-		if err != nil {
-			return false, err
-		}
-		scheme = "https"
-		transport.TLSClientConfig = tlsConfig
-	}
-
-	client := &http.Client{
-		Timeout:   time.Second * 3,
-		Transport: transport,
-	}
-
-	target := strings.Replace(endpoint.URL, "tcp://", scheme+"://", 1)
-	return pingOperation(client, target)
-}
-
 // ExecutePingOperation will send a SystemPing operation HTTP request to a Docker environment
 // using the specified host and optional TLS configuration.
 func ExecutePingOperation(host string, tlsConfig *tls.Config) (bool, error) {

api/http/handler/endpoint.go

@@ -121,7 +121,7 @@ func (handler *EndpointHandler) handleGetEndpoints(w http.ResponseWriter, r *htt
 }
 
 func (handler *EndpointHandler) createTLSSecuredEndpoint(payload *postEndpointPayload) (*portainer.Endpoint, error) {
-	tlsConfig, err := crypto.CreateTLSConfig(payload.caCert, payload.cert, payload.key, payload.skipTLSClientVerification, payload.skipTLSServerVerification)
+	tlsConfig, err := crypto.CreateTLSConfigurationFromBytes(payload.caCert, payload.cert, payload.key, payload.skipTLSClientVerification, payload.skipTLSServerVerification)
 	if err != nil {
 		return nil, err
 	}

api/http/handler/websocket.go

@@ -188,7 +188,7 @@ func createDial(endpoint *portainer.Endpoint) (net.Conn, error) {
 	}
 
 	if endpoint.TLSConfig.TLS {
-		tlsConfig, err := crypto.CreateTLSConfiguration(&endpoint.TLSConfig)
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
 		if err != nil {
 			return nil, err
 		}

api/http/proxy/factory.go

@@ -29,7 +29,7 @@ func (factory *proxyFactory) newDockerHTTPSProxy(u *url.URL, tlsConfig *portaine
 	u.Scheme = "https"
 
 	proxy := factory.createDockerReverseProxy(u, enableSignature)
-	config, err := crypto.CreateTLSConfiguration(tlsConfig)
+	config, err := crypto.CreateTLSConfigurationFromDisk(tlsConfig.TLSCACertPath, tlsConfig.TLSCertPath, tlsConfig.TLSKeyPath, tlsConfig.TLSSkipVerify)
 	if err != nil {
 		return nil, err
 	}

api/http/proxy/manager.go

@@ -60,7 +60,7 @@ func (manager *Manager) CreateAndRegisterProxy(endpoint *portainer.Endpoint) (ht
 	}
 
 	if endpointURL.Scheme == "tcp" {
-		if endpoint.TLSConfig.TLS || endpoint.TLSConfig.TLSSkipVerify {
+		if endpoint.TLSConfig.TLS {
 			proxy, err = manager.proxyFactory.newDockerHTTPSProxy(endpointURL, &endpoint.TLSConfig, enableSignature)
 			if err != nil {
 				return nil, err

api/ldap/ldap.go

@@ -55,7 +55,7 @@ func searchUser(username string, conn *ldap.Conn, settings []portainer.LDAPSearc
 func createConnection(settings *portainer.LDAPSettings) (*ldap.Conn, error) {
 
 	if settings.TLSConfig.TLS || settings.StartTLS {
-		config, err := crypto.CreateTLSConfiguration(&settings.TLSConfig)
+		config, err := crypto.CreateTLSConfigurationFromDisk(settings.TLSConfig.TLSCACertPath, settings.TLSConfig.TLSCertPath, settings.TLSConfig.TLSKeyPath, settings.TLSConfig.TLSSkipVerify)
 		if err != nil {
 			return nil, err
 		}

api/portainer.go

@@ -16,14 +16,14 @@ type (
 		AdminPasswordFile *string
 		Assets            *string
 		Data              *string
-		Endpoint          *string
+		EndpointURL       *string
 		ExternalEndpoints *string
 		Labels            *[]Pair
 		Logo              *string
 		NoAuth            *bool
 		NoAnalytics       *bool
 		Templates         *string
-		TLSVerify         *bool
+		TLS               *bool
 		TLSSkipVerify     *bool
 		TLSCacert         *string
 		TLSCert           *string
@@ -445,7 +445,7 @@ const (
 	// APIVersion is the version number of the Portainer API.
 	APIVersion = "1.17.0"
 	// DBVersion is the version number of the Portainer database.
-	DBVersion = 10
+	DBVersion = 11
 	// DefaultTemplatesURL represents the default URL for the templates definitions.
 	DefaultTemplatesURL = "https://raw.githubusercontent.com/portainer/templates/master/templates.json"
 	// PortainerAgentHeader represents the name of the header available in any agent response