Fix(kube) advanced deployment CE-83 (#4866)

* refactor(http/kube): convert compose format

* feat(kube/deploy): deploy to agent

* feat(kube/deploy): show more details about error

* refactor(kube): return string from deploy

* feat(kube/deploy): revert to use local kubectl

* Revert "feat(kube/deploy): revert to use local kubectl"

This reverts commit 7c4a1c70

* feat(kube/deploy): GH#4321 use the v2 version of agent api instead of v3

Co-authored-by: Chaim Lev-Ari <chiptus@gmail.com>
Co-authored-by: Simon Meng <simon.meng@portainer.io>

api/cmd/portainer/main.go

@@ -88,8 +88,8 @@ func initSwarmStackManager(assetsPath string, dataStorePath string, signatureSer
 	return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService)
 }
 
-func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer {
+func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer {
-	return exec.NewKubernetesDeployer(assetsPath)
+	return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath)
 }
 
 func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) {
@@ -397,7 +397,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager)
 
-	kubernetesDeployer := initKubernetesDeployer(*flags.Assets)
+	kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets)
 
 	if dataStore.IsNew() {
 		err = updateSettingsFromFlags(dataStore, flags)

api/exec/kubernetes_deploy.go

@@ -2,71 +2,188 @@ package exec
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os/exec"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/crypto"
 )
 
 // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment.
 type KubernetesDeployer struct {
-	binaryPath string
+	binaryPath           string
+	dataStore            portainer.DataStore
+	reverseTunnelService portainer.ReverseTunnelService
+	signatureService     portainer.DigitalSignatureService
 }
 
 // NewKubernetesDeployer initializes a new KubernetesDeployer service.
-func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer {
+func NewKubernetesDeployer(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer {
 	return &KubernetesDeployer{
-		binaryPath: binaryPath,
+		binaryPath:           binaryPath,
+		dataStore:            datastore,
+		reverseTunnelService: reverseTunnelService,
+		signatureService:     signatureService,
 	}
 }
 
 // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint.
-// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest.
 // Otherwise it will use kubectl to deploy the manifest.
-func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) {
-	if composeFormat {
+	if endpoint.Type == portainer.KubernetesLocalEnvironment {
-		convertedData, err := deployer.convertComposeData(data)
+		token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-		data = string(convertedData)
+
+		command := path.Join(deployer.binaryPath, "kubectl")
+		if runtime.GOOS == "windows" {
+			command = path.Join(deployer.binaryPath, "kubectl.exe")
+		}
+
+		args := make([]string, 0)
+		args = append(args, "--server", endpoint.URL)
+		args = append(args, "--insecure-skip-tls-verify")
+		args = append(args, "--token", string(token))
+		args = append(args, "--namespace", namespace)
+		args = append(args, "apply", "-f", "-")
+
+		var stderr bytes.Buffer
+		cmd := exec.Command(command, args...)
+		cmd.Stderr = &stderr
+		cmd.Stdin = strings.NewReader(stackConfig)
+
+		output, err := cmd.Output()
+		if err != nil {
+			return "", errors.New(stderr.String())
+		}
+
+		return string(output), nil
 	}
 
-	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
+	// agent
+
+	endpointURL := endpoint.URL
+	if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment {
+		tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID)
+		if tunnel.Status == portainer.EdgeAgentIdle {
+
+			err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID)
+			if err != nil {
+				return "", err
+			}
+
+			settings, err := deployer.dataStore.Settings().Settings()
+			if err != nil {
+				return "", err
+			}
+
+			waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second
+			time.Sleep(waitForAgentToConnect * 2)
+		}
+
+		endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port)
+	}
+
+	transport := &http.Transport{}
+
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return "", err
+		}
+		transport.TLSClientConfig = tlsConfig
+	}
+
+	httpCli := &http.Client{
+		Transport: transport,
+	}
+
+	if !strings.HasPrefix(endpointURL, "http") {
+		endpointURL = fmt.Sprintf("https://%s", endpointURL)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL))
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	command := path.Join(deployer.binaryPath, "kubectl")
+	reqPayload, err := json.Marshal(
-	if runtime.GOOS == "windows" {
+		struct {
-		command = path.Join(deployer.binaryPath, "kubectl.exe")
+			StackConfig string
-	}
+			Namespace   string
-
+		}{
-	args := make([]string, 0)
+			StackConfig: stackConfig,
-	args = append(args, "--server", endpoint.URL)
+			Namespace:   namespace,
-	args = append(args, "--insecure-skip-tls-verify")
+		})
-	args = append(args, "--token", string(token))
-	args = append(args, "--namespace", namespace)
-	args = append(args, "apply", "-f", "-")
-
-	var stderr bytes.Buffer
-	cmd := exec.Command(command, args...)
-	cmd.Stderr = &stderr
-	cmd.Stdin = strings.NewReader(data)
-
-	output, err := cmd.Output()
 	if err != nil {
-		return nil, errors.New(stderr.String())
+		return "", err
 	}
 
-	return output, nil
+	req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload))
+	if err != nil {
+		return "", err
+	}
+
+	signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey())
+	req.Header.Set(portainer.PortainerAgentSignatureHeader, signature)
+
+	resp, err := httpCli.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		var errorResponseData struct {
+			Message string
+			Details string
+		}
+		err = json.NewDecoder(resp.Body).Decode(&errorResponseData)
+		if err != nil {
+			output, parseStringErr := ioutil.ReadAll(resp.Body)
+			if parseStringErr != nil {
+				return "", parseStringErr
+			}
+
+			return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err)
+
+		}
+
+		return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details)
+	}
+
+	var responseData struct{ Output string }
+	err = json.NewDecoder(resp.Body).Decode(&responseData)
+	if err != nil {
+		parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body)
+		if parseStringErr != nil {
+			return "", parseStringErr
+		}
+
+		return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err)
+	}
+
+	return responseData.Output, nil
+
 }
 
-func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) {
+// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest.
+func (deployer *KubernetesDeployer) ConvertCompose(data string) ([]byte, error) {
 	command := path.Join(deployer.binaryPath, "kompose")
 	if runtime.GOOS == "windows" {
 		command = path.Join(deployer.binaryPath, "kompose.exe")

api/http/handler/stacks/create_kubernetes_stack.go

@@ -10,6 +10,7 @@ import (
 	"github.com/portainer/libhttp/request"
 	"github.com/portainer/libhttp/response"
 	portainer "github.com/portainer/portainer/api"
+	endpointutils "github.com/portainer/portainer/api/internal/endpoint"
 )
 
 type kubernetesStackPayload struct {
@@ -33,6 +34,10 @@ type createKubernetesStackResponse struct {
 }
 
 func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError {
+	if !endpointutils.IsKubernetesEndpoint(endpoint) {
+		return &httperror.HandlerError{http.StatusBadRequest, "Endpoint type does not match", errors.New("Endpoint type does not match")}
+	}
+
 	var payload kubernetesStackPayload
 	err := request.DecodeAndValidateJSONPayload(r, &payload)
 	if err != nil {
@@ -45,15 +50,24 @@ func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Req
 	}
 
 	resp := &createKubernetesStackResponse{
-		Output: string(output),
+		Output: output,
 	}
 
 	return response.JSON(w, resp)
 }
 
-func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) {
+func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string) (string, error) {
 	handler.stackCreationMutex.Lock()
 	defer handler.stackCreationMutex.Unlock()
 
-	return handler.KubernetesDeployer.Deploy(endpoint, data, composeFormat, namespace)
+	if composeFormat {
+		convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(stackConfig)
+		if err != nil {
+			return "", err
+		}
+		stackConfig = string(convertedConfig)
+	}
+
+	return handler.KubernetesDeployer.Deploy(endpoint, stackConfig, namespace)
+
 }

api/internal/endpoint/endpoint.go

@@ -0,0 +1,17 @@
+package endpoint
+
+import portainer "github.com/portainer/portainer/api"
+
+// IsKubernetesEndpoint returns true if this is a kubernetes endpoint
+func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool {
+	return endpoint.Type == portainer.KubernetesLocalEnvironment ||
+		endpoint.Type == portainer.AgentOnKubernetesEnvironment ||
+		endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment
+}
+
+// IsDocketEndpoint returns true if this is a docker endpoint
+func IsDocketEndpoint(endpoint *portainer.Endpoint) bool {
+	return endpoint.Type == portainer.DockerEnvironment ||
+		endpoint.Type == portainer.AgentOnDockerEnvironment ||
+		endpoint.Type == portainer.EdgeAgentOnDockerEnvironment
+}

api/portainer.go

@@ -1158,7 +1158,8 @@ type (
 
 	// KubernetesDeployer represents a service to deploy a manifest inside a Kubernetes endpoint
 	KubernetesDeployer interface {
-		Deploy(endpoint *Endpoint, data string, composeFormat bool, namespace string) ([]byte, error)
+		Deploy(endpoint *Endpoint, data string, namespace string) (string, error)
+		ConvertCompose(data string) ([]byte, error)
 	}
 
 	// KubernetesSnapshotter represents a service used to create Kubernetes endpoint snapshots