From defd929366afb9ead8ac5fdc835aa96e2ecace8f Mon Sep 17 00:00:00 2001 From: cong meng Date: Wed, 9 Jun 2021 11:55:17 +1200 Subject: [PATCH] Fix(kube) advanced deployment CE-83 (#4866) * refactor(http/kube): convert compose format * feat(kube/deploy): deploy to agent * feat(kube/deploy): show more details about error * refactor(kube): return string from deploy * feat(kube/deploy): revert to use local kubectl * Revert "feat(kube/deploy): revert to use local kubectl" This reverts commit 7c4a1c70 * feat(kube/deploy): GH#4321 use the v2 version of agent api instead of v3 Co-authored-by: Chaim Lev-Ari Co-authored-by: Simon Meng --- api/cmd/portainer/main.go | 6 +- api/exec/kubernetes_deploy.go | 181 ++++++++++++++---- .../handler/stacks/create_kubernetes_stack.go | 20 +- api/internal/endpoint/endpoint.go | 17 ++ api/portainer.go | 3 +- 5 files changed, 188 insertions(+), 39 deletions(-) create mode 100644 api/internal/endpoint/endpoint.go diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 89d1cee08..8945fea5f 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -88,8 +88,8 @@ func initSwarmStackManager(assetsPath string, dataStorePath string, signatureSer return exec.NewSwarmStackManager(assetsPath, dataStorePath, signatureService, fileService, reverseTunnelService) } -func initKubernetesDeployer(assetsPath string) portainer.KubernetesDeployer { - return exec.NewKubernetesDeployer(assetsPath) +func initKubernetesDeployer(dataStore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, assetsPath string) portainer.KubernetesDeployer { + return exec.NewKubernetesDeployer(dataStore, reverseTunnelService, signatureService, assetsPath) } func initJWTService(dataStore portainer.DataStore) (portainer.JWTService, error) { @@ -397,7 +397,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { composeStackManager := initComposeStackManager(*flags.Assets, *flags.Data, reverseTunnelService, proxyManager) - kubernetesDeployer := initKubernetesDeployer(*flags.Assets) + kubernetesDeployer := initKubernetesDeployer(dataStore, reverseTunnelService, digitalSignatureService, *flags.Assets) if dataStore.IsNew() { err = updateSettingsFromFlags(dataStore, flags) diff --git a/api/exec/kubernetes_deploy.go b/api/exec/kubernetes_deploy.go index 0330237e0..9d3a07d1d 100644 --- a/api/exec/kubernetes_deploy.go +++ b/api/exec/kubernetes_deploy.go @@ -2,71 +2,188 @@ package exec import ( "bytes" + "encoding/json" "errors" + "fmt" "io/ioutil" + "net/http" + "net/url" "os/exec" "path" "runtime" "strings" + "time" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/crypto" ) // KubernetesDeployer represents a service to deploy resources inside a Kubernetes environment. type KubernetesDeployer struct { - binaryPath string + binaryPath string + dataStore portainer.DataStore + reverseTunnelService portainer.ReverseTunnelService + signatureService portainer.DigitalSignatureService } // NewKubernetesDeployer initializes a new KubernetesDeployer service. -func NewKubernetesDeployer(binaryPath string) *KubernetesDeployer { +func NewKubernetesDeployer(datastore portainer.DataStore, reverseTunnelService portainer.ReverseTunnelService, signatureService portainer.DigitalSignatureService, binaryPath string) *KubernetesDeployer { return &KubernetesDeployer{ - binaryPath: binaryPath, + binaryPath: binaryPath, + dataStore: datastore, + reverseTunnelService: reverseTunnelService, + signatureService: signatureService, } } // Deploy will deploy a Kubernetes manifest inside a specific namespace in a Kubernetes endpoint. -// If composeFormat is set to true, it will leverage the kompose binary to deploy a compose compliant manifest. // Otherwise it will use kubectl to deploy the manifest. -func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) { - if composeFormat { - convertedData, err := deployer.convertComposeData(data) +func (deployer *KubernetesDeployer) Deploy(endpoint *portainer.Endpoint, stackConfig string, namespace string) (string, error) { + if endpoint.Type == portainer.KubernetesLocalEnvironment { + token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") if err != nil { - return nil, err + return "", err } - data = string(convertedData) + + command := path.Join(deployer.binaryPath, "kubectl") + if runtime.GOOS == "windows" { + command = path.Join(deployer.binaryPath, "kubectl.exe") + } + + args := make([]string, 0) + args = append(args, "--server", endpoint.URL) + args = append(args, "--insecure-skip-tls-verify") + args = append(args, "--token", string(token)) + args = append(args, "--namespace", namespace) + args = append(args, "apply", "-f", "-") + + var stderr bytes.Buffer + cmd := exec.Command(command, args...) + cmd.Stderr = &stderr + cmd.Stdin = strings.NewReader(stackConfig) + + output, err := cmd.Output() + if err != nil { + return "", errors.New(stderr.String()) + } + + return string(output), nil } - token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + // agent + + endpointURL := endpoint.URL + if endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment { + tunnel := deployer.reverseTunnelService.GetTunnelDetails(endpoint.ID) + if tunnel.Status == portainer.EdgeAgentIdle { + + err := deployer.reverseTunnelService.SetTunnelStatusToRequired(endpoint.ID) + if err != nil { + return "", err + } + + settings, err := deployer.dataStore.Settings().Settings() + if err != nil { + return "", err + } + + waitForAgentToConnect := time.Duration(settings.EdgeAgentCheckinInterval) * time.Second + time.Sleep(waitForAgentToConnect * 2) + } + + endpointURL = fmt.Sprintf("http://127.0.0.1:%d", tunnel.Port) + } + + transport := &http.Transport{} + + if endpoint.TLSConfig.TLS { + tlsConfig, err := crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify) + if err != nil { + return "", err + } + transport.TLSClientConfig = tlsConfig + } + + httpCli := &http.Client{ + Transport: transport, + } + + if !strings.HasPrefix(endpointURL, "http") { + endpointURL = fmt.Sprintf("https://%s", endpointURL) + } + + url, err := url.Parse(fmt.Sprintf("%s/v2/kubernetes/stack", endpointURL)) if err != nil { - return nil, err + return "", err } - command := path.Join(deployer.binaryPath, "kubectl") - if runtime.GOOS == "windows" { - command = path.Join(deployer.binaryPath, "kubectl.exe") - } - - args := make([]string, 0) - args = append(args, "--server", endpoint.URL) - args = append(args, "--insecure-skip-tls-verify") - args = append(args, "--token", string(token)) - args = append(args, "--namespace", namespace) - args = append(args, "apply", "-f", "-") - - var stderr bytes.Buffer - cmd := exec.Command(command, args...) - cmd.Stderr = &stderr - cmd.Stdin = strings.NewReader(data) - - output, err := cmd.Output() + reqPayload, err := json.Marshal( + struct { + StackConfig string + Namespace string + }{ + StackConfig: stackConfig, + Namespace: namespace, + }) if err != nil { - return nil, errors.New(stderr.String()) + return "", err } - return output, nil + req, err := http.NewRequest(http.MethodPost, url.String(), bytes.NewReader(reqPayload)) + if err != nil { + return "", err + } + + signature, err := deployer.signatureService.CreateSignature(portainer.PortainerAgentSignatureMessage) + if err != nil { + return "", err + } + + req.Header.Set(portainer.PortainerAgentPublicKeyHeader, deployer.signatureService.EncodedPublicKey()) + req.Header.Set(portainer.PortainerAgentSignatureHeader, signature) + + resp, err := httpCli.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + var errorResponseData struct { + Message string + Details string + } + err = json.NewDecoder(resp.Body).Decode(&errorResponseData) + if err != nil { + output, parseStringErr := ioutil.ReadAll(resp.Body) + if parseStringErr != nil { + return "", parseStringErr + } + + return "", fmt.Errorf("Failed parsing, body: %s, error: %w", output, err) + + } + + return "", fmt.Errorf("Deployment to agent failed: %s", errorResponseData.Details) + } + + var responseData struct{ Output string } + err = json.NewDecoder(resp.Body).Decode(&responseData) + if err != nil { + parsedOutput, parseStringErr := ioutil.ReadAll(resp.Body) + if parseStringErr != nil { + return "", parseStringErr + } + + return "", fmt.Errorf("Failed decoding, body: %s, err: %w", parsedOutput, err) + } + + return responseData.Output, nil + } -func (deployer *KubernetesDeployer) convertComposeData(data string) ([]byte, error) { +// ConvertCompose leverages the kompose binary to deploy a compose compliant manifest. +func (deployer *KubernetesDeployer) ConvertCompose(data string) ([]byte, error) { command := path.Join(deployer.binaryPath, "kompose") if runtime.GOOS == "windows" { command = path.Join(deployer.binaryPath, "kompose.exe") diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index 61ca665a5..7f1904e71 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -10,6 +10,7 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + endpointutils "github.com/portainer/portainer/api/internal/endpoint" ) type kubernetesStackPayload struct { @@ -33,6 +34,10 @@ type createKubernetesStackResponse struct { } func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Request, endpoint *portainer.Endpoint) *httperror.HandlerError { + if !endpointutils.IsKubernetesEndpoint(endpoint) { + return &httperror.HandlerError{http.StatusBadRequest, "Endpoint type does not match", errors.New("Endpoint type does not match")} + } + var payload kubernetesStackPayload err := request.DecodeAndValidateJSONPayload(r, &payload) if err != nil { @@ -45,15 +50,24 @@ func (handler *Handler) createKubernetesStack(w http.ResponseWriter, r *http.Req } resp := &createKubernetesStackResponse{ - Output: string(output), + Output: output, } return response.JSON(w, resp) } -func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) { +func (handler *Handler) deployKubernetesStack(endpoint *portainer.Endpoint, stackConfig string, composeFormat bool, namespace string) (string, error) { handler.stackCreationMutex.Lock() defer handler.stackCreationMutex.Unlock() - return handler.KubernetesDeployer.Deploy(endpoint, data, composeFormat, namespace) + if composeFormat { + convertedConfig, err := handler.KubernetesDeployer.ConvertCompose(stackConfig) + if err != nil { + return "", err + } + stackConfig = string(convertedConfig) + } + + return handler.KubernetesDeployer.Deploy(endpoint, stackConfig, namespace) + } diff --git a/api/internal/endpoint/endpoint.go b/api/internal/endpoint/endpoint.go new file mode 100644 index 000000000..378ca70e5 --- /dev/null +++ b/api/internal/endpoint/endpoint.go @@ -0,0 +1,17 @@ +package endpoint + +import portainer "github.com/portainer/portainer/api" + +// IsKubernetesEndpoint returns true if this is a kubernetes endpoint +func IsKubernetesEndpoint(endpoint *portainer.Endpoint) bool { + return endpoint.Type == portainer.KubernetesLocalEnvironment || + endpoint.Type == portainer.AgentOnKubernetesEnvironment || + endpoint.Type == portainer.EdgeAgentOnKubernetesEnvironment +} + +// IsDocketEndpoint returns true if this is a docker endpoint +func IsDocketEndpoint(endpoint *portainer.Endpoint) bool { + return endpoint.Type == portainer.DockerEnvironment || + endpoint.Type == portainer.AgentOnDockerEnvironment || + endpoint.Type == portainer.EdgeAgentOnDockerEnvironment +} diff --git a/api/portainer.go b/api/portainer.go index 2292aab1e..1b11b7783 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -1158,7 +1158,8 @@ type ( // KubernetesDeployer represents a service to deploy a manifest inside a Kubernetes endpoint KubernetesDeployer interface { - Deploy(endpoint *Endpoint, data string, composeFormat bool, namespace string) ([]byte, error) + Deploy(endpoint *Endpoint, data string, namespace string) (string, error) + ConvertCompose(data string) ([]byte, error) } // KubernetesSnapshotter represents a service used to create Kubernetes endpoint snapshots