fix(gitops): only attempt to redeploy when the environment appears to be online EE-6182 (#10464)

api/stacks/deployments/deploy.go

@@ -1,10 +1,13 @@
 package deployments
 
 import (
+	"crypto/tls"
 	"fmt"
 	"time"
 
 	portainer "github.com/portainer/portainer/api"
+	"github.com/portainer/portainer/api/agent"
+	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/git/update"
 	"github.com/portainer/portainer/api/http/security"
@@ -70,6 +73,10 @@ func RedeployWhenChanged(stackID portainer.StackID, deployer StackDeployer, data
 		return &StackAuthorMissingErr{int(stack.ID), author}
 	}
 
+	if !isEnvironmentOnline(endpoint) {
+		return nil
+	}
+
 	var gitCommitChangedOrForceUpdate bool
 	if !stack.FromAppTemplate {
 		updated, newHash, err := update.UpdateGitObject(gitService, fmt.Sprintf("stack:%d", stackID), stack.GitConfig, false, false, stack.ProjectPath)
@@ -162,3 +169,17 @@ func getUserRegistries(datastore dataservices.DataStore, user *portainer.User, e
 
 	return filteredRegistries, nil
 }
+
+func isEnvironmentOnline(endpoint *portainer.Endpoint) bool {
+	var err error
+	var tlsConfig *tls.Config
+	if endpoint.TLSConfig.TLS {
+		tlsConfig, err = crypto.CreateTLSConfigurationFromDisk(endpoint.TLSConfig.TLSCACertPath, endpoint.TLSConfig.TLSCertPath, endpoint.TLSConfig.TLSKeyPath, endpoint.TLSConfig.TLSSkipVerify)
+		if err != nil {
+			return false
+		}
+	}
+
+	_, _, err = agent.GetAgentVersionAndPlatform(endpoint.URL, tlsConfig)
+	return err == nil
+}

api/stacks/deployments/deploy_test.go

@@ -1,7 +1,11 @@
 package deployments
 
 import (
+	"context"
+	"crypto/tls"
 	"errors"
+	"net/http"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -9,10 +13,66 @@ import (
 	"github.com/portainer/portainer/api/datastore"
 	gittypes "github.com/portainer/portainer/api/git/types"
 	"github.com/portainer/portainer/api/internal/testhelpers"
+	"github.com/portainer/portainer/pkg/libhttp/response"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+const localhostCert = `-----BEGIN CERTIFICATE-----
+MIIEOjCCAiKgAwIBAgIRALg8rJET2/9LjKSxHj0dQhYwDQYJKoZIhvcNAQELBQAw
+FzEVMBMGA1UEAxMMUG9ydGFpbmVyIENBMB4XDTIzMTAxMTE5NDcxMVoXDTI1MDQx
+MTE5NTM0MVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAx4nNGiwcCqUCxZyVLIHqvjTy20ZtZDVCedssTv1W5tmz
+YqOIYGaW3CqzlRn6vBHu9bMHXef4+XfS0igKBn76MAKn5IcTccIWIal+5jq48pI3
+c2FzQ3qNujX2zqZPjAjhJnVeVCP3kJu4wUtuubswLPBVLdktGa6EkL+8nu6o0Phw
+6scV6s3gUmQk5/lpH4FIff8M7NAdTOxiFImQ1M0vplKtaEeiCnskpgyI8CbZl7X0
+38Pu178W3+LqB7N4iMy2gKnBwjsXzw/+1dfUGkKjYdDBD+kNEKrQ4dwkjkrkQVdt
+Z+GN26NvXHoeeyX/MLnVgdLbiIjvsf0DDIhabKqTcwIDAQABo4GDMIGAMA4GA1Ud
+DwEB/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0O
+BBYEFPCefmK5Szzlfs8FRCa5+kRCIEWuMB8GA1UdIwQYMBaAFKZZ074SR/ajD3zE
+gxpLGRvFT3XAMA8GA1UdEQQIMAaHBH8AAAEwDQYJKoZIhvcNAQELBQADggIBABcQ
+/WPSUpuQvrcVBsmIlOMz74cDZYuIDls/mAcB/yP3mm+oWlO0qvH/F/BMs1P/bkgj
+fByQZq8Zmi6/TEZNlGvW7KGx077VxDKi8jd1jL3gLDPmkFjYuGeIWQusgxBu1y3m
+0WoTTqnkoism1mzV/dgNwrm3YQIV4H/fi9EEdQSm0UFRTKSAGBkwS7N2pmNb5yQO
+U8glFpyznCv4evDJbs/JUUXKYExgFFhWUd25P7iBRLXg/BFfqdSTiUGUj/Msz0pO
+Evqmq78eIiXjyyKSxzve6/mEIeq6AE3AC9zH+fwTd6Mhp+T2P/S/iO4EU19IMR4m
+sbNBd6h/3GvRekO1KbqQ42awuMnxvWT0NVclSxiU1lMpAmRmk/w9z7wB3r4n7oh4
+iiOTl5VSw1UBkcLDOJw+HB/FU2PdVFfIJKRfjLCZOGrcJX9vEcz7dYGpB5HrdqOc
+/8q5j1g6f/pGE+20HITrtz6ChguETzqw5dLNeKeolC6bVH8yEtmpnP2n8VPnT9Di
+V+hnONcJ+wd/dkBqabGr7LPG24Kj1F2Zp3CDDvJA94FaEsgaLfSg3JD+43uRCOWM
+RuqU8bGuhQRqilR2dSIOrFaW2+MeUHsb24cUn/pkHqKpSg+RBEnf6QfGDlIgqYEl
+19f/HFVBc/a8lM/D81lMyDbjQ9zH4LDYj4ipBbkL
+-----END CERTIFICATE-----`
+
+const localhostKey = `-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAx4nNGiwcCqUCxZyVLIHqvjTy20ZtZDVCedssTv1W5tmzYqOI
+YGaW3CqzlRn6vBHu9bMHXef4+XfS0igKBn76MAKn5IcTccIWIal+5jq48pI3c2Fz
+Q3qNujX2zqZPjAjhJnVeVCP3kJu4wUtuubswLPBVLdktGa6EkL+8nu6o0Phw6scV
+6s3gUmQk5/lpH4FIff8M7NAdTOxiFImQ1M0vplKtaEeiCnskpgyI8CbZl7X038Pu
+178W3+LqB7N4iMy2gKnBwjsXzw/+1dfUGkKjYdDBD+kNEKrQ4dwkjkrkQVdtZ+GN
+26NvXHoeeyX/MLnVgdLbiIjvsf0DDIhabKqTcwIDAQABAoIBAQCqSP6BPG195A52
+iEeCISksw9ERsou+fflKNvIcQvV7swP0xOyooERUhhiVwQMKpx9QDUXXLRV8CHch
+JExR+OEYQdv4GhJM/b6XYafLYQfe80thKyQLzTXQWSdUeffe4OEMShODKOKoRUyp
+oO9Qj9/wKfX3V6S2iwnU4dxdofztv+YP9rYQyjnhKbv/9OfeCp2Pb9eFKKRsA+QQ
+xneDz1+wr8ToTuiTn8HBPNSeSAKvhzXuzyluI7VAetRloNgCtumrA9kpVbW2cDgE
+Gk0q3RY125ejFELQO/cOJFuBsqoJlvPxzg8/vHyfyF9hFMqbqvcUw2e1eqHpnJd5
+dP4+ZGYZAoGBAOOFuPXMLBts0rN9mfNbVfx36H+aOCL77SafZvWm0D+rH69QN3/q
+/ZSWQEjwH5Tzn1e+NVcl/Um2vL/dIyEGBklXQ7yAyJo25gpEOD/rt1U94HKzMOwy
+yKtsKghRAOx0piie7ORS6MGbEOQxU3/1Eg1uvd0qoSnALqJ/le75QpFXAoGBAOCD
+aZQTszzDddr1cFPzLyqjIGJWfPcDYSONXVcCeQmhvC7mkfw9SWdIfku7JbdNgFYq
+ZAAU0klsLX0lEe8f4A12FnHNylKoxmTWdE3wWPptejdA1KUgzt/2kNljgOMFuY0Q
+rlCEW/Jabrg5aFMwVVG8bHLZR0xalfniDvXLvnFFAoGACdztJLKiIto31BIYz2Th
+OF2WVZnA3ztej3MPioydsHThnb7zePcd4QgWZ1MJe3KIMMyNEWcTMNPcINEcSb0y
+HpHK3OwURiMlG8LTUWoNe4OALFi6QTL+YfgBZnTkflucLFyfVlKFxobLV6kPvpdI
+Hg7z6heD/wRWwTKYtFBX42cCgYBIeoQJ9rYlRqB0eEm0AEzYweLBfFRJVgD0/j0E
+ytqSPnFG3s6AFLTur9t9zUPmwhFNP9Aaqp4cb9zbiq0YejzVe6rRQHMxbiTmBslz
+I8VFyzPqRHahfE7sxGeMlm/UWlPFc34ipigcvA8EUBwaxv60LVUBWp2Gy7OhANZ9
+iTHI1QKBgQCdHFj9dnbpaEHA426CoaPsyj5cv2nBLRf8p1cs71sq+qQOGlGJfajm
+L9x22ol5c5rToZa1qKSnSdSDCud298MyRujMUy2UcUKHeNs3MK9AT41sDv266I7b
+vJUUCFYm8+9p6gTVOcoMit+eGSwa81PCPEs1TnU1PV/PaDFeUhn/mg==
+-----END RSA PRIVATE KEY-----`
+
 type noopDeployer struct{}
 
 // without unpacker
@@ -54,6 +114,42 @@ func (s *noopDeployer) StopRemoteSwarmStack(stack *portainer.Stack, endpoint *po
 	return nil
 }
 
+func agentServer(t *testing.T) string {
+	h := http.NewServeMux()
+
+	h.HandleFunc("/ping", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set(portainer.PortainerAgentHeader, "v2.19.0")
+		w.Header().Set(portainer.HTTPResponseAgentPlatform, strconv.Itoa(int(portainer.AgentPlatformDocker)))
+
+		response.Empty(w)
+	})
+
+	cert, err := tls.X509KeyPair([]byte(localhostCert), []byte(localhostKey))
+	require.NoError(t, err)
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+	}
+
+	l, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig)
+	require.NoError(t, err)
+
+	s := &http.Server{
+		Handler: h,
+	}
+
+	go func() {
+		err := s.Serve(l)
+		require.ErrorIs(t, err, http.ErrServerClosed)
+	}()
+
+	t.Cleanup(func() {
+		s.Shutdown(context.Background())
+	})
+
+	return "http://" + l.Addr().String()
+}
+
 func Test_redeployWhenChanged_FailsWhenCannotFindStack(t *testing.T) {
 	_, store := datastore.MustNewTestStore(t, true, true)
 
@@ -114,7 +210,12 @@ func Test_redeployWhenChanged_FailsWhenCannotClone(t *testing.T) {
 	assert.NoError(t, err, "error creating an admin")
 
 	err = store.Endpoint().Create(&portainer.Endpoint{
-		ID: 0,
+		ID:  0,
+		URL: agentServer(t),
+		TLSConfig: portainer.TLSConfiguration{
+			TLS:           true,
+			TLSSkipVerify: true,
+		},
 	})
 	assert.NoError(t, err, "error creating environment")