fix(kube): 30 second delay to storage detection EE-4822 (#8360)

api/datastore/migrator/migrate_dbversion80.go

@@ -16,6 +16,10 @@ func (m *Migrator) migrateDBVersionToDB80() error {
 		return err
 	}
 
+	if err := m.updateExistingEndpointsToNotDetectStorageAPIForDB80(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -40,6 +44,27 @@ func (m *Migrator) updateExistingEndpointsToNotDetectMetricsAPIForDB80() error {
 	return nil
 }
 
+func (m *Migrator) updateExistingEndpointsToNotDetectStorageAPIForDB80() error {
+	log.Info().Msg("updating existing endpoints to not detect metrics API for existing endpoints (k8s)")
+
+	endpoints, err := m.endpointService.Endpoints()
+	if err != nil {
+		return err
+	}
+
+	for _, endpoint := range endpoints {
+		if endpointutils.IsKubernetesEndpoint(&endpoint) {
+			endpoint.Kubernetes.Flags.IsServerStorageDetected = true
+			err = m.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 func (m *Migrator) updateEdgeStackStatusForDB80() error {
 	log.Info().Msg("transfer type field to details field for edge stack status")
 

api/datastore/test_data/output_24_to_latest.json

@@ -63,7 +63,8 @@
           "UseServerMetrics": false
         },
         "Flags": {
-          "IsServerMetricsDetected": false
+          "IsServerMetricsDetected": false,
+          "IsServerStorageDetected": false
         },
         "Snapshots": []
       },

api/http/handler/endpoints/endpoint_inspect.go

@@ -61,6 +61,15 @@ func (handler *Handler) endpointInspect(w http.ResponseWriter, r *http.Request)
 		)
 	}
 
+	isServerStorageDetected := endpoint.Kubernetes.Flags.IsServerStorageDetected
+	if !isServerStorageDetected && handler.K8sClientFactory != nil {
+		endpointutils.InitialStorageDetection(
+			endpoint,
+			handler.DataStore.Endpoint(),
+			handler.K8sClientFactory,
+		)
+	}
+
 	return response.JSON(w, endpoint)
 }
 

api/internal/endpointutils/endpointutils.go

@@ -1,7 +1,9 @@
 package endpointutils
 
 import (
+	"fmt"
 	"strings"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/dataservices"
@@ -127,17 +129,21 @@ func InitialMetricsDetection(endpoint *portainer.Endpoint, endpointService datas
 	}
 }
 
-func InitialStorageDetection(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) {
+func storageDetect(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) error {
 	cli, err := factory.GetKubeClient(endpoint)
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to create Kubernetes client for initial storage detection")
-		return
+		return err
 	}
 
 	storage, err := cli.GetStorage()
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to fetch storage classes: leaving storage classes disabled")
-		return
+		return err
+	}
+	if len(storage) == 0 {
+		log.Info().Err(err).Msg("zero storage classes found: they may be still building, retrying in 30 seconds")
+		return fmt.Errorf("zero storage classes found: they may be still building, retrying in 30 seconds")
 	}
 	endpoint.Kubernetes.Configuration.StorageClasses = storage
 	err = endpointService.UpdateEndpoint(
@@ -146,6 +152,23 @@ func InitialStorageDetection(endpoint *portainer.Endpoint, endpointService datas
 	)
 	if err != nil {
 		log.Debug().Err(err).Msg("unable to enable storage class inside the database")
+		return err
+	}
+	return nil
+}
+
+func InitialStorageDetection(endpoint *portainer.Endpoint, endpointService dataservices.EndpointService, factory *cli.ClientFactory) {
+	log.Info().Msg("attempting to detect storage classes in the cluster")
+	err := storageDetect(endpoint, endpointService, factory)
+	if err == nil {
 		return
 	}
+	log.Err(err).Msg("error while detecting storage classes")
+	go func() {
+		// Retry after 30 seconds if the initial detection failed.
+		log.Info().Msg("retrying storage detection in 30 seconds")
+		time.Sleep(30 * time.Second)
+		err := storageDetect(endpoint, endpointService, factory)
+		log.Err(err).Msg("final error while detecting storage classes")
+	}()
 }

api/portainer.go

@@ -567,6 +567,7 @@ type (
 
 	KubernetesFlags struct {
 		IsServerMetricsDetected bool `json:"IsServerMetricsDetected"`
+		IsServerStorageDetected bool `json:"IsServerStorageDetected"`
 	}
 
 	// KubernetesSnapshot represents a snapshot of a specific Kubernetes environment(endpoint) at a specific time