fix(endpoints): ensure endpoint is up to date after snapshot (#2460)

* feat(snapshots): fix a potential concurrency issue with endpoint snapshots

* fix(endpoints): ensure endpoint is up to date after snapshot

api/cron/job_snapshot.go

@@ -42,6 +42,8 @@ func (runner *SnapshotJobRunner) GetSchedule() *portainer.Schedule {
 // Run triggers the execution of the schedule.
 // It will iterate through all the endpoints available in the database to
 // create a snapshot of each one of them.
+// As a snapshot can be a long process, to avoid any concurrency issue we
+// retrieve the latest version of the endpoint right after a snapshot.
 func (runner *SnapshotJobRunner) Run() {
 	endpoints, err := runner.context.endpointService.Endpoints()
 	if err != nil {
@@ -54,18 +56,25 @@ func (runner *SnapshotJobRunner) Run() {
 			continue
 		}
 
-		snapshot, err := runner.context.snapshotter.CreateSnapshot(&endpoint)
-		endpoint.Status = portainer.EndpointStatusUp
-		if err != nil {
-			log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
-			endpoint.Status = portainer.EndpointStatusDown
+		snapshot, snapshotError := runner.context.snapshotter.CreateSnapshot(&endpoint)
+
+		latestEndpointReference, err := runner.context.endpointService.Endpoint(endpoint.ID)
+		if latestEndpointReference == nil {
+			log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+			continue
+		}
+
+		latestEndpointReference.Status = portainer.EndpointStatusUp
+		if snapshotError != nil {
+			log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, snapshotError)
+			latestEndpointReference.Status = portainer.EndpointStatusDown
 		}
 
 		if snapshot != nil {
-			endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+			latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
 		}
 
-		err = runner.context.endpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		err = runner.context.endpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
 		if err != nil {
 			log.Printf("background schedule error (endpoint snapshot). Unable to update endpoint (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
 			return

api/http/handler/endpoints/endpoint_snapshot.go

@@ -3,6 +3,7 @@ package endpoints
 import (
 	"log"
 	"net/http"
+	"time"
 
 	httperror "github.com/portainer/libhttp/error"
 	"github.com/portainer/libhttp/response"
@@ -21,18 +22,27 @@ func (handler *Handler) endpointSnapshot(w http.ResponseWriter, r *http.Request)
 			continue
 		}
 
-		snapshot, err := handler.Snapshotter.CreateSnapshot(&endpoint)
-		endpoint.Status = portainer.EndpointStatusUp
-		if err != nil {
-			log.Printf("http error: endpoint snapshot error (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
-			endpoint.Status = portainer.EndpointStatusDown
+		time.Sleep(10 * time.Second)
+
+		snapshot, snapshotError := handler.Snapshotter.CreateSnapshot(&endpoint)
+
+		latestEndpointReference, err := handler.EndpointService.Endpoint(endpoint.ID)
+		if latestEndpointReference == nil {
+			log.Printf("background schedule error (endpoint snapshot). Endpoint not found inside the database anymore (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
+			continue
+		}
+
+		latestEndpointReference.Status = portainer.EndpointStatusUp
+		if snapshotError != nil {
+			log.Printf("background schedule error (endpoint snapshot). Unable to create snapshot (endpoint=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, snapshotError)
+			latestEndpointReference.Status = portainer.EndpointStatusDown
 		}
 
 		if snapshot != nil {
-			endpoint.Snapshots = []portainer.Snapshot{*snapshot}
+			latestEndpointReference.Snapshots = []portainer.Snapshot{*snapshot}
 		}
 
-		err = handler.EndpointService.UpdateEndpoint(endpoint.ID, &endpoint)
+		err = handler.EndpointService.UpdateEndpoint(latestEndpointReference.ID, latestEndpointReference)
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to persist endpoint changes inside the database", err}
 		}