feat(templates): re-introduce external template management (#2119)

* feat(templates): re-introduce external template management

* refactor(api): review error handling

api/cmd/portainer/main.go

@@ -193,6 +193,10 @@ func initSettings(settingsService portainer.SettingsService, flags *portainer.CL
 }
 
 func initTemplates(templateService portainer.TemplateService, fileService portainer.FileService, templateURL, templateFile string) error {
+	if templateURL != "" {
+		log.Printf("Portainer started with the --templates flag. Using external templates, template management will be disabled.")
+		return nil
+	}
 
 	existingTemplates, err := templateService.Templates()
 	if err != nil {
@@ -204,32 +208,14 @@ func initTemplates(templateService portainer.TemplateService, fileService portai
 		return nil
 	}
 
-	var templatesJSON []byte
-	if templateURL == "" {
-		return loadTemplatesFromFile(fileService, templateService, templateFile)
-	}
-
-	templatesJSON, err = client.Get(templateURL)
-	if err != nil {
-		log.Println("Unable to retrieve templates via HTTP")
-		return err
-	}
-
-	return unmarshalAndPersistTemplates(templateService, templatesJSON)
-}
-
-func loadTemplatesFromFile(fileService portainer.FileService, templateService portainer.TemplateService, templateFile string) error {
 	templatesJSON, err := fileService.GetFileContent(templateFile)
 	if err != nil {
-		log.Println("Unable to retrieve template via filesystem")
+		log.Println("Unable to retrieve template definitions via filesystem")
 		return err
 	}
-	return unmarshalAndPersistTemplates(templateService, templatesJSON)
-}
 
-func unmarshalAndPersistTemplates(templateService portainer.TemplateService, templateData []byte) error {
 	var templates []portainer.Template
-	err := json.Unmarshal(templateData, &templates)
+	err = json.Unmarshal(templatesJSON, &templates)
 	if err != nil {
 		log.Println("Unable to parse templates file. Please review your template definition file.")
 		return err
@@ -241,6 +227,7 @@ func unmarshalAndPersistTemplates(templateService portainer.TemplateService, tem
 			return err
 		}
 	}
+
 	return nil
 }
 

api/http/handler/settings/settings_public.go

@@ -13,6 +13,7 @@ type publicSettingsResponse struct {
 	AuthenticationMethod               portainer.AuthenticationMethod `json:"AuthenticationMethod"`
 	AllowBindMountsForRegularUsers     bool                           `json:"AllowBindMountsForRegularUsers"`
 	AllowPrivilegedModeForRegularUsers bool                           `json:"AllowPrivilegedModeForRegularUsers"`
+	ExternalTemplates                  bool                           `json:"ExternalTemplates"`
 }
 
 // GET request on /api/settings/public
@@ -27,6 +28,11 @@ func (handler *Handler) settingsPublic(w http.ResponseWriter, r *http.Request) *
 		AuthenticationMethod:               settings.AuthenticationMethod,
 		AllowBindMountsForRegularUsers:     settings.AllowBindMountsForRegularUsers,
 		AllowPrivilegedModeForRegularUsers: settings.AllowPrivilegedModeForRegularUsers,
+		ExternalTemplates:                  false,
+	}
+
+	if settings.TemplatesURL != "" {
+		publicSettings.ExternalTemplates = true
 	}
 
 	return response.JSON(w, publicSettings)

api/http/handler/settings/settings_update.go

@@ -19,6 +19,7 @@ type settingsUpdatePayload struct {
 	AllowBindMountsForRegularUsers     *bool
 	AllowPrivilegedModeForRegularUsers *bool
 	SnapshotInterval                   *string
+	TemplatesURL                       *string
 }
 
 func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
@@ -28,6 +29,9 @@ func (payload *settingsUpdatePayload) Validate(r *http.Request) error {
 	if payload.LogoURL != nil && *payload.LogoURL != "" && !govalidator.IsURL(*payload.LogoURL) {
 		return portainer.Error("Invalid logo URL. Must correspond to a valid URL format")
 	}
+	if payload.TemplatesURL != nil && *payload.TemplatesURL != "" && !govalidator.IsURL(*payload.TemplatesURL) {
+		return portainer.Error("Invalid external templates URL. Must correspond to a valid URL format")
+	}
 	return nil
 }
 
@@ -52,6 +56,10 @@ func (handler *Handler) settingsUpdate(w http.ResponseWriter, r *http.Request) *
 		settings.LogoURL = *payload.LogoURL
 	}
 
+	if payload.TemplatesURL != nil {
+		settings.TemplatesURL = *payload.TemplatesURL
+	}
+
 	if payload.BlackListedLabels != nil {
 		settings.BlackListedLabels = payload.BlackListedLabels
 	}

api/http/handler/templates/handler.go

@@ -9,10 +9,15 @@ import (
 	"github.com/portainer/portainer/http/security"
 )
 
+const (
+	errTemplateManagementDisabled = portainer.Error("Template management is disabled")
+)
+
 // Handler represents an HTTP API handler for managing templates.
 type Handler struct {
 	*mux.Router
 	TemplateService portainer.TemplateService
+	SettingsService portainer.SettingsService
 }
 
 // NewHandler returns a new instance of Handler.
@@ -20,15 +25,32 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
 	h := &Handler{
 		Router: mux.NewRouter(),
 	}
+
 	h.Handle("/templates",
 		bouncer.RestrictedAccess(httperror.LoggerHandler(h.templateList))).Methods(http.MethodGet)
 	h.Handle("/templates",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateCreate))).Methods(http.MethodPost)
+		bouncer.AdministratorAccess(h.templateManagementCheck(httperror.LoggerHandler(h.templateCreate)))).Methods(http.MethodPost)
 	h.Handle("/templates/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateInspect))).Methods(http.MethodGet)
+		bouncer.AdministratorAccess(h.templateManagementCheck(httperror.LoggerHandler(h.templateInspect)))).Methods(http.MethodGet)
 	h.Handle("/templates/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateUpdate))).Methods(http.MethodPut)
+		bouncer.AdministratorAccess(h.templateManagementCheck(httperror.LoggerHandler(h.templateUpdate)))).Methods(http.MethodPut)
 	h.Handle("/templates/{id}",
-		bouncer.AdministratorAccess(httperror.LoggerHandler(h.templateDelete))).Methods(http.MethodDelete)
+		bouncer.AdministratorAccess(h.templateManagementCheck(httperror.LoggerHandler(h.templateDelete)))).Methods(http.MethodDelete)
 	return h
 }
+
+func (handler *Handler) templateManagementCheck(next http.Handler) http.Handler {
+	return httperror.LoggerHandler(func(rw http.ResponseWriter, r *http.Request) *httperror.HandlerError {
+		settings, err := handler.SettingsService.Settings()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+		}
+
+		if settings.TemplatesURL != "" {
+			return &httperror.HandlerError{http.StatusServiceUnavailable, "Portainer is configured to use external templates, template management is disabled", errTemplateManagementDisabled}
+		}
+
+		next.ServeHTTP(rw, r)
+		return nil
+	})
+}

api/http/handler/templates/template_list.go

@@ -1,8 +1,11 @@
 package templates
 
 import (
+	"encoding/json"
 	"net/http"
 
+	"github.com/portainer/portainer"
+	"github.com/portainer/portainer/http/client"
 	httperror "github.com/portainer/portainer/http/error"
 	"github.com/portainer/portainer/http/response"
 	"github.com/portainer/portainer/http/security"
@@ -10,10 +13,29 @@ import (
 
 // GET request on /api/templates
 func (handler *Handler) templateList(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
-	templates, err := handler.TemplateService.Templates()
+	settings, err := handler.SettingsService.Settings()
+	if err != nil {
+		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+	}
+
+	var templates []portainer.Template
+	if settings.TemplatesURL == "" {
+		templates, err = handler.TemplateService.Templates()
 		if err != nil {
 			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve templates from the database", err}
 		}
+	} else {
+		var templateData []byte
+		templateData, err = client.Get(settings.TemplatesURL)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve external templates", err}
+		}
+
+		err = json.Unmarshal(templateData, &templates)
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to parse external templates", err}
+		}
+	}
 
 	securityContext, err := security.RetrieveRestrictedRequestContext(r)
 	if err != nil {
@@ -21,6 +43,5 @@ func (handler *Handler) templateList(w http.ResponseWriter, r *http.Request) *ht
 	}
 
 	filteredTemplates := security.FilterTemplates(templates, securityContext)
-
 	return response.JSON(w, filteredTemplates)
 }

api/http/server.go

@@ -151,6 +151,7 @@ func (server *Server) Start() error {
 
 	var templatesHandler = templates.NewHandler(requestBouncer)
 	templatesHandler.TemplateService = server.TemplateService
+	templatesHandler.SettingsService = server.SettingsService
 
 	var uploadHandler = upload.NewHandler(requestBouncer)
 	uploadHandler.FileService = server.FileService

api/portainer.go

@@ -88,11 +88,11 @@ type (
 		AllowBindMountsForRegularUsers     bool                 `json:"AllowBindMountsForRegularUsers"`
 		AllowPrivilegedModeForRegularUsers bool                 `json:"AllowPrivilegedModeForRegularUsers"`
 		SnapshotInterval                   string               `json:"SnapshotInterval"`
+		TemplatesURL                       string               `json:"TemplatesURL"`
 
 		// Deprecated fields
 		DisplayDonationHeader       bool
 		DisplayExternalContributors bool
-		TemplatesURL                string
 	}
 
 	// User represents a user account.

app/portainer/models/settings.js

@@ -6,6 +6,8 @@ function SettingsViewModel(data) {
   this.AllowBindMountsForRegularUsers = data.AllowBindMountsForRegularUsers;
   this.AllowPrivilegedModeForRegularUsers = data.AllowPrivilegedModeForRegularUsers;
   this.SnapshotInterval = data.SnapshotInterval;
+  this.TemplatesURL = data.TemplatesURL;
+  this.ExternalTemplates = data.ExternalTemplates;
 }
 
 function LDAPSettingsViewModel(data) {

app/portainer/views/settings/settings.html

@@ -44,6 +44,37 @@
             </div>
           </div>
           <!-- !logo -->
+          <!-- templates -->
+          <div class="col-sm-12 form-section-title">
+            App Templates
+          </div>
+          <div class="form-group">
+            <div class="col-sm-12">
+              <label for="toggle_templates" class="control-label text-left">
+                Use external templates
+                <portainer-tooltip position="bottom" message="When using external templates, in-app template management will be disabled."></portainer-tooltip>
+              </label>
+              <label class="switch" style="margin-left: 20px;">
+                <input type="checkbox" name="toggle_templates" ng-model="formValues.externalTemplates"><i></i>
+              </label>
+            </div>
+          </div>
+          <div ng-if="formValues.externalTemplates">
+            <div class="form-group">
+              <span class="col-sm-12 text-muted small">
+                You can specify the URL to your own template definitions file here. See <a href="https://portainer.readthedocs.io/en/stable/templates.html" target="_blank">Portainer documentation</a> for more details.
+              </span>
+            </div>
+            <div class="form-group" >
+              <label for="templates_url" class="col-sm-1 control-label text-left">
+                URL
+              </label>
+              <div class="col-sm-11">
+                <input type="text" class="form-control" ng-model="settings.TemplatesURL" id="templates_url" placeholder="https://myserver.mydomain/templates.json">
+              </div>
+            </div>
+          </div>
+          <!-- !templates -->
           <!-- security -->
           <div class="col-sm-12 form-section-title">
             Security

app/portainer/views/settings/settingsController.js

@@ -8,6 +8,7 @@ function ($scope, $state, Notifications, SettingsService, StateManager, DEFAULT_
 
   $scope.formValues = {
     customLogo: false,
+    externalTemplates: false,
     restrictBindMounts: false,
     restrictPrivilegedMode: false,
     labelName: '',
@@ -39,6 +40,10 @@ function ($scope, $state, Notifications, SettingsService, StateManager, DEFAULT_
       settings.LogoURL = '';
     }
 
+    if (!$scope.formValues.externalTemplates) {
+      settings.TemplatesURL = '';
+    }
+
     settings.AllowBindMountsForRegularUsers = !$scope.formValues.restrictBindMounts;
     settings.AllowPrivilegedModeForRegularUsers = !$scope.formValues.restrictPrivilegedMode;
 
@@ -70,6 +75,9 @@ function ($scope, $state, Notifications, SettingsService, StateManager, DEFAULT_
       if (settings.LogoURL !== '') {
         $scope.formValues.customLogo = true;
       }
+      if (settings.TemplatesURL !== '') {
+        $scope.formValues.externalTemplates = true;
+      }
       $scope.formValues.restrictBindMounts = !settings.AllowBindMountsForRegularUsers;
       $scope.formValues.restrictPrivilegedMode = !settings.AllowPrivilegedModeForRegularUsers;
     })

app/portainer/views/templates/templates.html

@@ -353,9 +353,9 @@
       templates="templates"
       select-action="selectTemplate"
       delete-action="deleteTemplate"
-      show-add-action="isAdmin"
+      show-add-action="state.templateManagement && isAdmin"
-      show-update-action="isAdmin"
+      show-update-action="state.templateManagement && isAdmin"
-      show-delete-action="isAdmin"
+      show-delete-action="state.templateManagement && isAdmin"
       show-swarm-stacks="applicationState.endpoint.mode.provider === 'DOCKER_SWARM_MODE' && applicationState.endpoint.mode.role === 'MANAGER' && applicationState.endpoint.apiVersion >= 1.25"
     ></template-list>
   </div>

app/portainer/views/templates/templatesController.js

@@ -5,7 +5,8 @@ function ($scope, $q, $state, $transition$, $anchorScroll, ContainerService, Ima
     selectedTemplate: null,
     showAdvancedOptions: false,
     formValidationError: '',
-    actionInProgress: false
+    actionInProgress: false,
+    templateManagement: true
   };
 
   $scope.formValues = {
@@ -253,6 +254,7 @@ function ($scope, $q, $state, $transition$, $anchorScroll, ContainerService, Ima
       $scope.availableNetworks = networks;
       var settings = data.settings;
       $scope.allowBindMounts = settings.AllowBindMountsForRegularUsers;
+      $scope.state.templateManagement = !settings.ExternalTemplates;
     })
     .catch(function error(err) {
       $scope.templates = [];