From c650868fe9cc416d6ca50418dd9377bb35229614 Mon Sep 17 00:00:00 2001 From: Oscar Zhou <100548325+oscarzhou-portainer@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:44:42 +1200 Subject: [PATCH] feat(templates): allow managing git based templates [EE-2600] (#7855) Co-authored-by: itsconquest Co-authored-by: oscarzhou Co-authored-by: Chaim Lev-Ari --- .../customtemplates/customtemplate_create.go | 67 ++++--- .../customtemplates/customtemplate_file.go | 6 +- .../customtemplate_git_fetch.go | 124 +++++++++++++ .../customtemplate_git_fetch_test.go | 174 ++++++++++++++++++ .../customtemplates/customtemplate_update.go | 82 ++++++++- api/http/handler/customtemplates/handler.go | 18 +- .../handler/gitops/git_repo_file_preview.go | 89 +++++++++ api/http/handler/gitops/handler.go | 33 ++++ api/http/handler/handler.go | 6 + .../handler/stacks/create_kubernetes_stack.go | 7 +- api/http/server.go | 9 +- api/portainer.go | 3 + .../stackbuilders/k8s_file_content_builder.go | 1 + api/stacks/stackbuilders/stack_git_builder.go | 8 +- api/stacks/stackutils/gitops.go | 6 +- ...-create-custom-template-view.controller.js | 17 +- .../kube-create-custom-template-view.html | 2 + ...be-edit-custom-template-view.controller.js | 87 ++++++++- .../kube-edit-custom-template-view.html | 23 ++- app/kubernetes/views/deploy/deploy.html | 11 ++ .../views/deploy/deployController.js | 24 ++- .../code-editor/codeEditorController.js | 11 +- app/portainer/rest/customTemplate.js | 1 + app/portainer/services/api/customTemplate.js | 7 +- .../customTemplatesView.html | 18 +- .../customTemplatesViewController.js | 12 +- .../editCustomTemplateView.html | 23 ++- .../editCustomTemplateViewController.js | 90 ++++++++- .../stacks/create/createStackController.js | 23 ++- .../views/stacks/create/createstack.html | 37 ++-- app/react/portainer/gitops/gitops.service.ts | 25 +++ .../portainer/gitops/queries/useSearch.ts | 1 + 32 files changed, 944 insertions(+), 101 deletions(-) create mode 100644 api/http/handler/customtemplates/customtemplate_git_fetch.go create mode 100644 api/http/handler/customtemplates/customtemplate_git_fetch_test.go create mode 100644 api/http/handler/gitops/git_repo_file_preview.go create mode 100644 api/http/handler/gitops/handler.go create mode 100644 app/react/portainer/gitops/gitops.service.ts diff --git a/api/http/handler/customtemplates/customtemplate_create.go b/api/http/handler/customtemplates/customtemplate_create.go index e0d273efc..03c532aa7 100644 --- a/api/http/handler/customtemplates/customtemplate_create.go +++ b/api/http/handler/customtemplates/customtemplate_create.go @@ -3,7 +3,6 @@ package customtemplates import ( "encoding/json" "errors" - "fmt" "net/http" "os" "regexp" @@ -18,6 +17,7 @@ import ( gittypes "github.com/portainer/portainer/api/git/types" "github.com/portainer/portainer/api/http/security" "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/stacks/stackutils" "github.com/rs/zerolog/log" ) @@ -135,6 +135,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") } + // Platform validation is only for docker related stack (docker standalone and docker swarm) if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack { return errors.New("Invalid custom template type") } @@ -215,6 +216,8 @@ type customTemplateFromGitRepositoryPayload struct { Variables []portainer.CustomTemplateVariableDefinition // TLSSkipVerify skips SSL verification when cloning the Git repository TLSSkipVerify bool `example:"false"` + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error { @@ -234,14 +237,11 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName } - if payload.Type == portainer.KubernetesStack { - return errors.New("Creating a Kubernetes custom template from git is not supported") - } - - if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { + // Platform validation is only for docker related stack (docker standalone and docker swarm) + if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") } - if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack { + if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack { return errors.New("Invalid custom template type") } if !isValidNote(payload.Note) { @@ -260,35 +260,44 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier() customTemplate := &portainer.CustomTemplate{ - ID: portainer.CustomTemplateID(customTemplateID), - Title: payload.Title, - EntryPoint: payload.ComposeFilePathInRepository, - Description: payload.Description, - Note: payload.Note, - Platform: payload.Platform, - Type: payload.Type, - Logo: payload.Logo, - Variables: payload.Variables, + ID: portainer.CustomTemplateID(customTemplateID), + Title: payload.Title, + Description: payload.Description, + Note: payload.Note, + Platform: payload.Platform, + Type: payload.Type, + Logo: payload.Logo, + Variables: payload.Variables, + IsComposeFormat: payload.IsComposeFormat, } - projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID)) + getProjectPath := func() string { + return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID)) + } + projectPath := getProjectPath() customTemplate.ProjectPath = projectPath - repositoryUsername := payload.RepositoryUsername - repositoryPassword := payload.RepositoryPassword - if !payload.RepositoryAuthentication { - repositoryUsername = "" - repositoryPassword = "" + gitConfig := &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFilePathInRepository, } - err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword, payload.TLSSkipVerify) - if err != nil { - if err == gittypes.ErrAuthenticationFailure { - return nil, fmt.Errorf("invalid git credential") + if payload.RepositoryAuthentication { + gitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, } + } + + commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath) + if err != nil { return nil, err } + gitConfig.ConfigHash = commitHash + customTemplate.GitConfig = gitConfig + isValidProject := true defer func() { if !isValidProject { @@ -298,7 +307,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( } }() - entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint) + entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath) exists, err := handler.FileService.FileExists(entryPath) if err != nil || !exists { @@ -310,6 +319,9 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) ( } if !exists { + if payload.Type == portainer.KubernetesStack { + return nil, errors.New("Invalid Manifest file, ensure that the Manifest file path is correct") + } return nil, errors.New("Invalid Compose file, ensure that the Compose file path is correct") } @@ -369,6 +381,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true) templatePlatform := portainer.CustomTemplatePlatform(platform) + // Platform validation is only for docker related stack (docker standalone and docker swarm) if templateType != portainer.KubernetesStack && templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") } diff --git a/api/http/handler/customtemplates/customtemplate_file.go b/api/http/handler/customtemplates/customtemplate_file.go index 7666bae25..3749d6c66 100644 --- a/api/http/handler/customtemplates/customtemplate_file.go +++ b/api/http/handler/customtemplates/customtemplate_file.go @@ -40,7 +40,11 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err) } - fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint) + entryPath := customTemplate.EntryPoint + if customTemplate.GitConfig != nil { + entryPath = customTemplate.GitConfig.ConfigFilePath + } + fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath) if err != nil { return httperror.InternalServerError("Unable to retrieve custom template file from disk", err) } diff --git a/api/http/handler/customtemplates/customtemplate_git_fetch.go b/api/http/handler/customtemplates/customtemplate_git_fetch.go new file mode 100644 index 000000000..fc7d68a85 --- /dev/null +++ b/api/http/handler/customtemplates/customtemplate_git_fetch.go @@ -0,0 +1,124 @@ +package customtemplates + +import ( + "fmt" + "net/http" + "os" + "sync" + + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/stacks/stackutils" + "github.com/rs/zerolog/log" +) + +// @id CustomTemplateGitFetch +// @summary Fetch the latest config file content based on custom template's git repository configuration +// @description Retrieve details about a template created from git repository method. +// @description **Access policy**: authenticated +// @tags custom_templates +// @security ApiKeyAuth +// @security jwt +// @produce json +// @param id path int true "Template identifier" +// @success 200 {object} fileResponse "Success" +// @failure 400 "Invalid request" +// @failure 404 "Custom template not found" +// @failure 500 "Server error" +// @router /custom_templates/{id}/git_fetch [put] +func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id") + if err != nil { + return httperror.BadRequest("Invalid Custom template identifier route variable", err) + } + + customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID)) + if handler.DataStore.IsErrObjectNotFound(err) { + return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err) + } else if err != nil { + return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err) + } + + if customTemplate.GitConfig == nil { + return httperror.BadRequest("Git configuration does not exist in this custom template", err) + } + + // If multiple users are trying to fetch the same custom template simultaneously, a lock needs to be added + mu, ok := handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] + if !ok { + mu = &sync.Mutex{} + handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] = mu + } + mu.Lock() + defer mu.Unlock() + + // back up the current custom template folder + backupPath, err := backupCustomTemplate(customTemplate.ProjectPath) + if err != nil { + return httperror.InternalServerError("Failed to backup the custom template folder", err) + } + + // remove backup custom template folder + defer cleanUpBackupCustomTemplate(backupPath) + + commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string { + return customTemplate.ProjectPath + }) + if err != nil { + log.Warn().Err(err).Msg("failed to download git repository") + rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath) + if err != nil { + return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr) + } + return httperror.InternalServerError("Failed to download git repository", err) + } + + if customTemplate.GitConfig.ConfigHash != commitHash { + customTemplate.GitConfig.ConfigHash = commitHash + + err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate) + if err != nil { + return httperror.InternalServerError("Unable to persist custom template changes inside the database", err) + } + } + + fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.GitConfig.ConfigFilePath) + if err != nil { + return httperror.InternalServerError("Unable to retrieve custom template file from disk", err) + } + + return response.JSON(w, &fileResponse{FileContent: string(fileContent)}) +} + +func backupCustomTemplate(projectPath string) (string, error) { + stat, err := os.Stat(projectPath) + if err != nil { + return "", err + } + + backupPath := fmt.Sprintf("%s-backup", projectPath) + err = os.Rename(projectPath, backupPath) + if err != nil { + return "", err + } + + err = os.Mkdir(projectPath, stat.Mode()) + if err != nil { + return backupPath, err + } + return backupPath, nil +} + +func rollbackCustomTemplate(backupPath, projectPath string) error { + err := os.RemoveAll(projectPath) + if err != nil { + return err + } + return os.Rename(backupPath, projectPath) +} + +func cleanUpBackupCustomTemplate(backupPath string) error { + return os.RemoveAll(backupPath) +} diff --git a/api/http/handler/customtemplates/customtemplate_git_fetch_test.go b/api/http/handler/customtemplates/customtemplate_git_fetch_test.go new file mode 100644 index 000000000..53e48720a --- /dev/null +++ b/api/http/handler/customtemplates/customtemplate_git_fetch_test.go @@ -0,0 +1,174 @@ +package customtemplates + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/datastore" + gittypes "github.com/portainer/portainer/api/git/types" + "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/internal/authorization" + "github.com/portainer/portainer/api/jwt" + "github.com/stretchr/testify/assert" +) + +var testFileContent string = "abcdefg" + +type TestGitService struct { + portainer.GitService + targetFilePath string +} + +func (g *TestGitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string, tlsSkipVerify bool) error { + time.Sleep(100 * time.Millisecond) + return createTestFile(g.targetFilePath) +} + +func (g *TestGitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) { + return "", nil +} + +type TestFileService struct { + portainer.FileService +} + +func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]byte, error) { + return os.ReadFile(filepath.Join(projectPath, configFilePath)) +} + +func createTestFile(targetPath string) error { + f, err := os.Create(targetPath) + if err != nil { + return err + } + defer f.Close() + _, err = f.WriteString(testFileContent) + return err +} + +func prepareTestFolder(projectPath, filename string) error { + err := os.MkdirAll(projectPath, fs.ModePerm) + if err != nil { + return err + } + + return createTestFile(filepath.Join(projectPath, filename)) +} + +func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect string) { + type response struct { + FileContent string + } + + req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}"))) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt)) + + rr := httptest.NewRecorder() + h.ServeHTTP(rr, req) + + is.Equal(http.StatusOK, rr.Code) + + body, err := io.ReadAll(rr.Body) + is.NoError(err, "ReadAll should not return error") + + var resp response + err = json.Unmarshal(body, &resp) + is.NoError(err, "response should be list json") + is.Equal(resp.FileContent, expect) +} + +func Test_customTemplateGitFetch(t *testing.T) { + is := assert.New(t) + + _, store, teardown := datastore.MustNewTestStore(t, true, true) + defer teardown() + + // create user(s) + user1 := &portainer.User{ID: 1, Username: "user-1", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()} + err := store.User().Create(user1) + is.NoError(err, "error creating user 1") + + user2 := &portainer.User{ID: 2, Username: "user-2", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()} + err = store.User().Create(user2) + is.NoError(err, "error creating user 2") + + dir, err := os.Getwd() + is.NoError(err, "error to get working directory") + + template1 := &portainer.CustomTemplate{ID: 1, Title: "custom-template-1", ProjectPath: filepath.Join(dir, "fixtures/custom_template_1"), GitConfig: &gittypes.RepoConfig{ConfigFilePath: "test-config-path.txt"}} + err = store.CustomTemplateService.Create(template1) + is.NoError(err, "error creating custom template 1") + + // prepare testing folder + err = prepareTestFolder(template1.ProjectPath, template1.GitConfig.ConfigFilePath) + is.NoError(err, "error creating testing folder") + + defer os.RemoveAll(filepath.Join(dir, "fixtures")) + + // setup services + jwtService, err := jwt.NewService("1h", store) + is.NoError(err, "Error initiating jwt service") + requestBouncer := security.NewRequestBouncer(store, jwtService, nil) + + gitService := &TestGitService{ + targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath), + } + fileService := &TestFileService{} + + h := NewHandler(requestBouncer, store, fileService, gitService) + + // generate two standard users' tokens + jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role}) + jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role}) + + t.Run("can return the expected file content by a single call from one user", func(t *testing.T) { + singleAPIRequest(h, jwt1, is, "abcdefg") + }) + + t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) { + var wg sync.WaitGroup + wg.Add(5) + for i := 0; i < 5; i++ { + go func() { + singleAPIRequest(h, jwt1, is, "abcdefg") + wg.Done() + }() + } + wg.Wait() + }) + + t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) { + var wg sync.WaitGroup + wg.Add(10) + for i := 0; i < 10; i++ { + go func(j int) { + if j%1 == 0 { + singleAPIRequest(h, jwt1, is, "abcdefg") + } else { + singleAPIRequest(h, jwt2, is, "abcdefg") + } + wg.Done() + }(i) + } + wg.Wait() + }) + + t.Run("can return the expected file content after a new commit is made", func(t *testing.T) { + singleAPIRequest(h, jwt1, is, "abcdefg") + + testFileContent = "gfedcba" + + singleAPIRequest(h, jwt2, is, "gfedcba") + }) +} diff --git a/api/http/handler/customtemplates/customtemplate_update.go b/api/http/handler/customtemplates/customtemplate_update.go index 38e10fe9a..52d008e22 100644 --- a/api/http/handler/customtemplates/customtemplate_update.go +++ b/api/http/handler/customtemplates/customtemplate_update.go @@ -10,8 +10,11 @@ import ( "github.com/portainer/libhttp/request" "github.com/portainer/libhttp/response" portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/filesystem" + gittypes "github.com/portainer/portainer/api/git/types" httperrors "github.com/portainer/portainer/api/http/errors" "github.com/portainer/portainer/api/http/security" + "github.com/portainer/portainer/api/stacks/stackutils" ) type customTemplateUpdatePayload struct { @@ -29,18 +32,37 @@ type customTemplateUpdatePayload struct { Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"` // Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes) Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"` + // URL of a Git repository hosting the Stack file + RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"` + // Reference name of a Git repository hosting the Stack file + RepositoryReferenceName string `example:"refs/heads/master"` + // Use basic authentication to clone the Git repository + RepositoryAuthentication bool `example:"true"` + // Username used in basic authentication. Required when RepositoryAuthentication is true + // and RepositoryGitCredentialID is 0 + RepositoryUsername string `example:"myGitUsername"` + // Password used in basic authentication. Required when RepositoryAuthentication is true + // and RepositoryGitCredentialID is 0 + RepositoryPassword string `example:"myGitPassword"` + // GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication + // is true and RepositoryUsername/RepositoryPassword are not provided + RepositoryGitCredentialID int `example:"0"` + // Path to the Stack file inside the Git repository + ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"` // Content of stack file FileContent string `validate:"required"` // Definitions of variables in the stack file Variables []portainer.CustomTemplateVariableDefinition + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error { if govalidator.IsNull(payload.Title) { return errors.New("Invalid custom template title") } - if govalidator.IsNull(payload.FileContent) { - return errors.New("Invalid file content") + if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) { + return errors.New("Either file content or git repository url need to be provided") } if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows { return errors.New("Invalid custom template platform") @@ -55,7 +77,19 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error { return errors.New("Invalid note. tag is not supported") } - return validateVariablesDefinitions(payload.Variables) + if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) { + return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled") + } + if govalidator.IsNull(payload.ComposeFilePathInRepository) { + payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName + } + + err := validateVariablesDefinitions(payload.Variables) + if err != nil { + return err + } + + return nil } // @id CustomTemplateUpdate @@ -115,12 +149,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied) } - templateFolder := strconv.Itoa(customTemplateID) - _, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent)) - if err != nil { - return httperror.InternalServerError("Unable to persist updated custom template file on disk", err) - } - customTemplate.Title = payload.Title customTemplate.Logo = payload.Logo customTemplate.Description = payload.Description @@ -128,6 +156,42 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ customTemplate.Platform = payload.Platform customTemplate.Type = payload.Type customTemplate.Variables = payload.Variables + customTemplate.IsComposeFormat = payload.IsComposeFormat + + if payload.RepositoryURL != "" { + if !govalidator.IsURL(payload.RepositoryURL) { + return httperror.BadRequest("Invalid repository URL. Must correspond to a valid URL format", err) + } + + gitConfig := &gittypes.RepoConfig{ + URL: payload.RepositoryURL, + ReferenceName: payload.RepositoryReferenceName, + ConfigFilePath: payload.ComposeFilePathInRepository, + } + + if payload.RepositoryAuthentication { + gitConfig.Authentication = &gittypes.GitAuthentication{ + Username: payload.RepositoryUsername, + Password: payload.RepositoryPassword, + } + } + + commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string { + return customTemplate.ProjectPath + }) + if err != nil { + return httperror.InternalServerError(err.Error(), err) + } + + gitConfig.ConfigHash = commitHash + customTemplate.GitConfig = gitConfig + } else { + templateFolder := strconv.Itoa(customTemplateID) + _, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent)) + if err != nil { + return httperror.InternalServerError("Unable to persist updated custom template file on disk", err) + } + } err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate) if err != nil { diff --git a/api/http/handler/customtemplates/handler.go b/api/http/handler/customtemplates/handler.go index 54fad7d67..7be8084a5 100644 --- a/api/http/handler/customtemplates/handler.go +++ b/api/http/handler/customtemplates/handler.go @@ -2,6 +2,7 @@ package customtemplates import ( "net/http" + "sync" "github.com/gorilla/mux" httperror "github.com/portainer/libhttp/error" @@ -13,15 +14,20 @@ import ( // Handler is the HTTP handler used to handle environment(endpoint) group operations. type Handler struct { *mux.Router - DataStore dataservices.DataStore - FileService portainer.FileService - GitService portainer.GitService + DataStore dataservices.DataStore + FileService portainer.FileService + GitService portainer.GitService + gitFetchMutexs map[portainer.TemplateID]*sync.Mutex } // NewHandler creates a handler to manage environment(endpoint) group operations. -func NewHandler(bouncer *security.RequestBouncer) *Handler { +func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, fileService portainer.FileService, gitService portainer.GitService) *Handler { h := &Handler{ - Router: mux.NewRouter(), + Router: mux.NewRouter(), + DataStore: dataStore, + FileService: fileService, + GitService: gitService, + gitFetchMutexs: make(map[portainer.TemplateID]*sync.Mutex), } h.Handle("/custom_templates", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateCreate))).Methods(http.MethodPost) @@ -35,6 +41,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler { bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut) h.Handle("/custom_templates/{id}", bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete) + h.Handle("/custom_templates/{id}/git_fetch", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut) return h } diff --git a/api/http/handler/gitops/git_repo_file_preview.go b/api/http/handler/gitops/git_repo_file_preview.go new file mode 100644 index 000000000..6d1218411 --- /dev/null +++ b/api/http/handler/gitops/git_repo_file_preview.go @@ -0,0 +1,89 @@ +package gitops + +import ( + "errors" + "fmt" + "net/http" + + "github.com/asaskevich/govalidator" + httperror "github.com/portainer/libhttp/error" + "github.com/portainer/libhttp/request" + "github.com/portainer/libhttp/response" + gittypes "github.com/portainer/portainer/api/git/types" +) + +type fileResponse struct { + FileContent string +} + +type repositoryFilePreviewPayload struct { + Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"` + Reference string `json:"reference" example:"refs/heads/master"` + Username string `json:"username" example:"myGitUsername"` + Password string `json:"password" example:"myGitPassword"` + // Path to file whose content will be read + TargetFile string `json:"targetFile" example:"docker-compose.yml"` + // TLSSkipVerify skips SSL verification when cloning the Git repository + TLSSkipVerify bool `example:"false"` +} + +func (payload *repositoryFilePreviewPayload) Validate(r *http.Request) error { + if govalidator.IsNull(payload.Repository) || !govalidator.IsURL(payload.Repository) { + return errors.New("Invalid repository URL. Must correspond to a valid URL format") + } + + if govalidator.IsNull(payload.Reference) { + payload.Reference = "refs/heads/main" + } + + if govalidator.IsNull(payload.TargetFile) { + return errors.New("Invalid target filename.") + } + + return nil +} + +// @id GitOperationRepoFilePreview +// @summary preview the content of target file in the git repository +// @description Retrieve the compose file content based on git repository configuration +// @description **Access policy**: authenticated +// @tags gitops +// @security ApiKeyAuth +// @security jwt +// @produce json +// @param body body repositoryFilePreviewPayload true "Template details" +// @success 200 {object} fileResponse "Success" +// @failure 400 "Invalid request" +// @failure 500 "Server error" +// @router /gitops/repo/file/preview [post] +func (handler *Handler) gitOperationRepoFilePreview(w http.ResponseWriter, r *http.Request) *httperror.HandlerError { + var payload repositoryFilePreviewPayload + err := request.DecodeAndValidateJSONPayload(r, &payload) + if err != nil { + return httperror.BadRequest("Invalid request payload", err) + } + + projectPath, err := handler.fileService.GetTemporaryPath() + if err != nil { + return httperror.InternalServerError("Unable to create temporary folder", err) + } + + err = handler.gitService.CloneRepository(projectPath, payload.Repository, payload.Reference, payload.Username, payload.Password, payload.TLSSkipVerify) + if err != nil { + if err == gittypes.ErrAuthenticationFailure { + return httperror.BadRequest("Invalid git credential", err) + } + + newErr := fmt.Errorf("unable to clone git repository: %w", err) + return httperror.InternalServerError(newErr.Error(), newErr) + } + + defer handler.fileService.RemoveDirectory(projectPath) + + fileContent, err := handler.fileService.GetFileContent(projectPath, payload.TargetFile) + if err != nil { + return httperror.InternalServerError("Unable to retrieve custom template file from disk", err) + } + + return response.JSON(w, &fileResponse{FileContent: string(fileContent)}) +} diff --git a/api/http/handler/gitops/handler.go b/api/http/handler/gitops/handler.go new file mode 100644 index 000000000..7b6693201 --- /dev/null +++ b/api/http/handler/gitops/handler.go @@ -0,0 +1,33 @@ +package gitops + +import ( + "net/http" + + "github.com/gorilla/mux" + httperror "github.com/portainer/libhttp/error" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/dataservices" + "github.com/portainer/portainer/api/http/security" +) + +// Handler is the HTTP handler used to handle git repo operation +type Handler struct { + *mux.Router + dataStore dataservices.DataStore + gitService portainer.GitService + fileService portainer.FileService +} + +func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService) *Handler { + h := &Handler{ + Router: mux.NewRouter(), + dataStore: dataStore, + gitService: gitService, + fileService: fileService, + } + + h.Handle("/gitops/repo/file/preview", + bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.gitOperationRepoFilePreview))).Methods(http.MethodPost) + + return h +} diff --git a/api/http/handler/handler.go b/api/http/handler/handler.go index 50a7dc040..48d7dd092 100644 --- a/api/http/handler/handler.go +++ b/api/http/handler/handler.go @@ -17,6 +17,7 @@ import ( "github.com/portainer/portainer/api/http/handler/endpointproxy" "github.com/portainer/portainer/api/http/handler/endpoints" "github.com/portainer/portainer/api/http/handler/file" + "github.com/portainer/portainer/api/http/handler/gitops" "github.com/portainer/portainer/api/http/handler/helm" "github.com/portainer/portainer/api/http/handler/hostmanagement/fdo" "github.com/portainer/portainer/api/http/handler/hostmanagement/openamt" @@ -56,6 +57,7 @@ type Handler struct { EndpointHandler *endpoints.Handler EndpointHelmHandler *helm.Handler EndpointProxyHandler *endpointproxy.Handler + GitOperationHandler *gitops.Handler HelmTemplatesHandler *helm.Handler KubernetesHandler *kubernetes.Handler FileHandler *file.Handler @@ -121,6 +123,8 @@ type Handler struct { // @tag.description Manage Docker environments(endpoints) // @tag.name endpoint_groups // @tag.description Manage environment(endpoint) groups +// @tag.name gitops +// @tag.description Operate git repository // @tag.name kubernetes // @tag.description Manage Kubernetes cluster // @tag.name motd @@ -203,6 +207,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r) } + case strings.HasPrefix(r.URL.Path, "/api/gitops"): + http.StripPrefix("/api", h.GitOperationHandler).ServeHTTP(w, r) case strings.HasPrefix(r.URL.Path, "/api/ldap"): http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r) case strings.HasPrefix(r.URL.Path, "/api/motd"): diff --git a/api/http/handler/stacks/create_kubernetes_stack.go b/api/http/handler/stacks/create_kubernetes_stack.go index da20789db..7d38b66b2 100644 --- a/api/http/handler/stacks/create_kubernetes_stack.go +++ b/api/http/handler/stacks/create_kubernetes_stack.go @@ -23,14 +23,17 @@ type kubernetesStringDeploymentPayload struct { ComposeFormat bool Namespace string StackFileContent string + // Whether the stack is from a app template + FromAppTemplate bool `example:"false"` } -func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload { +func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat, fromAppTemplate bool) stackbuilders.StackPayload { return stackbuilders.StackPayload{ StackName: name, Namespace: namespace, StackFileContent: fileContent, ComposeFormat: composeFormat, + FromAppTemplate: fromAppTemplate, } } @@ -146,7 +149,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists} } - stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat) + stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate) k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore, handler.FileService, diff --git a/api/http/server.go b/api/http/server.go index a82697b4a..956ad1d13 100644 --- a/api/http/server.go +++ b/api/http/server.go @@ -28,6 +28,7 @@ import ( "github.com/portainer/portainer/api/http/handler/endpointproxy" "github.com/portainer/portainer/api/http/handler/endpoints" "github.com/portainer/portainer/api/http/handler/file" + "github.com/portainer/portainer/api/http/handler/gitops" "github.com/portainer/portainer/api/http/handler/helm" "github.com/portainer/portainer/api/http/handler/hostmanagement/fdo" "github.com/portainer/portainer/api/http/handler/hostmanagement/openamt" @@ -143,10 +144,7 @@ func (server *Server) Start() error { var roleHandler = roles.NewHandler(requestBouncer) roleHandler.DataStore = server.DataStore - var customTemplatesHandler = customtemplates.NewHandler(requestBouncer) - customTemplatesHandler.DataStore = server.DataStore - customTemplatesHandler.FileService = server.FileService - customTemplatesHandler.GitService = server.GitService + var customTemplatesHandler = customtemplates.NewHandler(requestBouncer, server.DataStore, server.FileService, server.GitService) var edgeGroupsHandler = edgegroups.NewHandler(requestBouncer) edgeGroupsHandler.DataStore = server.DataStore @@ -196,6 +194,8 @@ func (server *Server) Start() error { var endpointHelmHandler = helm.NewHandler(requestBouncer, server.DataStore, server.JWTService, server.KubernetesDeployer, server.HelmPackageManager, server.KubeClusterAccessService) + var gitOperationHandler = gitops.NewHandler(requestBouncer, server.DataStore, server.GitService, server.FileService) + var helmTemplatesHandler = helm.NewTemplateHandler(requestBouncer, server.HelmPackageManager) var ldapHandler = ldap.NewHandler(requestBouncer) @@ -297,6 +297,7 @@ func (server *Server) Start() error { EndpointHelmHandler: endpointHelmHandler, EndpointEdgeHandler: endpointEdgeHandler, EndpointProxyHandler: endpointProxyHandler, + GitOperationHandler: gitOperationHandler, FileHandler: fileHandler, LDAPHandler: ldapHandler, HelmTemplatesHandler: helmTemplatesHandler, diff --git a/api/portainer.go b/api/portainer.go index 194d4b7b3..ade4b73bb 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -181,6 +181,9 @@ type ( Type StackType `json:"Type" example:"1"` ResourceControl *ResourceControl `json:"ResourceControl"` Variables []CustomTemplateVariableDefinition + GitConfig *gittypes.RepoConfig `json:"GitConfig"` + // IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file + IsComposeFormat bool `example:"false"` } // CustomTemplateID represents a custom template identifier diff --git a/api/stacks/stackbuilders/k8s_file_content_builder.go b/api/stacks/stackbuilders/k8s_file_content_builder.go index ee0335465..783c2110b 100644 --- a/api/stacks/stackbuilders/k8s_file_content_builder.go +++ b/api/stacks/stackbuilders/k8s_file_content_builder.go @@ -53,6 +53,7 @@ func (b *K8sStackFileContentBuilder) SetUniqueInfo(payload *StackPayload) FileCo b.stack.Namespace = payload.Namespace b.stack.CreatedBy = b.User.Username b.stack.IsComposeFormat = payload.ComposeFormat + b.stack.FromAppTemplate = payload.FromAppTemplate return b } diff --git a/api/stacks/stackbuilders/stack_git_builder.go b/api/stacks/stackbuilders/stack_git_builder.go index 099b4c873..ac3ba0cb8 100644 --- a/api/stacks/stackbuilders/stack_git_builder.go +++ b/api/stacks/stackbuilders/stack_git_builder.go @@ -1,6 +1,7 @@ package stackbuilders import ( + "fmt" "strconv" "time" @@ -82,7 +83,12 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho // Set the project path on the disk b.stack.ProjectPath = b.fileService.GetStackProjectPath(stackFolder) - commitHash, err := stackutils.DownloadGitRepository(b.stack.ID, repoConfig, b.gitService, b.fileService) + getProjectPath := func() string { + stackFolder := fmt.Sprintf("%d", b.stack.ID) + return b.fileService.GetStackProjectPath(stackFolder) + } + + commitHash, err := stackutils.DownloadGitRepository(repoConfig, b.gitService, getProjectPath) if err != nil { b.err = httperror.InternalServerError(err.Error(), err) return b diff --git a/api/stacks/stackutils/gitops.go b/api/stacks/stackutils/gitops.go index dcb601763..6c3a18502 100644 --- a/api/stacks/stackutils/gitops.go +++ b/api/stacks/stackutils/gitops.go @@ -16,7 +16,7 @@ var ( // DownloadGitRepository downloads the target git repository on the disk // The first return value represents the commit hash of the downloaded git repository -func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig, gitService portainer.GitService, fileService portainer.FileService) (string, error) { +func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) { username := "" password := "" if config.Authentication != nil { @@ -24,9 +24,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig password = config.Authentication.Password } - stackFolder := fmt.Sprintf("%d", stackID) - projectPath := fileService.GetStackProjectPath(stackFolder) - + projectPath := getProjectPath() err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password, config.TLSSkipVerify) if err != nil { if err == gittypes.ErrAuthenticationFailure { diff --git a/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.controller.js b/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.controller.js index cdda36e32..84353e574 100644 --- a/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.controller.js +++ b/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.controller.js @@ -1,7 +1,7 @@ import { AccessControlFormData } from '@/portainer/components/accessControlForm/porAccessControlFormModel'; import { getTemplateVariables, intersectVariables } from '@/react/portainer/custom-templates/components/utils'; import { isBE } from '@/react/portainer/feature-flags/feature-flags.service'; -import { editor, upload } from '@@/BoxSelector/common-options/build-methods'; +import { editor, upload, git } from '@@/BoxSelector/common-options/build-methods'; import { confirmWebEditorDiscard } from '@@/modals/confirm'; class KubeCreateCustomTemplateViewController { @@ -9,7 +9,7 @@ class KubeCreateCustomTemplateViewController { constructor($async, $state, Authentication, CustomTemplateService, FormValidator, Notifications, ResourceControlService) { Object.assign(this, { $async, $state, Authentication, CustomTemplateService, FormValidator, Notifications, ResourceControlService }); - this.methodOptions = [editor, upload]; + this.methodOptions = [editor, upload, git]; this.templates = null; this.isTemplateVariablesEnabled = isBE; @@ -31,6 +31,13 @@ class KubeCreateCustomTemplateViewController { Logo: '', AccessControlData: new AccessControlFormData(), Variables: [], + RepositoryURL: '', + RepositoryURLValid: false, + RepositoryReferenceName: 'refs/heads/main', + RepositoryAuthentication: false, + RepositoryUsername: '', + RepositoryPassword: '', + ComposeFilePathInRepository: 'manifest.yml', }; this.onChangeFile = this.onChangeFile.bind(this); @@ -121,6 +128,8 @@ class KubeCreateCustomTemplateViewController { return this.createCustomTemplateFromFileContent(template); case 'upload': return this.createCustomTemplateFromFileUpload(template); + case 'repository': + return this.createCustomTemplateFromGitRepository(template); } } @@ -132,6 +141,10 @@ class KubeCreateCustomTemplateViewController { return this.CustomTemplateService.createCustomTemplateFromFileUpload(template); } + createCustomTemplateFromGitRepository(template) { + return this.CustomTemplateService.createCustomTemplateFromGitRepository(template); + } + validateForm(method) { this.state.formValidationError = ''; diff --git a/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.html b/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.html index 91dcd8299..c68dcf8ca 100644 --- a/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.html +++ b/app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.html @@ -35,6 +35,8 @@ You can upload a Manifest file from your computer. + + { + try { + this.formValues.FileContent = await getFilePreview(payload); + this.state.isEditorDirty = true; + + // check if the template contains mustache template symbol + this.parseTemplate(this.formValues.FileContent); + } catch (err) { + this.state.templatePreviewError = err.message; + this.state.templatePreviewFailed = true; + } + }); + } + validateForm() { this.state.formValidationError = ''; diff --git a/app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.html b/app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.html index 1c55d44ee..098f85e44 100644 --- a/app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.html +++ b/app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.html @@ -7,6 +7,22 @@
+ + +
+
+
+
+

+ + Custom template could not be loaded, {{ $ctrl.state.templatePreviewError }}.

+
+
+

Templates allow deploying any kind of Kubernetes resource (Deployment, Secret, ConfigMap...)

@@ -31,7 +48,11 @@ is-variables-names-from-parent="true" > - +
Actions
diff --git a/app/kubernetes/views/deploy/deploy.html b/app/kubernetes/views/deploy/deploy.html index dd44f2832..e13c1686a 100644 --- a/app/kubernetes/views/deploy/deploy.html +++ b/app/kubernetes/views/deploy/deploy.html @@ -105,6 +105,16 @@ >
+ +

+ Custom template could not be loaded, please + click here for configuration.

+

+ Custom template could not be loaded, please contact your administrator.

+ +

diff --git a/app/kubernetes/views/deploy/deployController.js b/app/kubernetes/views/deploy/deployController.js index 38eb34b22..eeda9a594 100644 --- a/app/kubernetes/views/deploy/deployController.js +++ b/app/kubernetes/views/deploy/deployController.js @@ -46,6 +46,13 @@ class KubernetesDeployController { template: null, baseWebhookUrl: baseStackWebhookUrl(), webhookId: createWebhookId(), + templateLoadFailed: false, + isEditorReadOnly: false, + }; + + this.currentUser = { + isAdmin: false, + id: null, }; this.formValues = { @@ -95,7 +102,7 @@ class KubernetesDeployController { const metadata = { type: buildLabel(this.state.BuildMethod), format: formatLabel(this.state.DeployType), - role: roleLabel(this.Authentication.isAdmin()), + role: roleLabel(this.currentUser.isAdmin), 'automatic-updates': automaticUpdatesLabel(this.formValues.RepositoryAutomaticUpdates, this.formValues.RepositoryMechanism), }; @@ -183,9 +190,15 @@ class KubernetesDeployController { this.state.template = template; try { - const fileContent = await this.CustomTemplateService.customTemplateFile(templateId); - this.state.templateContent = fileContent; - this.onChangeFileContent(fileContent); + try { + this.state.templateContent = await this.CustomTemplateService.customTemplateFile(templateId, template.GitConfig !== null); + this.onChangeFileContent(this.state.templateContent); + + this.state.isEditorReadOnly = true; + } catch (err) { + this.state.templateLoadFailed = true; + throw err; + } if (template.Variables && template.Variables.length > 0) { const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, ''])); @@ -318,6 +331,9 @@ class KubernetesDeployController { $onInit() { return this.$async(async () => { + this.currentUser.isAdmin = this.Authentication.isAdmin(); + this.currentUser.id = this.Authentication.getUserDetails().ID; + this.formValues.namespace_toggle = false; await this.getNamespaces(); diff --git a/app/portainer/components/code-editor/codeEditorController.js b/app/portainer/components/code-editor/codeEditorController.js index e50b3b411..73e773436 100644 --- a/app/portainer/components/code-editor/codeEditorController.js +++ b/app/portainer/components/code-editor/codeEditorController.js @@ -1,8 +1,15 @@ angular.module('portainer.app').controller('CodeEditorController', function CodeEditorController($document, CodeMirrorService, $scope) { var ctrl = this; - this.$onChanges = function $onChanges({ value }) { - if (value && value.currentValue && ctrl.editor && ctrl.editor.getValue() !== value.currentValue) { + this.$onChanges = function $onChanges({ value, readOnly }) { + if (!ctrl.editor) { + return; + } + + if (readOnly && typeof readOnly.currentValue === 'boolean' && ctrl.editor.getValue('readOnly') !== ctrl.readOnly) { + ctrl.editor.setOption('readOnly', ctrl.readOnly); + } + if (value && value.currentValue && ctrl.editor.getValue() !== value.currentValue) { ctrl.editor.setValue(value.currentValue); } diff --git a/app/portainer/rest/customTemplate.js b/app/portainer/rest/customTemplate.js index 6bd45c26e..a6d00bbf1 100644 --- a/app/portainer/rest/customTemplate.js +++ b/app/portainer/rest/customTemplate.js @@ -13,6 +13,7 @@ function CustomTemplatesFactory($resource, API_ENDPOINT_CUSTOM_TEMPLATES) { update: { method: 'PUT', params: { id: '@id' } }, remove: { method: 'DELETE', params: { id: '@id' } }, file: { method: 'GET', params: { id: '@id', action: 'file' } }, + gitFetch: { method: 'PUT', params: { id: '@id', action: 'git_fetch' } }, } ); } diff --git a/app/portainer/services/api/customTemplate.js b/app/portainer/services/api/customTemplate.js index 641f4bcd4..dc6ff8646 100644 --- a/app/portainer/services/api/customTemplate.js +++ b/app/portainer/services/api/customTemplate.js @@ -1,4 +1,5 @@ import angular from 'angular'; +import PortainerError from 'Portainer/error'; angular.module('portainer.app').factory('CustomTemplateService', CustomTemplateServiceFactory); @@ -24,12 +25,12 @@ function CustomTemplateServiceFactory($sanitize, CustomTemplates, FileUploadServ return CustomTemplates.remove({ id }).$promise; }; - service.customTemplateFile = async function customTemplateFile(id) { + service.customTemplateFile = async function customTemplateFile(id, remote = false) { try { - const { FileContent } = await CustomTemplates.file({ id }).$promise; + const { FileContent } = remote ? await CustomTemplates.gitFetch({ id }).$promise : await CustomTemplates.file({ id }).$promise; return FileContent; } catch (err) { - throw { msg: 'Unable to retrieve customTemplate content', err }; + throw new PortainerError('Unable to retrieve custom template content', err); } }; diff --git a/app/portainer/views/custom-templates/custom-templates-view/customTemplatesView.html b/app/portainer/views/custom-templates/custom-templates-view/customTemplatesView.html index f31271dfe..d7882eb5d 100644 --- a/app/portainer/views/custom-templates/custom-templates-view/customTemplatesView.html +++ b/app/portainer/views/custom-templates/custom-templates-view/customTemplatesView.html @@ -18,16 +18,27 @@ on-change="($ctrl.onChangeTemplateVariables)" > -

+
- Customize stack + {{ $ctrl.state.selectedTemplate.GitConfig !== null ? 'View' : 'Customize' }} stack - Hide custom stack + Hide {{ $ctrl.state.selectedTemplate.GitConfig === null ? 'custom' : '' }} stack
+ + +

+ Custom template could not be loaded, please + click here for configuration.

+

+ Custom template could not be loaded, please contact your administrator.

+ +

diff --git a/app/portainer/views/custom-templates/custom-templates-view/customTemplatesViewController.js b/app/portainer/views/custom-templates/custom-templates-view/customTemplatesViewController.js index 3baff508e..4aeac7ac9 100644 --- a/app/portainer/views/custom-templates/custom-templates-view/customTemplatesViewController.js +++ b/app/portainer/views/custom-templates/custom-templates-view/customTemplatesViewController.js @@ -44,10 +44,10 @@ class CustomTemplatesViewController { showAdvancedOptions: false, formValidationError: '', actionInProgress: false, - isEditorVisible: false, deployable: false, templateNameRegex: TEMPLATE_NAME_VALIDATION_REGEX, templateContent: '', + templateLoadFailed: false, }; this.currentUser = { @@ -204,6 +204,13 @@ class CustomTemplatesViewController { template.Selected = true; + try { + this.state.templateContent = this.formValues.fileContent = await this.CustomTemplateService.customTemplateFile(template.Id, template.GitConfig !== null); + } catch (err) { + this.state.templateLoadFailed = true; + this.Notifications.error('Failure', err, 'Unable to retrieve custom template data'); + } + this.formValues.network = _.find(this.availableNetworks, function (o) { return o.Name === 'bridge'; }); @@ -213,9 +220,6 @@ class CustomTemplatesViewController { this.$anchorScroll('view-top'); const applicationState = this.StateManager.getState(); this.state.deployable = this.isDeployable(applicationState.endpoint, template.Type); - const file = await this.CustomTemplateService.customTemplateFile(template.Id); - this.state.templateContent = file; - this.formValues.fileContent = file; if (template.Variables && template.Variables.length > 0) { const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, ''])); diff --git a/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateView.html b/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateView.html index 25e814d0e..4028d85f8 100644 --- a/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateView.html +++ b/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateView.html @@ -7,6 +7,22 @@ + + +

+
+
+
+

+ + Custom template could not be loaded, {{ $ctrl.state.templatePreviewError }}.

+
+
+

@@ -42,7 +59,11 @@ is-variables-names-from-parent="true" > - +

Actions
diff --git a/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateViewController.js b/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateViewController.js index 54353d2a8..f30fc8d9b 100644 --- a/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateViewController.js +++ b/app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateViewController.js @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { getFilePreview } from '@/react/portainer/gitops/gitops.service'; import { ResourceControlViewModel } from '@/react/portainer/access-control/models/ResourceControlViewModel'; import { AccessControlFormData } from 'Portainer/components/accessControlForm/porAccessControlFormModel'; @@ -13,11 +14,18 @@ class EditCustomTemplateViewController { this.isTemplateVariablesEnabled = isBE; - this.formValues = null; + this.formValues = { + Variables: [], + TLSSkipVerify: false, + }; this.state = { formValidationError: '', isEditorDirty: false, isTemplateValid: true, + isEditorReadOnly: false, + templateLoadFailed: false, + templatePreviewFailed: false, + templatePreviewError: '', }; this.templates = []; @@ -28,6 +36,7 @@ class EditCustomTemplateViewController { this.editorUpdate = this.editorUpdate.bind(this); this.onVariablesChange = this.onVariablesChange.bind(this); this.handleChange = this.handleChange.bind(this); + this.previewFileFromGitRepository = this.previewFileFromGitRepository.bind(this); } getTemplate() { @@ -35,14 +44,25 @@ class EditCustomTemplateViewController { } async getTemplateAsync() { try { - const [template, file] = await Promise.all([ - this.CustomTemplateService.customTemplate(this.$state.params.id), - this.CustomTemplateService.customTemplateFile(this.$state.params.id), - ]); - template.FileContent = file; + const template = await this.CustomTemplateService.customTemplate(this.$state.params.id); + + if (template.GitConfig !== null) { + this.state.isEditorReadOnly = true; + } + + try { + template.FileContent = await this.CustomTemplateService.customTemplateFile(this.$state.params.id, template.GitConfig !== null); + } catch (err) { + this.state.templateLoadFailed = true; + throw err; + } + template.Variables = template.Variables || []; - this.formValues = template; + + this.formValues = { ...this.formValues, ...template }; + this.parseTemplate(template.FileContent); + this.parseGitConfig(template.GitConfig); this.oldFileContent = this.formValues.FileContent; if (template.ResourceControl) { @@ -145,6 +165,62 @@ class EditCustomTemplateViewController { } } + parseGitConfig(config) { + if (config === null) { + return; + } + + let flatConfig = { + RepositoryURL: config.URL, + RepositoryReferenceName: config.ReferenceName, + ComposeFilePathInRepository: config.ConfigFilePath, + RepositoryAuthentication: config.Authentication !== null, + TLSSkipVerify: config.TLSSkipVerify, + }; + + if (config.Authentication) { + flatConfig = { + ...flatConfig, + RepositoryUsername: config.Authentication.Username, + RepositoryPassword: config.Authentication.Password, + }; + } + + this.formValues = { ...this.formValues, ...flatConfig }; + } + + previewFileFromGitRepository() { + this.state.templatePreviewFailed = false; + this.state.templatePreviewError = ''; + + let creds = {}; + if (this.formValues.RepositoryAuthentication) { + creds = { + username: this.formValues.RepositoryUsername, + password: this.formValues.RepositoryPassword, + }; + } + const payload = { + repository: this.formValues.RepositoryURL, + targetFile: this.formValues.ComposeFilePathInRepository, + tlsSkipVerify: this.formValues.TLSSkipVerify, + ...creds, + }; + + this.$async(async () => { + try { + this.formValues.FileContent = await getFilePreview(payload); + this.state.isEditorDirty = true; + + // check if the template contains mustache template symbol + this.parseTemplate(this.formValues.FileContent); + } catch (err) { + this.state.templatePreviewError = err.message; + this.state.templatePreviewFailed = true; + } + }); + } + async uiCanExit() { if (this.formValues.FileContent !== this.oldFileContent && this.state.isEditorDirty) { return confirmWebEditorDiscard(); diff --git a/app/portainer/views/stacks/create/createStackController.js b/app/portainer/views/stacks/create/createStackController.js index f7acd9de7..23491d1fe 100644 --- a/app/portainer/views/stacks/create/createStackController.js +++ b/app/portainer/views/stacks/create/createStackController.js @@ -39,7 +39,6 @@ angular $scope.stackWebhookFeature = FeatureId.STACK_WEBHOOK; $scope.buildMethods = [editor, upload, git, customTemplate]; $scope.STACK_NAME_VALIDATION_REGEX = STACK_NAME_VALIDATION_REGEX; - $scope.isAdmin = Authentication.isAdmin(); $scope.formValues = { Name: '', @@ -72,6 +71,13 @@ angular selectedTemplateId: null, baseWebhookUrl: baseStackWebhookUrl(), webhookId: createWebhookId(), + templateLoadFailed: false, + isEditorReadOnly: false, + }; + + $scope.currentUser = { + isAdmin: false, + id: null, }; $window.onbeforeunload = () => { @@ -296,9 +302,15 @@ angular $scope.state.selectedTemplateId = templateId; $scope.state.selectedTemplate = template; - const fileContent = await CustomTemplateService.customTemplateFile(templateId); - $scope.state.templateContent = fileContent; - onChangeFileContent(fileContent); + try { + $scope.state.templateContent = await this.CustomTemplateService.customTemplateFile(templateId, template.GitConfig !== null); + onChangeFileContent($scope.state.templateContent); + + $scope.state.isEditorReadOnly = true; + } catch (err) { + $scope.state.templateLoadFailed = true; + throw err; + } if (template.Variables && template.Variables.length > 0) { const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, ''])); @@ -321,6 +333,9 @@ angular } async function initView() { + $scope.currentUser.isAdmin = Authentication.isAdmin(); + $scope.currentUser.id = Authentication.getUserDetails().ID; + var endpointMode = $scope.applicationState.endpoint.mode; $scope.state.StackType = 2; $scope.isDockerStandalone = endpointMode.provider === 'DOCKER_STANDALONE'; diff --git a/app/portainer/views/stacks/create/createstack.html b/app/portainer/views/stacks/create/createstack.html index fb3dfb9c0..901b65bfc 100644 --- a/app/portainer/views/stacks/create/createstack.html +++ b/app/portainer/views/stacks/create/createstack.html @@ -91,19 +91,31 @@ >
- +
+ - + + + +

+ Custom template could not be loaded, please + click here for configuration.

+

+ Custom template could not be loaded, please contact your administrator.

+ +

diff --git a/app/react/portainer/gitops/gitops.service.ts b/app/react/portainer/gitops/gitops.service.ts new file mode 100644 index 000000000..9b7dd5b25 --- /dev/null +++ b/app/react/portainer/gitops/gitops.service.ts @@ -0,0 +1,25 @@ +import axios, { parseAxiosError } from '@/portainer/services/axios'; + +interface PreviewPayload { + repository: string; + targetFile: string; + reference?: string; + username?: string; + password?: string; + tlsSkipVerify?: boolean; +} + +interface PreviewResponse { + FileContent: string; +} + +export async function getFilePreview(payload: PreviewPayload) { + try { + const { + data: { FileContent }, + } = await axios.post('/gitops/repo/file/preview', payload); + return FileContent; + } catch (e) { + throw parseAxiosError(e as Error, 'Unable to fetch file from git'); + } +} diff --git a/app/react/portainer/gitops/queries/useSearch.ts b/app/react/portainer/gitops/queries/useSearch.ts index 4a026422f..2cc0b6314 100644 --- a/app/react/portainer/gitops/queries/useSearch.ts +++ b/app/react/portainer/gitops/queries/useSearch.ts @@ -8,6 +8,7 @@ interface SearchPayload { reference?: string; username?: string; password?: string; + tlsSkipVerify?: boolean; } export function useSearch(payload: SearchPayload, enabled: boolean) {