github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(templates): allow managing git based templates [EE-2600] (#7855) 

Co-authored-by: itsconquest <william.conquest@portainer.io>
Co-authored-by: oscarzhou <oscar.zhou@portainer.io>
Co-authored-by: Chaim Lev-Ari <chiptus@users.noreply.github.com>
pull/8738/head
Oscar Zhou 2023-04-04 12:44:42 +12:00 committed by GitHub
parent 30a2bb0495
commit c650868fe9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 944 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
api/http/handler/customtemplates/customtemplate_create.go
View File

@ -3,7 +3,6 @@ package customtemplates
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"regexp"
@ -18,6 +17,7 @@ import (
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/rs/zerolog/log"
)
@ -135,6 +135,7 @@ func (payload *customTemplateFromFileContentPayload) Validate(r *http.Request) e
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if payload.Type != portainer.KubernetesStack && payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
return errors.New("Invalid custom template type")
}
@ -215,6 +216,8 @@ type customTemplateFromGitRepositoryPayload struct {
Variables []portainer.CustomTemplateVariableDefinition
// TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"`
// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
IsComposeFormat bool `example:"false"`
}
func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request) error {
@ -234,14 +237,11 @@ func (payload *customTemplateFromGitRepositoryPayload) Validate(r *http.Request)
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}
if payload.Type == portainer.KubernetesStack {
return errors.New("Creating a Kubernetes custom template from git is not supported")
}
if payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}
if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack {
if payload.Type != portainer.DockerSwarmStack && payload.Type != portainer.DockerComposeStack && payload.Type != portainer.KubernetesStack {
return errors.New("Invalid custom template type")
}
if !isValidNote(payload.Note) {
@ -260,35 +260,44 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
customTemplateID := handler.DataStore.CustomTemplate().GetNextIdentifier()
customTemplate := &portainer.CustomTemplate{
ID: portainer.CustomTemplateID(customTemplateID),
Title: payload.Title,
EntryPoint: payload.ComposeFilePathInRepository,
Description: payload.Description,
Note: payload.Note,
Platform: payload.Platform,
Type: payload.Type,
Logo: payload.Logo,
Variables: payload.Variables,
ID: portainer.CustomTemplateID(customTemplateID),
Title: payload.Title,
Description: payload.Description,
Note: payload.Note,
Platform: payload.Platform,
Type: payload.Type,
Logo: payload.Logo,
Variables: payload.Variables,
IsComposeFormat: payload.IsComposeFormat,
}
projectPath := handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
getProjectPath := func() string {
return handler.FileService.GetCustomTemplateProjectPath(strconv.Itoa(customTemplateID))
}
projectPath := getProjectPath()
customTemplate.ProjectPath = projectPath
repositoryUsername := payload.RepositoryUsername
repositoryPassword := payload.RepositoryPassword
if !payload.RepositoryAuthentication {
repositoryUsername = ""
repositoryPassword = ""
gitConfig := &gittypes.RepoConfig{
URL: payload.RepositoryURL,
ReferenceName: payload.RepositoryReferenceName,
ConfigFilePath: payload.ComposeFilePathInRepository,
}
err = handler.GitService.CloneRepository(projectPath, payload.RepositoryURL, payload.RepositoryReferenceName, repositoryUsername, repositoryPassword, payload.TLSSkipVerify)
if err != nil {
if err == gittypes.ErrAuthenticationFailure {
return nil, fmt.Errorf("invalid git credential")
if payload.RepositoryAuthentication {
gitConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
}
}
commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, getProjectPath)
if err != nil {
return nil, err
}
gitConfig.ConfigHash = commitHash
customTemplate.GitConfig = gitConfig
isValidProject := true
defer func() {
if !isValidProject {
@ -298,7 +307,7 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
}
}()
entryPath := filesystem.JoinPaths(projectPath, customTemplate.EntryPoint)
entryPath := filesystem.JoinPaths(projectPath, gitConfig.ConfigFilePath)
exists, err := handler.FileService.FileExists(entryPath)
if err != nil || !exists {
@ -310,6 +319,9 @@ func (handler *Handler) createCustomTemplateFromGitRepository(r *http.Request) (
}
if !exists {
if payload.Type == portainer.KubernetesStack {
return nil, errors.New("Invalid Manifest file, ensure that the Manifest file path is correct")
}
return nil, errors.New("Invalid Compose file, ensure that the Compose file path is correct")
}
@ -369,6 +381,7 @@ func (payload *customTemplateFromFileUploadPayload) Validate(r *http.Request) er
platform, _ := request.RetrieveNumericMultiPartFormValue(r, "Platform", true)
templatePlatform := portainer.CustomTemplatePlatform(platform)
// Platform validation is only for docker related stack (docker standalone and docker swarm)
if templateType != portainer.KubernetesStack && templatePlatform != portainer.CustomTemplatePlatformLinux && templatePlatform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
}

6
api/http/handler/customtemplates/customtemplate_file.go
View File

@ -40,7 +40,11 @@ func (handler *Handler) customTemplateFile(w http.ResponseWriter, r *http.Reques
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.EntryPoint)
entryPath := customTemplate.EntryPoint
if customTemplate.GitConfig != nil {
entryPath = customTemplate.GitConfig.ConfigFilePath
}
fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, entryPath)
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
}

124
api/http/handler/customtemplates/customtemplate_git_fetch.go Normal file
View File

@ -0,0 +1,124 @@
package customtemplates
import (
"fmt"
"net/http"
"os"
"sync"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/stacks/stackutils"
"github.com/rs/zerolog/log"
)
// @id CustomTemplateGitFetch
// @summary Fetch the latest config file content based on custom template's git repository configuration
// @description Retrieve details about a template created from git repository method.
// @description **Access policy**: authenticated
// @tags custom_templates
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param id path int true "Template identifier"
// @success 200 {object} fileResponse "Success"
// @failure 400 "Invalid request"
// @failure 404 "Custom template not found"
// @failure 500 "Server error"
// @router /custom_templates/{id}/git_fetch [put]
func (handler *Handler) customTemplateGitFetch(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
customTemplateID, err := request.RetrieveNumericRouteVariableValue(r, "id")
if err != nil {
return httperror.BadRequest("Invalid Custom template identifier route variable", err)
}
customTemplate, err := handler.DataStore.CustomTemplate().CustomTemplate(portainer.CustomTemplateID(customTemplateID))
if handler.DataStore.IsErrObjectNotFound(err) {
return httperror.NotFound("Unable to find a custom template with the specified identifier inside the database", err)
} else if err != nil {
return httperror.InternalServerError("Unable to find a custom template with the specified identifier inside the database", err)
}
if customTemplate.GitConfig == nil {
return httperror.BadRequest("Git configuration does not exist in this custom template", err)
}
// If multiple users are trying to fetch the same custom template simultaneously, a lock needs to be added
mu, ok := handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)]
if !ok {
mu = &sync.Mutex{}
handler.gitFetchMutexs[portainer.TemplateID(customTemplateID)] = mu
}
mu.Lock()
defer mu.Unlock()
// back up the current custom template folder
backupPath, err := backupCustomTemplate(customTemplate.ProjectPath)
if err != nil {
return httperror.InternalServerError("Failed to backup the custom template folder", err)
}
// remove backup custom template folder
defer cleanUpBackupCustomTemplate(backupPath)
commitHash, err := stackutils.DownloadGitRepository(*customTemplate.GitConfig, handler.GitService, func() string {
return customTemplate.ProjectPath
})
if err != nil {
log.Warn().Err(err).Msg("failed to download git repository")
rbErr := rollbackCustomTemplate(backupPath, customTemplate.ProjectPath)
if err != nil {
return httperror.InternalServerError("Failed to rollback the custom template folder", rbErr)
}
return httperror.InternalServerError("Failed to download git repository", err)
}
if customTemplate.GitConfig.ConfigHash != commitHash {
customTemplate.GitConfig.ConfigHash = commitHash
err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
if err != nil {
return httperror.InternalServerError("Unable to persist custom template changes inside the database", err)
}
}
fileContent, err := handler.FileService.GetFileContent(customTemplate.ProjectPath, customTemplate.GitConfig.ConfigFilePath)
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
}
return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
}
func backupCustomTemplate(projectPath string) (string, error) {
stat, err := os.Stat(projectPath)
if err != nil {
return "", err
}
backupPath := fmt.Sprintf("%s-backup", projectPath)
err = os.Rename(projectPath, backupPath)
if err != nil {
return "", err
}
err = os.Mkdir(projectPath, stat.Mode())
if err != nil {
return backupPath, err
}
return backupPath, nil
}
func rollbackCustomTemplate(backupPath, projectPath string) error {
err := os.RemoveAll(projectPath)
if err != nil {
return err
}
return os.Rename(backupPath, projectPath)
}
func cleanUpBackupCustomTemplate(backupPath string) error {
return os.RemoveAll(backupPath)
}

174
api/http/handler/customtemplates/customtemplate_git_fetch_test.go Normal file
View File

@ -0,0 +1,174 @@
package customtemplates
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/fs"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"testing"
"time"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/datastore"
gittypes "github.com/portainer/portainer/api/git/types"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/internal/authorization"
"github.com/portainer/portainer/api/jwt"
"github.com/stretchr/testify/assert"
)
var testFileContent string = "abcdefg"
type TestGitService struct {
portainer.GitService
targetFilePath string
}
func (g *TestGitService) CloneRepository(destination string, repositoryURL, referenceName string, username, password string, tlsSkipVerify bool) error {
time.Sleep(100 * time.Millisecond)
return createTestFile(g.targetFilePath)
}
func (g *TestGitService) LatestCommitID(repositoryURL, referenceName, username, password string, tlsSkipVerify bool) (string, error) {
return "", nil
}
type TestFileService struct {
portainer.FileService
}
func (f *TestFileService) GetFileContent(projectPath, configFilePath string) ([]byte, error) {
return os.ReadFile(filepath.Join(projectPath, configFilePath))
}
func createTestFile(targetPath string) error {
f, err := os.Create(targetPath)
if err != nil {
return err
}
defer f.Close()
_, err = f.WriteString(testFileContent)
return err
}
func prepareTestFolder(projectPath, filename string) error {
err := os.MkdirAll(projectPath, fs.ModePerm)
if err != nil {
return err
}
return createTestFile(filepath.Join(projectPath, filename))
}
func singleAPIRequest(h *Handler, jwt string, is *assert.Assertions, expect string) {
type response struct {
FileContent string
}
req := httptest.NewRequest(http.MethodPut, "/custom_templates/1/git_fetch", bytes.NewBuffer([]byte("{}")))
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", jwt))
rr := httptest.NewRecorder()
h.ServeHTTP(rr, req)
is.Equal(http.StatusOK, rr.Code)
body, err := io.ReadAll(rr.Body)
is.NoError(err, "ReadAll should not return error")
var resp response
err = json.Unmarshal(body, &resp)
is.NoError(err, "response should be list json")
is.Equal(resp.FileContent, expect)
}
func Test_customTemplateGitFetch(t *testing.T) {
is := assert.New(t)
_, store, teardown := datastore.MustNewTestStore(t, true, true)
defer teardown()
// create user(s)
user1 := &portainer.User{ID: 1, Username: "user-1", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
err := store.User().Create(user1)
is.NoError(err, "error creating user 1")
user2 := &portainer.User{ID: 2, Username: "user-2", Role: portainer.StandardUserRole, PortainerAuthorizations: authorization.DefaultPortainerAuthorizations()}
err = store.User().Create(user2)
is.NoError(err, "error creating user 2")
dir, err := os.Getwd()
is.NoError(err, "error to get working directory")
template1 := &portainer.CustomTemplate{ID: 1, Title: "custom-template-1", ProjectPath: filepath.Join(dir, "fixtures/custom_template_1"), GitConfig: &gittypes.RepoConfig{ConfigFilePath: "test-config-path.txt"}}
err = store.CustomTemplateService.Create(template1)
is.NoError(err, "error creating custom template 1")
// prepare testing folder
err = prepareTestFolder(template1.ProjectPath, template1.GitConfig.ConfigFilePath)
is.NoError(err, "error creating testing folder")
defer os.RemoveAll(filepath.Join(dir, "fixtures"))
// setup services
jwtService, err := jwt.NewService("1h", store)
is.NoError(err, "Error initiating jwt service")
requestBouncer := security.NewRequestBouncer(store, jwtService, nil)
gitService := &TestGitService{
targetFilePath: filepath.Join(template1.ProjectPath, template1.GitConfig.ConfigFilePath),
}
fileService := &TestFileService{}
h := NewHandler(requestBouncer, store, fileService, gitService)
// generate two standard users' tokens
jwt1, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user1.ID, Username: user1.Username, Role: user1.Role})
jwt2, _ := jwtService.GenerateToken(&portainer.TokenData{ID: user2.ID, Username: user2.Username, Role: user2.Role})
t.Run("can return the expected file content by a single call from one user", func(t *testing.T) {
singleAPIRequest(h, jwt1, is, "abcdefg")
})
t.Run("can return the expected file content by multiple calls from one user", func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(5)
for i := 0; i < 5; i++ {
go func() {
singleAPIRequest(h, jwt1, is, "abcdefg")
wg.Done()
}()
}
wg.Wait()
})
t.Run("can return the expected file content by multiple calls from different users", func(t *testing.T) {
var wg sync.WaitGroup
wg.Add(10)
for i := 0; i < 10; i++ {
go func(j int) {
if j%1 == 0 {
singleAPIRequest(h, jwt1, is, "abcdefg")
} else {
singleAPIRequest(h, jwt2, is, "abcdefg")
}
wg.Done()
}(i)
}
wg.Wait()
})
t.Run("can return the expected file content after a new commit is made", func(t *testing.T) {
singleAPIRequest(h, jwt1, is, "abcdefg")
testFileContent = "gfedcba"
singleAPIRequest(h, jwt2, is, "gfedcba")
})
}

82
api/http/handler/customtemplates/customtemplate_update.go
View File

@ -10,8 +10,11 @@ import (
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/filesystem"
gittypes "github.com/portainer/portainer/api/git/types"
httperrors "github.com/portainer/portainer/api/http/errors"
"github.com/portainer/portainer/api/http/security"
"github.com/portainer/portainer/api/stacks/stackutils"
)
type customTemplateUpdatePayload struct {
@ -29,18 +32,37 @@ type customTemplateUpdatePayload struct {
Platform portainer.CustomTemplatePlatform `example:"1" enums:"1,2"`
// Type of created stack (1 - swarm, 2 - compose, 3 - kubernetes)
Type portainer.StackType `example:"1" enums:"1,2,3" validate:"required"`
// URL of a Git repository hosting the Stack file
RepositoryURL string `example:"https://github.com/openfaas/faas" validate:"required"`
// Reference name of a Git repository hosting the Stack file
RepositoryReferenceName string `example:"refs/heads/master"`
// Use basic authentication to clone the Git repository
RepositoryAuthentication bool `example:"true"`
// Username used in basic authentication. Required when RepositoryAuthentication is true
// and RepositoryGitCredentialID is 0
RepositoryUsername string `example:"myGitUsername"`
// Password used in basic authentication. Required when RepositoryAuthentication is true
// and RepositoryGitCredentialID is 0
RepositoryPassword string `example:"myGitPassword"`
// GitCredentialID used to identify the bound git credential. Required when RepositoryAuthentication
// is true and RepositoryUsername/RepositoryPassword are not provided
RepositoryGitCredentialID int `example:"0"`
// Path to the Stack file inside the Git repository
ComposeFilePathInRepository string `example:"docker-compose.yml" default:"docker-compose.yml"`
// Content of stack file
FileContent string `validate:"required"`
// Definitions of variables in the stack file
Variables []portainer.CustomTemplateVariableDefinition
// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
IsComposeFormat bool `example:"false"`
}
func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Title) {
return errors.New("Invalid custom template title")
}
if govalidator.IsNull(payload.FileContent) {
return errors.New("Invalid file content")
if govalidator.IsNull(payload.FileContent) && govalidator.IsNull(payload.RepositoryURL) {
return errors.New("Either file content or git repository url need to be provided")
}
if payload.Type != portainer.KubernetesStack && payload.Platform != portainer.CustomTemplatePlatformLinux && payload.Platform != portainer.CustomTemplatePlatformWindows {
return errors.New("Invalid custom template platform")
@ -55,7 +77,19 @@ func (payload *customTemplateUpdatePayload) Validate(r *http.Request) error {
return errors.New("Invalid note. <img> tag is not supported")
}
return validateVariablesDefinitions(payload.Variables)
if payload.RepositoryAuthentication && (govalidator.IsNull(payload.RepositoryUsername) || govalidator.IsNull(payload.RepositoryPassword)) {
return errors.New("Invalid repository credentials. Username and password must be specified when authentication is enabled")
}
if govalidator.IsNull(payload.ComposeFilePathInRepository) {
payload.ComposeFilePathInRepository = filesystem.ComposeFileDefaultName
}
err := validateVariablesDefinitions(payload.Variables)
if err != nil {
return err
}
return nil
}
// @id CustomTemplateUpdate
@ -115,12 +149,6 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
return httperror.Forbidden("Access denied to resource", httperrors.ErrResourceAccessDenied)
}
templateFolder := strconv.Itoa(customTemplateID)
_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
if err != nil {
return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
}
customTemplate.Title = payload.Title
customTemplate.Logo = payload.Logo
customTemplate.Description = payload.Description
@ -128,6 +156,42 @@ func (handler *Handler) customTemplateUpdate(w http.ResponseWriter, r *http.Requ
customTemplate.Platform = payload.Platform
customTemplate.Type = payload.Type
customTemplate.Variables = payload.Variables
customTemplate.IsComposeFormat = payload.IsComposeFormat
if payload.RepositoryURL != "" {
if !govalidator.IsURL(payload.RepositoryURL) {
return httperror.BadRequest("Invalid repository URL. Must correspond to a valid URL format", err)
}
gitConfig := &gittypes.RepoConfig{
URL: payload.RepositoryURL,
ReferenceName: payload.RepositoryReferenceName,
ConfigFilePath: payload.ComposeFilePathInRepository,
}
if payload.RepositoryAuthentication {
gitConfig.Authentication = &gittypes.GitAuthentication{
Username: payload.RepositoryUsername,
Password: payload.RepositoryPassword,
}
}
commitHash, err := stackutils.DownloadGitRepository(*gitConfig, handler.GitService, func() string {
return customTemplate.ProjectPath
})
if err != nil {
return httperror.InternalServerError(err.Error(), err)
}
gitConfig.ConfigHash = commitHash
customTemplate.GitConfig = gitConfig
} else {
templateFolder := strconv.Itoa(customTemplateID)
_, err = handler.FileService.StoreCustomTemplateFileFromBytes(templateFolder, customTemplate.EntryPoint, []byte(payload.FileContent))
if err != nil {
return httperror.InternalServerError("Unable to persist updated custom template file on disk", err)
}
}
err = handler.DataStore.CustomTemplate().UpdateCustomTemplate(customTemplate.ID, customTemplate)
if err != nil {

18
api/http/handler/customtemplates/handler.go
View File

@ -2,6 +2,7 @@ package customtemplates
import (
"net/http"
"sync"
"github.com/gorilla/mux"
httperror "github.com/portainer/libhttp/error"
@ -13,15 +14,20 @@ import (
// Handler is the HTTP handler used to handle environment(endpoint) group operations.
type Handler struct {
*mux.Router
DataStore dataservices.DataStore
FileService portainer.FileService
GitService portainer.GitService
DataStore dataservices.DataStore
FileService portainer.FileService
GitService portainer.GitService
gitFetchMutexs map[portainer.TemplateID]*sync.Mutex
}
// NewHandler creates a handler to manage environment(endpoint) group operations.
func NewHandler(bouncer *security.RequestBouncer) *Handler {
func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, fileService portainer.FileService, gitService portainer.GitService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
Router: mux.NewRouter(),
DataStore: dataStore,
FileService: fileService,
GitService: gitService,
gitFetchMutexs: make(map[portainer.TemplateID]*sync.Mutex),
}
h.Handle("/custom_templates",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateCreate))).Methods(http.MethodPost)
@ -35,6 +41,8 @@ func NewHandler(bouncer *security.RequestBouncer) *Handler {
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateUpdate))).Methods(http.MethodPut)
h.Handle("/custom_templates/{id}",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateDelete))).Methods(http.MethodDelete)
h.Handle("/custom_templates/{id}/git_fetch",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.customTemplateGitFetch))).Methods(http.MethodPut)
return h
}

89
api/http/handler/gitops/git_repo_file_preview.go Normal file
View File

@ -0,0 +1,89 @@
package gitops
import (
"errors"
"fmt"
"net/http"
"github.com/asaskevich/govalidator"
httperror "github.com/portainer/libhttp/error"
"github.com/portainer/libhttp/request"
"github.com/portainer/libhttp/response"
gittypes "github.com/portainer/portainer/api/git/types"
)
type fileResponse struct {
FileContent string
}
type repositoryFilePreviewPayload struct {
Repository string `json:"repository" example:"https://github.com/openfaas/faas" validate:"required"`
Reference string `json:"reference" example:"refs/heads/master"`
Username string `json:"username" example:"myGitUsername"`
Password string `json:"password" example:"myGitPassword"`
// Path to file whose content will be read
TargetFile string `json:"targetFile" example:"docker-compose.yml"`
// TLSSkipVerify skips SSL verification when cloning the Git repository
TLSSkipVerify bool `example:"false"`
}
func (payload *repositoryFilePreviewPayload) Validate(r *http.Request) error {
if govalidator.IsNull(payload.Repository) || !govalidator.IsURL(payload.Repository) {
return errors.New("Invalid repository URL. Must correspond to a valid URL format")
}
if govalidator.IsNull(payload.Reference) {
payload.Reference = "refs/heads/main"
}
if govalidator.IsNull(payload.TargetFile) {
return errors.New("Invalid target filename.")
}
return nil
}
// @id GitOperationRepoFilePreview
// @summary preview the content of target file in the git repository
// @description Retrieve the compose file content based on git repository configuration
// @description **Access policy**: authenticated
// @tags gitops
// @security ApiKeyAuth
// @security jwt
// @produce json
// @param body body repositoryFilePreviewPayload true "Template details"
// @success 200 {object} fileResponse "Success"
// @failure 400 "Invalid request"
// @failure 500 "Server error"
// @router /gitops/repo/file/preview [post]
func (handler *Handler) gitOperationRepoFilePreview(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload repositoryFilePreviewPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
}
projectPath, err := handler.fileService.GetTemporaryPath()
if err != nil {
return httperror.InternalServerError("Unable to create temporary folder", err)
}
err = handler.gitService.CloneRepository(projectPath, payload.Repository, payload.Reference, payload.Username, payload.Password, payload.TLSSkipVerify)
if err != nil {
if err == gittypes.ErrAuthenticationFailure {
return httperror.BadRequest("Invalid git credential", err)
}
newErr := fmt.Errorf("unable to clone git repository: %w", err)
return httperror.InternalServerError(newErr.Error(), newErr)
}
defer handler.fileService.RemoveDirectory(projectPath)
fileContent, err := handler.fileService.GetFileContent(projectPath, payload.TargetFile)
if err != nil {
return httperror.InternalServerError("Unable to retrieve custom template file from disk", err)
}
return response.JSON(w, &fileResponse{FileContent: string(fileContent)})
}

33
api/http/handler/gitops/handler.go Normal file
View File

@ -0,0 +1,33 @@
package gitops
import (
"net/http"
"github.com/gorilla/mux"
httperror "github.com/portainer/libhttp/error"
portainer "github.com/portainer/portainer/api"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/http/security"
)
// Handler is the HTTP handler used to handle git repo operation
type Handler struct {
*mux.Router
dataStore dataservices.DataStore
gitService portainer.GitService
fileService portainer.FileService
}
func NewHandler(bouncer *security.RequestBouncer, dataStore dataservices.DataStore, gitService portainer.GitService, fileService portainer.FileService) *Handler {
h := &Handler{
Router: mux.NewRouter(),
dataStore: dataStore,
gitService: gitService,
fileService: fileService,
}
h.Handle("/gitops/repo/file/preview",
bouncer.AuthenticatedAccess(httperror.LoggerHandler(h.gitOperationRepoFilePreview))).Methods(http.MethodPost)
return h
}

6
api/http/handler/handler.go
View File

@ -17,6 +17,7 @@ import (
"github.com/portainer/portainer/api/http/handler/endpointproxy"
"github.com/portainer/portainer/api/http/handler/endpoints"
"github.com/portainer/portainer/api/http/handler/file"
"github.com/portainer/portainer/api/http/handler/gitops"
"github.com/portainer/portainer/api/http/handler/helm"
"github.com/portainer/portainer/api/http/handler/hostmanagement/fdo"
"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
@ -56,6 +57,7 @@ type Handler struct {
EndpointHandler *endpoints.Handler
EndpointHelmHandler *helm.Handler
EndpointProxyHandler *endpointproxy.Handler
GitOperationHandler *gitops.Handler
HelmTemplatesHandler *helm.Handler
KubernetesHandler *kubernetes.Handler
FileHandler *file.Handler
@ -121,6 +123,8 @@ type Handler struct {
// @tag.description Manage Docker environments(endpoints)
// @tag.name endpoint_groups
// @tag.description Manage environment(endpoint) groups
// @tag.name gitops
// @tag.description Operate git repository
// @tag.name kubernetes
// @tag.description Manage Kubernetes cluster
// @tag.name motd
@ -203,6 +207,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
default:
http.StripPrefix("/api", h.EndpointHandler).ServeHTTP(w, r)
}
case strings.HasPrefix(r.URL.Path, "/api/gitops"):
http.StripPrefix("/api", h.GitOperationHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/ldap"):
http.StripPrefix("/api", h.LDAPHandler).ServeHTTP(w, r)
case strings.HasPrefix(r.URL.Path, "/api/motd"):

7
api/http/handler/stacks/create_kubernetes_stack.go
View File

@ -23,14 +23,17 @@ type kubernetesStringDeploymentPayload struct {
ComposeFormat bool
Namespace string
StackFileContent string
// Whether the stack is from a app template
FromAppTemplate bool `example:"false"`
}
func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat bool) stackbuilders.StackPayload {
func createStackPayloadFromK8sFileContentPayload(name, namespace, fileContent string, composeFormat, fromAppTemplate bool) stackbuilders.StackPayload {
return stackbuilders.StackPayload{
StackName: name,
Namespace: namespace,
StackFileContent: fileContent,
ComposeFormat: composeFormat,
FromAppTemplate: fromAppTemplate,
}
}
@ -146,7 +149,7 @@ func (handler *Handler) createKubernetesStackFromFileContent(w http.ResponseWrit
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A stack with the name '%s' already exists", payload.StackName), Err: stackutils.ErrStackAlreadyExists}
}
stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat)
stackPayload := createStackPayloadFromK8sFileContentPayload(payload.StackName, payload.Namespace, payload.StackFileContent, payload.ComposeFormat, payload.FromAppTemplate)
k8sStackBuilder := stackbuilders.CreateK8sStackFileContentBuilder(handler.DataStore,
handler.FileService,

9
api/http/server.go
View File

@ -28,6 +28,7 @@ import (
"github.com/portainer/portainer/api/http/handler/endpointproxy"
"github.com/portainer/portainer/api/http/handler/endpoints"
"github.com/portainer/portainer/api/http/handler/file"
"github.com/portainer/portainer/api/http/handler/gitops"
"github.com/portainer/portainer/api/http/handler/helm"
"github.com/portainer/portainer/api/http/handler/hostmanagement/fdo"
"github.com/portainer/portainer/api/http/handler/hostmanagement/openamt"
@ -143,10 +144,7 @@ func (server *Server) Start() error {
var roleHandler = roles.NewHandler(requestBouncer)
roleHandler.DataStore = server.DataStore
var customTemplatesHandler = customtemplates.NewHandler(requestBouncer)
customTemplatesHandler.DataStore = server.DataStore
customTemplatesHandler.FileService = server.FileService
customTemplatesHandler.GitService = server.GitService
var customTemplatesHandler = customtemplates.NewHandler(requestBouncer, server.DataStore, server.FileService, server.GitService)
var edgeGroupsHandler = edgegroups.NewHandler(requestBouncer)
edgeGroupsHandler.DataStore = server.DataStore
@ -196,6 +194,8 @@ func (server *Server) Start() error {
var endpointHelmHandler = helm.NewHandler(requestBouncer, server.DataStore, server.JWTService, server.KubernetesDeployer, server.HelmPackageManager, server.KubeClusterAccessService)
var gitOperationHandler = gitops.NewHandler(requestBouncer, server.DataStore, server.GitService, server.FileService)
var helmTemplatesHandler = helm.NewTemplateHandler(requestBouncer, server.HelmPackageManager)
var ldapHandler = ldap.NewHandler(requestBouncer)
@ -297,6 +297,7 @@ func (server *Server) Start() error {
EndpointHelmHandler: endpointHelmHandler,
EndpointEdgeHandler: endpointEdgeHandler,
EndpointProxyHandler: endpointProxyHandler,
GitOperationHandler: gitOperationHandler,
FileHandler: fileHandler,
LDAPHandler: ldapHandler,
HelmTemplatesHandler: helmTemplatesHandler,

3
api/portainer.go
View File

@ -181,6 +181,9 @@ type (
Type StackType `json:"Type" example:"1"`
ResourceControl *ResourceControl `json:"ResourceControl"`
Variables []CustomTemplateVariableDefinition
GitConfig *gittypes.RepoConfig `json:"GitConfig"`
// IsComposeFormat indicates if the Kubernetes template is created from a Docker Compose file
IsComposeFormat bool `example:"false"`
}
// CustomTemplateID represents a custom template identifier

1
api/stacks/stackbuilders/k8s_file_content_builder.go
View File

@ -53,6 +53,7 @@ func (b *K8sStackFileContentBuilder) SetUniqueInfo(payload *StackPayload) FileCo
b.stack.Namespace = payload.Namespace
b.stack.CreatedBy = b.User.Username
b.stack.IsComposeFormat = payload.ComposeFormat
b.stack.FromAppTemplate = payload.FromAppTemplate
return b
}

8
api/stacks/stackbuilders/stack_git_builder.go
View File

@ -1,6 +1,7 @@
package stackbuilders
import (
"fmt"
"strconv"
"time"
@ -82,7 +83,12 @@ func (b *GitMethodStackBuilder) SetGitRepository(payload *StackPayload) GitMetho
// Set the project path on the disk
b.stack.ProjectPath = b.fileService.GetStackProjectPath(stackFolder)
commitHash, err := stackutils.DownloadGitRepository(b.stack.ID, repoConfig, b.gitService, b.fileService)
getProjectPath := func() string {
stackFolder := fmt.Sprintf("%d", b.stack.ID)
return b.fileService.GetStackProjectPath(stackFolder)
}
commitHash, err := stackutils.DownloadGitRepository(repoConfig, b.gitService, getProjectPath)
if err != nil {
b.err = httperror.InternalServerError(err.Error(), err)
return b

6
api/stacks/stackutils/gitops.go
View File

@ -16,7 +16,7 @@ var (
// DownloadGitRepository downloads the target git repository on the disk
// The first return value represents the commit hash of the downloaded git repository
func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig, gitService portainer.GitService, fileService portainer.FileService) (string, error) {
func DownloadGitRepository(config gittypes.RepoConfig, gitService portainer.GitService, getProjectPath func() string) (string, error) {
username := ""
password := ""
if config.Authentication != nil {
@ -24,9 +24,7 @@ func DownloadGitRepository(stackID portainer.StackID, config gittypes.RepoConfig
password = config.Authentication.Password
}
stackFolder := fmt.Sprintf("%d", stackID)
projectPath := fileService.GetStackProjectPath(stackFolder)
projectPath := getProjectPath()
err := gitService.CloneRepository(projectPath, config.URL, config.ReferenceName, username, password, config.TLSSkipVerify)
if err != nil {
if err == gittypes.ErrAuthenticationFailure {

17
app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.controller.js
View File

@ -1,7 +1,7 @@
import { AccessControlFormData } from '@/portainer/components/accessControlForm/porAccessControlFormModel';
import { getTemplateVariables, intersectVariables } from '@/react/portainer/custom-templates/components/utils';
import { isBE } from '@/react/portainer/feature-flags/feature-flags.service';
import { editor, upload } from '@@/BoxSelector/common-options/build-methods';
import { editor, upload, git } from '@@/BoxSelector/common-options/build-methods';
import { confirmWebEditorDiscard } from '@@/modals/confirm';
class KubeCreateCustomTemplateViewController {
@ -9,7 +9,7 @@ class KubeCreateCustomTemplateViewController {
constructor($async, $state, Authentication, CustomTemplateService, FormValidator, Notifications, ResourceControlService) {
Object.assign(this, { $async, $state, Authentication, CustomTemplateService, FormValidator, Notifications, ResourceControlService });
this.methodOptions = [editor, upload];
this.methodOptions = [editor, upload, git];
this.templates = null;
this.isTemplateVariablesEnabled = isBE;
@ -31,6 +31,13 @@ class KubeCreateCustomTemplateViewController {
Logo: '',
AccessControlData: new AccessControlFormData(),
Variables: [],
RepositoryURL: '',
RepositoryURLValid: false,
RepositoryReferenceName: 'refs/heads/main',
RepositoryAuthentication: false,
RepositoryUsername: '',
RepositoryPassword: '',
ComposeFilePathInRepository: 'manifest.yml',
};
this.onChangeFile = this.onChangeFile.bind(this);
@ -121,6 +128,8 @@ class KubeCreateCustomTemplateViewController {
return this.createCustomTemplateFromFileContent(template);
case 'upload':
return this.createCustomTemplateFromFileUpload(template);
case 'repository':
return this.createCustomTemplateFromGitRepository(template);
}
}
@ -132,6 +141,10 @@ class KubeCreateCustomTemplateViewController {
return this.CustomTemplateService.createCustomTemplateFromFileUpload(template);
}
createCustomTemplateFromGitRepository(template) {
return this.CustomTemplateService.createCustomTemplateFromGitRepository(template);
}
validateForm(method) {
this.state.formValidationError = '';

2
app/kubernetes/custom-templates/kube-create-custom-template-view/kube-create-custom-template-view.html
View File

@ -35,6 +35,8 @@
<file-upload-description> You can upload a Manifest file from your computer. </file-upload-description>
</file-upload-form>
<git-form deploy-method="kubernetes" ng-if="$ctrl.state.method === 'repository'" value="$ctrl.formValues" on-change="($ctrl.handleChange)"></git-form>
<custom-templates-variables-definition-field
ng-if="$ctrl.isTemplateVariablesEnabled"
value="$ctrl.formValues.Variables"

87
app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.controller.js
View File

@ -3,6 +3,7 @@ import { AccessControlFormData } from '@/portainer/components/accessControlForm/
import { isBE } from '@/react/portainer/feature-flags/feature-flags.service';
import { getTemplateVariables, intersectVariables } from '@/react/portainer/custom-templates/components/utils';
import { confirmWebEditorDiscard } from '@@/modals/confirm';
import { getFilePreview } from '@/react/portainer/gitops/gitops.service';
class KubeEditCustomTemplateViewController {
/* @ngInject */
@ -11,11 +12,18 @@ class KubeEditCustomTemplateViewController {
this.isTemplateVariablesEnabled = isBE;
this.formValues = null;
this.formValues = {
Variables: [],
TLSSkipVerify: false,
};
this.state = {
formValidationError: '',
isEditorDirty: false,
isTemplateValid: true,
isEditorReadOnly: false,
templateLoadFailed: false,
templatePreviewFailed: false,
templatePreviewError: '',
};
this.templates = [];
@ -25,6 +33,7 @@ class KubeEditCustomTemplateViewController {
this.onBeforeUnload = this.onBeforeUnload.bind(this);
this.handleChange = this.handleChange.bind(this);
this.onVariablesChange = this.onVariablesChange.bind(this);
this.previewFileFromGitRepository = this.previewFileFromGitRepository.bind(this);
}
getTemplate() {
@ -32,13 +41,25 @@ class KubeEditCustomTemplateViewController {
try {
const { id } = this.$state.params;
const [template, file] = await Promise.all([this.CustomTemplateService.customTemplate(id), this.CustomTemplateService.customTemplateFile(id)]);
template.FileContent = file;
const template = await this.CustomTemplateService.customTemplate(id);
if (template.GitConfig !== null) {
this.state.isEditorReadOnly = true;
}
try {
template.FileContent = await this.CustomTemplateService.customTemplateFile(id, template.GitConfig !== null);
} catch (err) {
this.state.templateLoadFailed = true;
throw err;
}
template.Variables = template.Variables || [];
this.formValues = template;
this.formValues = { ...this.formValues, ...template };
this.parseTemplate(file);
this.parseTemplate(template.FileContent);
this.parseGitConfig(template.GitConfig);
this.oldFileContent = this.formValues.FileContent;
@ -79,6 +100,62 @@ class KubeEditCustomTemplateViewController {
}
}
parseGitConfig(config) {
if (config === null) {
return;
}
let flatConfig = {
RepositoryURL: config.URL,
RepositoryReferenceName: config.ReferenceName,
ComposeFilePathInRepository: config.ConfigFilePath,
RepositoryAuthentication: config.Authentication !== null,
TLSSkipVerify: config.TLSSkipVerify,
};
if (config.Authentication) {
flatConfig = {
...flatConfig,
RepositoryUsername: config.Authentication.Username,
RepositoryPassword: config.Authentication.Password,
};
}
this.formValues = { ...this.formValues, ...flatConfig };
}
previewFileFromGitRepository() {
this.state.templatePreviewFailed = false;
this.state.templatePreviewError = '';
let creds = {};
if (this.formValues.RepositoryAuthentication) {
creds = {
username: this.formValues.RepositoryUsername,
password: this.formValues.RepositoryPassword,
};
}
const payload = {
repository: this.formValues.RepositoryURL,
targetFile: this.formValues.ComposeFilePathInRepository,
tlsSkipVerify: this.formValues.TLSSkipVerify,
...creds,
};
this.$async(async () => {
try {
this.formValues.FileContent = await getFilePreview(payload);
this.state.isEditorDirty = true;
// check if the template contains mustache template symbol
this.parseTemplate(this.formValues.FileContent);
} catch (err) {
this.state.templatePreviewError = err.message;
this.state.templatePreviewFailed = true;
}
});
}
validateForm() {
this.state.formValidationError = '';

23
app/kubernetes/custom-templates/kube-edit-custom-template-view/kube-edit-custom-template-view.html
View File

@ -7,6 +7,22 @@
<form class="form-horizontal" name="$ctrl.form">
<custom-template-common-fields form-values="$ctrl.formValues"></custom-template-common-fields>
<git-form value="$ctrl.formValues" on-change="($ctrl.handleChange)" ng-if="$ctrl.formValues.GitConfig"></git-form>
<div class="form-group">
<div class="col-sm-12"
><button type="button" class="btn btn-sm btn-light !ml-0" ng-if="$ctrl.formValues.GitConfig" ng-click="$ctrl.previewFileFromGitRepository()">
<pr-icon icon="'refresh-cw'" feather="true"></pr-icon>Reload custom template</button
>
</div>
<div class="col-sm-12" ng-if="$ctrl.state.templatePreviewFailed">
<p class="small vertical-center text-danger mt-5">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>
Custom template could not be loaded, {{ $ctrl.state.templatePreviewError }}.</p
>
</div>
</div>
<web-editor-form
identifier="template-editor"
value="$ctrl.formValues.FileContent"
@ -14,6 +30,7 @@
ng-required="true"
yml="true"
placeholder="Define or paste the content of your manifest file here"
read-only="$ctrl.state.isEditorReadOnly"
>
<editor-description>
<p>Templates allow deploying any kind of Kubernetes resource (Deployment, Secret, ConfigMap...)</p>
@ -31,7 +48,11 @@
is-variables-names-from-parent="true"
></custom-templates-variables-definition-field>
<por-access-control-form form-data="$ctrl.formValues.AccessControlData" resource-control="$ctrl.formValues.ResourceControl"></por-access-control-form>
<por-access-control-form
form-data="$ctrl.formValues.AccessControlData"
resource-control="$ctrl.formValues.ResourceControl"
ng-if="$ctrl.formValues.AccessControlData"
></por-access-control-form>
<div class="col-sm-12 form-section-title"> Actions </div>
<div class="form-group">

11
app/kubernetes/views/deploy/deploy.html
View File

@ -105,6 +105,16 @@
></custom-templates-variables-field>
</div>
<span ng-if="ctrl.state.BuildMethod === ctrl.BuildMethods.CUSTOM_TEMPLATE && ctrl.state.templateId && ctrl.state.templateLoadFailed">
<p class="small vertical-center text-danger mb-5" ng-if="ctrl.currentUser.isAdmin || ctrl.currentUser.id === ctrl.state.template.CreatedByUserId">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please
<a ui-sref="kubernetes.templates.custom.edit({id: ctrl.state.templateId})">click here</a> for configuration.</p
>
<p class="small vertical-center text-danger mb-5" ng-if="!(ctrl.currentUser.isAdmin || ctrl.currentUser.id === ctrl.state.template.CreatedByUserId)">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please contact your administrator.</p
>
</span>
<!-- editor -->
<div class="mt-4">
<web-editor-form
@ -115,6 +125,7 @@
ng-required="true"
yml="true"
placeholder="Define or paste the content of your manifest file here"
read-only="ctrl.state.isEditorReadOnly"
>
<editor-description>
<p class="vertical-center">

24
app/kubernetes/views/deploy/deployController.js
View File

@ -46,6 +46,13 @@ class KubernetesDeployController {
template: null,
baseWebhookUrl: baseStackWebhookUrl(),
webhookId: createWebhookId(),
templateLoadFailed: false,
isEditorReadOnly: false,
};
this.currentUser = {
isAdmin: false,
id: null,
};
this.formValues = {
@ -95,7 +102,7 @@ class KubernetesDeployController {
const metadata = {
type: buildLabel(this.state.BuildMethod),
format: formatLabel(this.state.DeployType),
role: roleLabel(this.Authentication.isAdmin()),
role: roleLabel(this.currentUser.isAdmin),
'automatic-updates': automaticUpdatesLabel(this.formValues.RepositoryAutomaticUpdates, this.formValues.RepositoryMechanism),
};
@ -183,9 +190,15 @@ class KubernetesDeployController {
this.state.template = template;
try {
const fileContent = await this.CustomTemplateService.customTemplateFile(templateId);
this.state.templateContent = fileContent;
this.onChangeFileContent(fileContent);
try {
this.state.templateContent = await this.CustomTemplateService.customTemplateFile(templateId, template.GitConfig !== null);
this.onChangeFileContent(this.state.templateContent);
this.state.isEditorReadOnly = true;
} catch (err) {
this.state.templateLoadFailed = true;
throw err;
}
if (template.Variables && template.Variables.length > 0) {
const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, '']));
@ -318,6 +331,9 @@ class KubernetesDeployController {
$onInit() {
return this.$async(async () => {
this.currentUser.isAdmin = this.Authentication.isAdmin();
this.currentUser.id = this.Authentication.getUserDetails().ID;
this.formValues.namespace_toggle = false;
await this.getNamespaces();

11
app/portainer/components/code-editor/codeEditorController.js
View File

@ -1,8 +1,15 @@
angular.module('portainer.app').controller('CodeEditorController', function CodeEditorController($document, CodeMirrorService, $scope) {
var ctrl = this;
this.$onChanges = function $onChanges({ value }) {
if (value && value.currentValue && ctrl.editor && ctrl.editor.getValue() !== value.currentValue) {
this.$onChanges = function $onChanges({ value, readOnly }) {
if (!ctrl.editor) {
return;
}
if (readOnly && typeof readOnly.currentValue === 'boolean' && ctrl.editor.getValue('readOnly') !== ctrl.readOnly) {
ctrl.editor.setOption('readOnly', ctrl.readOnly);
}
if (value && value.currentValue && ctrl.editor.getValue() !== value.currentValue) {
ctrl.editor.setValue(value.currentValue);
}

1
app/portainer/rest/customTemplate.js
View File

@ -13,6 +13,7 @@ function CustomTemplatesFactory($resource, API_ENDPOINT_CUSTOM_TEMPLATES) {
update: { method: 'PUT', params: { id: '@id' } },
remove: { method: 'DELETE', params: { id: '@id' } },
file: { method: 'GET', params: { id: '@id', action: 'file' } },
gitFetch: { method: 'PUT', params: { id: '@id', action: 'git_fetch' } },
}
);
}

7
app/portainer/services/api/customTemplate.js
View File

@ -1,4 +1,5 @@
import angular from 'angular';
import PortainerError from 'Portainer/error';
angular.module('portainer.app').factory('CustomTemplateService', CustomTemplateServiceFactory);
@ -24,12 +25,12 @@ function CustomTemplateServiceFactory($sanitize, CustomTemplates, FileUploadServ
return CustomTemplates.remove({ id }).$promise;
};
service.customTemplateFile = async function customTemplateFile(id) {
service.customTemplateFile = async function customTemplateFile(id, remote = false) {
try {
const { FileContent } = await CustomTemplates.file({ id }).$promise;
const { FileContent } = remote ? await CustomTemplates.gitFetch({ id }).$promise : await CustomTemplates.file({ id }).$promise;
return FileContent;
} catch (err) {
throw { msg: 'Unable to retrieve customTemplate content', err };
throw new PortainerError('Unable to retrieve custom template content', err);
}
};

18
app/portainer/views/custom-templates/custom-templates-view/customTemplatesView.html
View File

@ -18,16 +18,27 @@
on-change="($ctrl.onChangeTemplateVariables)"
></custom-templates-variables-field>
<div class="form-group">
<div class="form-group" ng-if="$ctrl.state.selectedTemplate && !$ctrl.state.templateLoadFailed">
<div class="col-sm-12">
<a class="small interactive vertical-center" ng-show="!$ctrl.state.showAdvancedOptions" ng-click="$ctrl.state.showAdvancedOptions = true;">
<pr-icon icon="'plus'" class-name="'space-right'"></pr-icon> Customize stack
<pr-icon icon="'plus'" class-name="space-right" feather="true"></pr-icon> {{ $ctrl.state.selectedTemplate.GitConfig !== null ? 'View' : 'Customize' }} stack
</a>
<a class="small interactive vertical-center" ng-show="$ctrl.state.showAdvancedOptions" ng-click="$ctrl.state.showAdvancedOptions = false;">
<pr-icon icon="'minus'" class-name="'space-right'"></pr-icon> Hide custom stack
<pr-icon icon="'minus'" class-name="space-right" feather="true"></pr-icon> Hide {{ $ctrl.state.selectedTemplate.GitConfig === null ? 'custom' : '' }} stack
</a>
</div>
</div>
<span ng-if="$ctrl.state.selectedTemplate && $ctrl.state.templateLoadFailed">
<p class="small vertical-center text-danger mb-5" ng-if="$ctrl.currentUser.isAdmin || $ctrl.currentUser.id === $ctrl.state.selectedTemplate.CreatedByUserId">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please
<a ui-sref="docker.templates.custom.edit({id: $ctrl.state.selectedTemplate.Id})">click here</a> for configuration.</p
>
<p class="small vertical-center text-danger mb-5" ng-if="!($ctrl.currentUser.isAdmin || $ctrl.currentUser.id === $ctrl.state.selectedTemplate.CreatedByUserId)">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please contact your administrator.</p
>
</span>
<!-- web-editor -->
<web-editor-form
ng-if="$ctrl.state.showAdvancedOptions"
@ -37,6 +48,7 @@
ng-required="true"
yml="true"
placeholder="Define or paste the content of your docker compose file here"
read-only="$ctrl.state.isEditorReadOnly"
>
<editor-description>
<p>

12
app/portainer/views/custom-templates/custom-templates-view/customTemplatesViewController.js
View File

@ -44,10 +44,10 @@ class CustomTemplatesViewController {
showAdvancedOptions: false,
formValidationError: '',
actionInProgress: false,
isEditorVisible: false,
deployable: false,
templateNameRegex: TEMPLATE_NAME_VALIDATION_REGEX,
templateContent: '',
templateLoadFailed: false,
};
this.currentUser = {
@ -204,6 +204,13 @@ class CustomTemplatesViewController {
template.Selected = true;
try {
this.state.templateContent = this.formValues.fileContent = await this.CustomTemplateService.customTemplateFile(template.Id, template.GitConfig !== null);
} catch (err) {
this.state.templateLoadFailed = true;
this.Notifications.error('Failure', err, 'Unable to retrieve custom template data');
}
this.formValues.network = _.find(this.availableNetworks, function (o) {
return o.Name === 'bridge';
});
@ -213,9 +220,6 @@ class CustomTemplatesViewController {
this.$anchorScroll('view-top');
const applicationState = this.StateManager.getState();
this.state.deployable = this.isDeployable(applicationState.endpoint, template.Type);
const file = await this.CustomTemplateService.customTemplateFile(template.Id);
this.state.templateContent = file;
this.formValues.fileContent = file;
if (template.Variables && template.Variables.length > 0) {
const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, '']));

23
app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateView.html
View File

@ -7,6 +7,22 @@
<form class="form-horizontal" name="customTemplateForm">
<custom-template-common-fields form-values="$ctrl.formValues" show-platform-field="true" show-type-field="true"></custom-template-common-fields>
<git-form value="$ctrl.formValues" on-change="($ctrl.handleChange)" ng-if="$ctrl.formValues.GitConfig"></git-form>
<div class="form-group">
<div class="col-sm-12"
><button type="button" class="btn btn-sm btn-light !ml-0" ng-if="$ctrl.formValues.GitConfig" ng-click="$ctrl.previewFileFromGitRepository()">
<pr-icon icon="'refresh-cw'" feather="true"></pr-icon>Reload custom template</button
>
</div>
<div class="col-sm-12" ng-if="$ctrl.state.templatePreviewFailed">
<p class="small vertical-center text-danger mt-5">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>
Custom template could not be loaded, {{ $ctrl.state.templatePreviewError }}.</p
>
</div>
</div>
<!-- web-editor -->
<web-editor-form
identifier="custom-template-creation-editor"
@ -15,6 +31,7 @@
ng-required="true"
yml="true"
placeholder="Define or paste the content of your docker compose file here"
read-only="$ctrl.state.isEditorReadOnly"
>
<editor-description>
<p>
@ -42,7 +59,11 @@
is-variables-names-from-parent="true"
></custom-templates-variables-definition-field>
<por-access-control-form form-data="$ctrl.formValues.AccessControlData" resource-control="$ctrl.formValues.ResourceControl"></por-access-control-form>
<por-access-control-form
form-data="$ctrl.formValues.AccessControlData"
resource-control="$ctrl.formValues.ResourceControl"
ng-if="$ctrl.formValues.AccessControlData"
></por-access-control-form>
<div class="col-sm-12 form-section-title"> Actions </div>
<div class="form-group">

90
app/portainer/views/custom-templates/edit-custom-template-view/editCustomTemplateViewController.js
View File

@ -1,4 +1,5 @@
import _ from 'lodash';
import { getFilePreview } from '@/react/portainer/gitops/gitops.service';
import { ResourceControlViewModel } from '@/react/portainer/access-control/models/ResourceControlViewModel';
import { AccessControlFormData } from 'Portainer/components/accessControlForm/porAccessControlFormModel';
@ -13,11 +14,18 @@ class EditCustomTemplateViewController {
this.isTemplateVariablesEnabled = isBE;
this.formValues = null;
this.formValues = {
Variables: [],
TLSSkipVerify: false,
};
this.state = {
formValidationError: '',
isEditorDirty: false,
isTemplateValid: true,
isEditorReadOnly: false,
templateLoadFailed: false,
templatePreviewFailed: false,
templatePreviewError: '',
};
this.templates = [];
@ -28,6 +36,7 @@ class EditCustomTemplateViewController {
this.editorUpdate = this.editorUpdate.bind(this);
this.onVariablesChange = this.onVariablesChange.bind(this);
this.handleChange = this.handleChange.bind(this);
this.previewFileFromGitRepository = this.previewFileFromGitRepository.bind(this);
}
getTemplate() {
@ -35,14 +44,25 @@ class EditCustomTemplateViewController {
}
async getTemplateAsync() {
try {
const [template, file] = await Promise.all([
this.CustomTemplateService.customTemplate(this.$state.params.id),
this.CustomTemplateService.customTemplateFile(this.$state.params.id),
]);
template.FileContent = file;
const template = await this.CustomTemplateService.customTemplate(this.$state.params.id);
if (template.GitConfig !== null) {
this.state.isEditorReadOnly = true;
}
try {
template.FileContent = await this.CustomTemplateService.customTemplateFile(this.$state.params.id, template.GitConfig !== null);
} catch (err) {
this.state.templateLoadFailed = true;
throw err;
}
template.Variables = template.Variables || [];
this.formValues = template;
this.formValues = { ...this.formValues, ...template };
this.parseTemplate(template.FileContent);
this.parseGitConfig(template.GitConfig);
this.oldFileContent = this.formValues.FileContent;
if (template.ResourceControl) {
@ -145,6 +165,62 @@ class EditCustomTemplateViewController {
}
}
parseGitConfig(config) {
if (config === null) {
return;
}
let flatConfig = {
RepositoryURL: config.URL,
RepositoryReferenceName: config.ReferenceName,
ComposeFilePathInRepository: config.ConfigFilePath,
RepositoryAuthentication: config.Authentication !== null,
TLSSkipVerify: config.TLSSkipVerify,
};
if (config.Authentication) {
flatConfig = {
...flatConfig,
RepositoryUsername: config.Authentication.Username,
RepositoryPassword: config.Authentication.Password,
};
}
this.formValues = { ...this.formValues, ...flatConfig };
}
previewFileFromGitRepository() {
this.state.templatePreviewFailed = false;
this.state.templatePreviewError = '';
let creds = {};
if (this.formValues.RepositoryAuthentication) {
creds = {
username: this.formValues.RepositoryUsername,
password: this.formValues.RepositoryPassword,
};
}
const payload = {
repository: this.formValues.RepositoryURL,
targetFile: this.formValues.ComposeFilePathInRepository,
tlsSkipVerify: this.formValues.TLSSkipVerify,
...creds,
};
this.$async(async () => {
try {
this.formValues.FileContent = await getFilePreview(payload);
this.state.isEditorDirty = true;
// check if the template contains mustache template symbol
this.parseTemplate(this.formValues.FileContent);
} catch (err) {
this.state.templatePreviewError = err.message;
this.state.templatePreviewFailed = true;
}
});
}
async uiCanExit() {
if (this.formValues.FileContent !== this.oldFileContent && this.state.isEditorDirty) {
return confirmWebEditorDiscard();

23
app/portainer/views/stacks/create/createStackController.js
View File

@ -39,7 +39,6 @@ angular
$scope.stackWebhookFeature = FeatureId.STACK_WEBHOOK;
$scope.buildMethods = [editor, upload, git, customTemplate];
$scope.STACK_NAME_VALIDATION_REGEX = STACK_NAME_VALIDATION_REGEX;
$scope.isAdmin = Authentication.isAdmin();
$scope.formValues = {
Name: '',
@ -72,6 +71,13 @@ angular
selectedTemplateId: null,
baseWebhookUrl: baseStackWebhookUrl(),
webhookId: createWebhookId(),
templateLoadFailed: false,
isEditorReadOnly: false,
};
$scope.currentUser = {
isAdmin: false,
id: null,
};
$window.onbeforeunload = () => {
@ -296,9 +302,15 @@ angular
$scope.state.selectedTemplateId = templateId;
$scope.state.selectedTemplate = template;
const fileContent = await CustomTemplateService.customTemplateFile(templateId);
$scope.state.templateContent = fileContent;
onChangeFileContent(fileContent);
try {
$scope.state.templateContent = await this.CustomTemplateService.customTemplateFile(templateId, template.GitConfig !== null);
onChangeFileContent($scope.state.templateContent);
$scope.state.isEditorReadOnly = true;
} catch (err) {
$scope.state.templateLoadFailed = true;
throw err;
}
if (template.Variables && template.Variables.length > 0) {
const variables = Object.fromEntries(template.Variables.map((variable) => [variable.name, '']));
@ -321,6 +333,9 @@ angular
}
async function initView() {
$scope.currentUser.isAdmin = Authentication.isAdmin();
$scope.currentUser.id = Authentication.getUserDetails().ID;
var endpointMode = $scope.applicationState.endpoint.mode;
$scope.state.StackType = 2;
$scope.isDockerStandalone = endpointMode.provider === 'DOCKER_STANDALONE';

37
app/portainer/views/stacks/create/createstack.html
View File

@ -91,19 +91,31 @@
></git-form>
<div ng-show="state.Method === 'template'">
<custom-template-selector
new-template-path="docker.templates.custom.new"
stack-type="state.StackType"
on-change="(onChangeTemplateId)"
value="state.selectedTemplateId"
></custom-template-selector>
<div class="col-sm-12">
<custom-template-selector
new-template-path="docker.templates.custom.new"
stack-type="state.StackType"
on-change="(onChangeTemplateId)"
value="state.selectedTemplateId"
></custom-template-selector>
<custom-templates-variables-field
ng-if="isTemplateVariablesEnabled && state.selectedTemplate"
definitions="state.selectedTemplate.Variables"
value="formValues.Variables"
on-change="(onChangeTemplateVariables)"
></custom-templates-variables-field>
<custom-templates-variables-field
ng-if="isTemplateVariablesEnabled && state.selectedTemplate"
definitions="state.selectedTemplate.Variables"
value="formValues.Variables"
on-change="(onChangeTemplateVariables)"
></custom-templates-variables-field>
<span ng-if="state.Method === 'template' && state.selectedTemplateId && state.templateLoadFailed">
<p class="small vertical-center text-danger mb-5" ng-if="currentUser.isAdmin || currentUser.id === state.selectedTemplate.CreatedByUserId">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please
<a ui-sref="docker.templates.custom.edit({id: state.selectedTemplateId})">click here</a> for configuration.</p
>
<p class="small vertical-center text-danger mb-5" ng-if="!(currentUser.isAdmin || currentUser.id === state.selectedTemplate.CreatedByUserId)">
<pr-icon icon="'alert-triangle'" mode="'danger'" size="'md'" feather="true"></pr-icon>Custom template could not be loaded, please contact your administrator.</p
>
</span>
</div>
</div>
<web-editor-form
@ -114,6 +126,7 @@
ng-required="true"
yml="true"
placeholder="Define or paste the content of your docker compose file here"
read-only="state.isEditorReadOnly"
>
<editor-description>
<p>

25
app/react/portainer/gitops/gitops.service.ts Normal file
View File

@ -0,0 +1,25 @@
import axios, { parseAxiosError } from '@/portainer/services/axios';
interface PreviewPayload {
repository: string;
targetFile: string;
reference?: string;
username?: string;
password?: string;
tlsSkipVerify?: boolean;
}
interface PreviewResponse {
FileContent: string;
}
export async function getFilePreview(payload: PreviewPayload) {
try {
const {
data: { FileContent },
} = await axios.post<PreviewResponse>('/gitops/repo/file/preview', payload);
return FileContent;
} catch (e) {
throw parseAxiosError(e as Error, 'Unable to fetch file from git');
}
}

1
app/react/portainer/gitops/queries/useSearch.ts
View File

@ -8,6 +8,7 @@ interface SearchPayload {
reference?: string;
username?: string;
password?: string;
tlsSkipVerify?: boolean;
}
export function useSearch(payload: SearchPayload, enabled: boolean) {