chore(portainer): clean up the code EE-5188 (#8660)

pull/8662/head
andres-portainer 2023-03-13 13:18:28 -03:00 committed by GitHub
parent 621a01ba3b
commit 15cbdb8af9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 111 additions and 65 deletions

View File

@ -72,6 +72,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
if err != nil {
panic(err)
}
*flags.Assets = filepath.Join(filepath.Dir(ex), *flags.Assets)
}
@ -80,7 +81,6 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
// ValidateFlags validates the values of the flags.
func (*Service) ValidateFlags(flags *portainer.CLIFlags) error {
displayDeprecationWarnings(flags)
err := validateEndpointURL(*flags.EndpointURL)
@ -111,31 +111,38 @@ func displayDeprecationWarnings(flags *portainer.CLIFlags) {
}
func validateEndpointURL(endpointURL string) error {
if endpointURL != "" {
if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
return errInvalidEndpointProtocol
}
if endpointURL == "" {
return nil
}
if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
socketPath := strings.TrimPrefix(endpointURL, "unix://")
socketPath = strings.TrimPrefix(socketPath, "npipe://")
if _, err := os.Stat(socketPath); err != nil {
if os.IsNotExist(err) {
return errSocketOrNamedPipeNotFound
}
return err
if !strings.HasPrefix(endpointURL, "unix://") && !strings.HasPrefix(endpointURL, "tcp://") && !strings.HasPrefix(endpointURL, "npipe://") {
return errInvalidEndpointProtocol
}
if strings.HasPrefix(endpointURL, "unix://") || strings.HasPrefix(endpointURL, "npipe://") {
socketPath := strings.TrimPrefix(endpointURL, "unix://")
socketPath = strings.TrimPrefix(socketPath, "npipe://")
if _, err := os.Stat(socketPath); err != nil {
if os.IsNotExist(err) {
return errSocketOrNamedPipeNotFound
}
return err
}
}
return nil
}
func validateSnapshotInterval(snapshotInterval string) error {
if snapshotInterval != "" {
_, err := time.ParseDuration(snapshotInterval)
if err != nil {
return errInvalidSnapshotInterval
}
if snapshotInterval == "" {
return nil
}
_, err := time.ParseDuration(snapshotInterval)
if err != nil {
return errInvalidSnapshotInterval
}
return nil
}

View File

@ -12,13 +12,14 @@ func Confirm(message string) (bool, error) {
fmt.Printf("%s [y/N]", message)
reader := bufio.NewReader(os.Stdin)
answer, err := reader.ReadString('\n')
if err != nil {
return false, err
}
answer = strings.Replace(answer, "\n", "", -1)
answer = strings.ReplaceAll(answer, "\n", "")
answer = strings.ToLower(answer)
return answer == "y" || answer == "yes", nil
}

View File

@ -7,7 +7,6 @@ import (
"crypto/x509"
"encoding/base64"
"encoding/hex"
"math/big"
"github.com/portainer/libcrypto"
)
@ -115,9 +114,6 @@ func (service *ECDSAService) CreateSignature(message string) (string, error) {
hash := libcrypto.HashFromBytes([]byte(message))
r := big.NewInt(0)
s := big.NewInt(0)
r, s, err := ecdsa.Sign(rand.Reader, service.privateKey, hash)
if err != nil {
return "", err

View File

@ -129,7 +129,7 @@ func Test_UnMarshalObjectUnencrypted(t *testing.T) {
var object string
err := conn.UnmarshalObject(test.object, &object)
is.NoError(err)
is.Equal(test.expected, string(object))
is.Equal(test.expected, object)
})
}
}

View File

@ -92,7 +92,7 @@ func (tx *DbTransaction) CreateObject(bucketName string, fn func(uint64) (int, i
return err
}
return bucket.Put(tx.conn.ConvertToKey(int(id)), data)
return bucket.Put(tx.conn.ConvertToKey(id), data)
}
func (tx *DbTransaction) CreateObjectWithId(bucketName string, id int, obj interface{}) error {

View File

@ -9,8 +9,7 @@ import (
// NewDatabase should use config options to return a connection to the requested database
func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
switch storeType {
case "boltdb":
if storeType == "boltdb" {
return &boltdb.DbConnection{
Path: storePath,
EncryptionKey: encryptionKey,

View File

@ -8,6 +8,7 @@ import (
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/docker"
"github.com/portainer/portainer/api/kubernetes/cli"
"github.com/rs/zerolog/log"
)
@ -17,11 +18,7 @@ type PostInitMigrator struct {
dataStore dataservices.DataStore
}
func NewPostInitMigrator(
kubeFactory *cli.ClientFactory,
dockerFactory *docker.ClientFactory,
dataStore dataservices.DataStore,
) *PostInitMigrator {
func NewPostInitMigrator(kubeFactory *cli.ClientFactory, dockerFactory *docker.ClientFactory, dataStore dataservices.DataStore) *PostInitMigrator {
return &PostInitMigrator{
kubeFactory: kubeFactory,
dockerFactory: dockerFactory,
@ -44,9 +41,10 @@ func (migrator *PostInitMigrator) PostInitMigrateIngresses() error {
if err != nil {
return err
}
for i := range endpoints {
// Early exit if we do not need to migrate!
if endpoints[i].PostInitMigrations.MigrateIngresses == false {
if !endpoints[i].PostInitMigrations.MigrateIngresses {
return nil
}
@ -67,10 +65,11 @@ func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
log.Err(err).Msg("failure getting endpoints")
return
}
for i := range environments {
if environments[i].Type == portainer.DockerEnvironment {
// // Early exit if we do not need to migrate!
if environments[i].PostInitMigrations.MigrateGPUs == false {
if !environments[i].PostInitMigrations.MigrateGPUs {
return
}
@ -102,11 +101,13 @@ func (migrator *PostInitMigrator) PostInitMigrateGPUs() {
log.Err(err).Msg("failed to inspect container")
return
}
deviceRequests := containerDetails.HostConfig.Resources.DeviceRequests
for _, deviceRequest := range deviceRequests {
if deviceRequest.Driver == "nvidia" {
environments[i].EnableGPUManagement = true
migrator.dataStore.Endpoint().UpdateEndpoint(environments[i].ID, &environments[i])
break containersLoop
}
}

View File

@ -38,17 +38,19 @@ func NewClientFactory(signatureService portainer.DigitalSignatureService, revers
// with an agent enabled environment(endpoint) to target a specific node in an agent cluster.
// The underlying http client timeout may be specified, a default value is used otherwise.
func (factory *ClientFactory) CreateClient(endpoint *portainer.Endpoint, nodeName string, timeout *time.Duration) (*client.Client, error) {
if endpoint.Type == portainer.AzureEnvironment {
switch endpoint.Type {
case portainer.AzureEnvironment:
return nil, errUnsupportedEnvironmentType
} else if endpoint.Type == portainer.AgentOnDockerEnvironment {
case portainer.AgentOnDockerEnvironment:
return createAgentClient(endpoint, factory.signatureService, nodeName, timeout)
} else if endpoint.Type == portainer.EdgeAgentOnDockerEnvironment {
case portainer.EdgeAgentOnDockerEnvironment:
return createEdgeClient(endpoint, factory.signatureService, factory.reverseTunnelService, nodeName, timeout)
}
if strings.HasPrefix(endpoint.URL, "unix://") || strings.HasPrefix(endpoint.URL, "npipe://") {
return createLocalClient(endpoint)
}
return createTCPClient(endpoint, timeout)
}

View File

@ -59,6 +59,7 @@ func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoin
if err != nil {
return err
}
for _, registry := range registries {
if registry.Authentication {
err = registryutils.EnsureRegTokenValid(manager.dataStore, &registry)
@ -75,6 +76,7 @@ func (manager *SwarmStackManager) Login(registries []portainer.Registry, endpoin
runCommandAndCaptureStdErr(command, registryArgs, nil, "")
}
}
return nil
}
@ -84,7 +86,9 @@ func (manager *SwarmStackManager) Logout(endpoint *portainer.Endpoint) error {
if err != nil {
return err
}
args = append(args, "logout")
return runCommandAndCaptureStdErr(command, args, nil, "")
}
@ -101,6 +105,7 @@ func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, pul
} else {
args = append(args, "stack", "deploy", "--with-registry-auth")
}
if !pullImage {
args = append(args, "--resolve-image=never")
}
@ -112,6 +117,7 @@ func (manager *SwarmStackManager) Deploy(stack *portainer.Stack, prune bool, pul
for _, envvar := range stack.Env {
env = append(env, envvar.Name+"="+envvar.Value)
}
return runCommandAndCaptureStdErr(command, args, env, stack.ProjectPath)
}
@ -121,7 +127,9 @@ func (manager *SwarmStackManager) Remove(stack *portainer.Stack, endpoint *porta
if err != nil {
return err
}
args = append(args, "stack", "rm", stack.Name)
return runCommandAndCaptureStdErr(command, args, nil, "")
}
@ -198,6 +206,7 @@ func (manager *SwarmStackManager) updateDockerCLIConfiguration(configPath string
if config["HttpHeaders"] == nil {
config["HttpHeaders"] = make(map[string]interface{})
}
headersObject := config["HttpHeaders"].(map[string]interface{})
headersObject["X-PortainerAgent-ManagerOperation"] = "1"
headersObject["X-PortainerAgent-Signature"] = signature
@ -230,5 +239,6 @@ func configureFilePaths(args []string, filePaths []string) []string {
for _, path := range filePaths {
args = append(args, "--compose-file", path)
}
return args
}

View File

@ -75,6 +75,7 @@ func newHttpClientForAzure() *http.Client {
}
client.InstallProtocol("https", githttp.NewClient(httpsCli))
return httpsCli
}
@ -98,10 +99,12 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
if err != nil {
return "", errors.WithMessage(err, "failed to parse url")
}
downloadUrl, err := a.buildDownloadUrl(config, opt.referenceName)
if err != nil {
return "", errors.WithMessage(err, "failed to build download url")
}
zipFile, err := os.CreateTemp("", "azure-git-repo-*.zip")
if err != nil {
return "", errors.WithMessage(err, "failed to create temp file")
@ -133,6 +136,7 @@ func (a *azureClient) downloadZipFromAzureDevOps(ctx context.Context, opt cloneO
if err != nil {
return "", errors.WithMessage(err, "failed to save HTTP response to a file")
}
return zipFile.Name(), nil
}
@ -141,6 +145,7 @@ func (a *azureClient) latestCommitID(ctx context.Context, opt fetchOption) (stri
if err != nil {
return "", err
}
return rootItem.CommitId, nil
}
@ -187,6 +192,7 @@ func (a *azureClient) getRootItem(ctx context.Context, opt fetchOption) (*azureI
if len(items.Value) == 0 || items.Value[0].CommitId == "" {
return nil, errors.Errorf("failed to get latest commitID in the repository")
}
return &items.Value[0], nil
}
@ -205,7 +211,7 @@ func parseUrl(rawUrl string) (*azureOptions, error) {
return nil, errors.Errorf("supported url schemes are https and ssh; recevied URL %s rawUrl", rawUrl)
}
var expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
const expectedSshUrl = "git@ssh.dev.azure.com:v3/Organisation/Project/Repository"
func parseSshUrl(rawUrl string) (*azureOptions, error) {
path := strings.Split(rawUrl, "/")
@ -343,6 +349,7 @@ func (a *azureClient) buildTreeUrl(config *azureOptions, rootObjectHash string)
if err != nil {
return "", errors.Wrapf(err, "failed to parse list tree url path %s", rawUrl)
}
q := u.Query()
// projectId={projectId}&recursive=true&fileName={fileName}&$format={$format}&api-version=6.0
q.Set("recursive", "true")
@ -361,9 +368,11 @@ func formatReferenceName(name string) string {
if strings.HasPrefix(name, branchPrefix) {
return strings.TrimPrefix(name, branchPrefix)
}
if strings.HasPrefix(name, tagPrefix) {
return strings.TrimPrefix(name, tagPrefix)
}
return name
}
@ -371,9 +380,11 @@ func getVersionType(name string) string {
if strings.HasPrefix(name, branchPrefix) {
return "branch"
}
if strings.HasPrefix(name, tagPrefix) {
return "tag"
}
return "commit"
}
@ -490,5 +501,6 @@ func checkAzureStatusCode(err error, code int) error {
} else if code == http.StatusUnauthorized || code == http.StatusNonAuthoritativeInfo {
return gittypes.ErrAuthenticationFailure
}
return err
}

View File

@ -8,14 +8,13 @@ import (
"testing"
"time"
_ "github.com/joho/godotenv/autoload"
gittypes "github.com/portainer/portainer/api/git/types"
_ "github.com/joho/godotenv/autoload"
"github.com/stretchr/testify/assert"
)
var (
privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
)
const privateAzureRepoURL = "https://portainer.visualstudio.com/gitops-test/_git/gitops-test"
func TestService_ClonePublicRepository_Azure(t *testing.T) {
ensureIntegrationTest(t)
@ -107,7 +106,7 @@ func TestService_ListRefs_Azure_Concurrently(t *testing.T) {
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go service.ListRefs(privateAzureRepoURL, username, accessToken, false)
service.ListRefs(privateAzureRepoURL, username, accessToken, false)
@ -269,7 +268,7 @@ func TestService_ListFiles_Azure_Concurrently(t *testing.T) {
accessToken := getRequiredValue(t, "AZURE_DEVOPS_PAT")
username := getRequiredValue(t, "AZURE_DEVOPS_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})
service.ListFiles(privateAzureRepoURL, "refs/heads/main", username, accessToken, false, []string{})

View File

@ -60,7 +60,7 @@ func TestService_ListRefs_Github_Concurrently(t *testing.T) {
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
repositoryUrl := privateGitRepoURL
go service.ListRefs(repositoryUrl, username, accessToken, false)
@ -224,7 +224,7 @@ func TestService_ListFiles_Github_Concurrently(t *testing.T) {
repositoryUrl := privateGitRepoURL
accessToken := getRequiredValue(t, "GITHUB_PAT")
username := getRequiredValue(t, "GITHUB_USERNAME")
service := newService(context.TODO(), REPOSITORY_CACHE_SIZE, 200*time.Millisecond)
service := newService(context.TODO(), repositoryCacheSize, 200*time.Millisecond)
go service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})
service.ListFiles(repositoryUrl, "refs/heads/main", username, accessToken, false, []string{})

View File

@ -95,10 +95,12 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int {
if err != nil {
t.Fatalf("can't open a git repo at %s with error %v", dir, err)
}
iter, err := repo.Log(&git.LogOptions{All: true})
if err != nil {
t.Fatalf("can't get a commit history iterator with error %v", err)
}
count := 0
err = iter.ForEach(func(_ *object.Commit) error {
count++
@ -107,6 +109,7 @@ func getCommitHistoryLength(t *testing.T, err error, dir string) int {
if err != nil {
t.Fatalf("can't iterate over the commit history with error %v", err)
}
return count
}

View File

@ -10,9 +10,9 @@ import (
"github.com/rs/zerolog/log"
)
var (
REPOSITORY_CACHE_SIZE = 4
REPOSITORY_CACHE_TTL = 5 * time.Minute
const (
repositoryCacheSize = 4
repositoryCacheTTL = 5 * time.Minute
)
// baseOption provides a minimum group of information to operate a git repository, like git-remote
@ -58,7 +58,7 @@ type Service struct {
// NewService initializes a new service.
func NewService(ctx context.Context) *Service {
return newService(ctx, REPOSITORY_CACHE_SIZE, REPOSITORY_CACHE_TTL)
return newService(ctx, repositoryCacheSize, repositoryCacheTTL)
}
func newService(ctx context.Context, cacheSize int, cacheTTL time.Duration) *Service {

View File

@ -34,7 +34,7 @@ func UpdateGitObject(gitService portainer.GitService, dataStore dataservices.Dat
return false, "", errors.WithMessagef(err, "failed to fetch latest commit id of %v", objId)
}
hashChanged := !strings.EqualFold(newHash, string(gitConfig.ConfigHash))
hashChanged := !strings.EqualFold(newHash, gitConfig.ConfigHash)
forceUpdate := autoUpdateConfig != nil && autoUpdateConfig.ForceUpdate
if !hashChanged && !forceUpdate {
log.Debug().

View File

@ -48,15 +48,16 @@ func (handler *Handler) createProfile(w http.ResponseWriter, r *http.Request) *h
return httperror.BadRequest("Invalid query parameter: method", err)
}
switch method {
case "editor":
if method == "editor" {
return handler.createFDOProfileFromFileContent(w, r)
}
return httperror.BadRequest("Invalid method. Value must be one of: editor", errors.New("invalid method"))
}
func (handler *Handler) createFDOProfileFromFileContent(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload createProfileFromFileContentPayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
@ -66,6 +67,7 @@ func (handler *Handler) createFDOProfileFromFileContent(w http.ResponseWriter, r
if err != nil {
return httperror.InternalServerError(err.Error(), err)
}
if !isUnique {
return &httperror.HandlerError{StatusCode: http.StatusConflict, Message: fmt.Sprintf("A profile with the name '%s' already exists", payload.Name), Err: errors.New("a profile already exists with this name")}
}
@ -80,6 +82,7 @@ func (handler *Handler) createFDOProfileFromFileContent(w http.ResponseWriter, r
if err != nil {
return httperror.InternalServerError("Unable to persist profile file on disk", err)
}
profile.FilePath = filePath
profile.DateCreated = time.Now().Unix()

View File

@ -63,6 +63,7 @@ func (handler *Handler) ifRequestedTemplateExists(payload *filePayload) *httperr
return nil
}
}
return httperror.InternalServerError("Invalid template", errors.New("requested template does not exist"))
}
@ -82,6 +83,7 @@ func (handler *Handler) ifRequestedTemplateExists(payload *filePayload) *httperr
// @router /templates/file [post]
func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *httperror.HandlerError {
var payload filePayload
err := request.DecodeAndValidateJSONPayload(r, &payload)
if err != nil {
return httperror.BadRequest("Invalid request payload", err)
@ -112,11 +114,9 @@ func (handler *Handler) templateFile(w http.ResponseWriter, r *http.Request) *ht
}
func (handler *Handler) cleanUp(projectPath string) error {
func (handler *Handler) cleanUp(projectPath string) {
err := handler.FileService.RemoveDirectory(projectPath)
if err != nil {
log.Debug().Err(err).Msg("HTTP error: unable to cleanup stack creation")
}
return nil
}

View File

@ -14,30 +14,35 @@ func streamFromWebsocketToWriter(websocketConn *websocket.Conn, writer io.Writer
_, in, err := websocketConn.ReadMessage()
if err != nil {
errorChan <- err
break
}
_, err = writer.Write(in)
if err != nil {
errorChan <- err
break
}
}
}
func streamFromReaderToWebsocket(websocketConn *websocket.Conn, reader io.Reader, errorChan chan error) {
out := make([]byte, readerBufferSize)
for {
out := make([]byte, readerBufferSize)
_, err := reader.Read(out)
if err != nil {
errorChan <- err
break
}
processedOutput := validString(string(out[:]))
processedOutput := validString(string(out))
err = websocketConn.WriteMessage(websocket.TextMessage, []byte(processedOutput))
if err != nil {
errorChan <- err
break
}
}
@ -46,6 +51,7 @@ func streamFromReaderToWebsocket(websocketConn *websocket.Conn, reader io.Reader
func validString(s string) string {
if !utf8.ValidString(s) {
v := make([]rune, 0, len(s))
for i, r := range s {
if r == utf8.RuneError {
_, size := utf8.DecodeRuneInString(s[i:])
@ -53,9 +59,12 @@ func validString(s string) string {
continue
}
}
v = append(v, r)
}
s = string(v)
}
return s
}

View File

@ -26,6 +26,7 @@ func (transport *Transport) applyPortainerContainers(resources []interface{}) ([
responseObject, _ = transport.applyPortainerContainer(responseObject)
decoratedResourceData = append(decoratedResourceData, responseObject)
}
return decoratedResourceData, nil
}
@ -34,8 +35,10 @@ func (transport *Transport) applyPortainerContainer(resourceObject map[string]in
if !ok {
return resourceObject, nil
}
if len(resourceId) >= 12 && resourceId[0:12] == portainerContainerId {
resourceObject["IsPortainer"] = true
}
return resourceObject, nil
}

View File

@ -150,8 +150,7 @@ func (transport *baseTransport) getRoundTripToken(request *http.Request, tokenMa
func decorateAgentRequest(r *http.Request, dataStore dataservices.DataStore) error {
requestPath := strings.TrimPrefix(r.URL.Path, "/v2")
switch {
case strings.HasPrefix(requestPath, "/dockerhub"):
if strings.HasPrefix(requestPath, "/dockerhub") {
return decorateAgentDockerHubRequest(r, dataStore)
}

View File

@ -8,13 +8,14 @@ import (
"strings"
"time"
"github.com/cbroglie/mustache"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/pkg/errors"
libstack "github.com/portainer/docker-compose-wrapper"
"github.com/portainer/portainer/api/filesystem"
"github.com/cbroglie/mustache"
"github.com/pkg/errors"
"github.com/rs/zerolog/log"
)
@ -65,7 +66,7 @@ func (service *service) upgradeDocker(licenseKey, version, envType string) error
projectName := fmt.Sprintf(
"portainer-upgrade-%d-%s",
timeId,
strings.Replace(version, ".", "-", -1))
strings.ReplaceAll(version, ".", "-"))
err = service.composeDeployer.Deploy(
ctx,

View File

@ -98,7 +98,7 @@ func (service *kubeClusterAccessService) GetData(hostURL string, endpointID port
// When the api call is internal, the baseURL should not be used.
if hostURL == "localhost" {
hostURL = hostURL + service.httpsBindAddr
hostURL += service.httpsBindAddr
baseURL = "/"
}

View File

@ -24,6 +24,7 @@ func GetStackFilePaths(stack *portainer.Stack, absolute bool) []string {
for _, file := range append([]string{stack.EntryPoint}, stack.AdditionalFiles...) {
filePaths = append(filePaths, filesystem.JoinPaths(stack.ProjectPath, file))
}
return filePaths
}