mirror of https://github.com/portainer/portainer
feat(database): add encryption support EE-1983 (#6316)
* bootstrap encryption key * secret key message change in cli and secret key file content trimmed * Migrate encryption code to latest version * pull in newer code * tidying up * working data encryption layer * fix tests * remove stray comment * fix a few minor issues and improve the comments * split out databasefilename with param to two methods to be more obvious * DB encryption integration (#6374) * json methods moved under DBConnection * store encryption fixed * cleaned * review comments addressed * newstore value fixed * backup test updated * logrus format config updated * Fix for newStore Co-authored-by: Matt Hook <hookenz@gmail.com> * Minor improvements * Improve the export code. Add missing webhook for import * rename HelmUserRepositorys to HelmUserRepositories * fix logging messages * when starting portainer with a key (first use) http is disabled by default. But when starting fresh without a key, http is enabled? * Fix bug for default settings on new installs Co-authored-by: Prabhat Khera <prabhat.khera@portainer.io> Co-authored-by: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com>pull/4499/merge
Matt Hook
GitHub
parent
59ec22f706
commit
34cc8ea96a
|
|
@ -57,6 +57,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
|
|||
Logo: kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
|
||||
Templates: kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
|
||||
BaseURL: kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(),
|
||||
SecretKeyName: kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
|
||||
}
|
||||
|
||||
kingpin.Parse()
|
||||
|
|
|
|||
|
|
@ -16,10 +16,11 @@ const (
|
|||
defaultTLSCertPath = "/certs/cert.pem"
|
||||
defaultTLSKeyPath = "/certs/key.pem"
|
||||
defaultHTTPDisabled = "false"
|
||||
defaultHTTPEnabled = "false"
|
||||
defaultHTTPEnabled = "true"
|
||||
defaultSSL = "false"
|
||||
defaultSSLCertPath = "/certs/portainer.crt"
|
||||
defaultSSLKeyPath = "/certs/portainer.key"
|
||||
defaultSnapshotInterval = "5m"
|
||||
defaultBaseURL = "/"
|
||||
defaultSecretKeyName = "portainer"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,10 +13,11 @@ const (
|
|||
defaultTLSCertPath = "C:\\certs\\cert.pem"
|
||||
defaultTLSKeyPath = "C:\\certs\\key.pem"
|
||||
defaultHTTPDisabled = "false"
|
||||
defaultHTTPEnabled = "false"
|
||||
defaultHTTPEnabled = "true"
|
||||
defaultSSL = "false"
|
||||
defaultSSLCertPath = "C:\\certs\\portainer.crt"
|
||||
defaultSSLKeyPath = "C:\\certs\\portainer.key"
|
||||
defaultSnapshotInterval = "5m"
|
||||
defaultBaseURL = "/"
|
||||
defaultSecretKeyName = "portainer"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
|
|||
importFile := "/data/import.json"
|
||||
if exists, _ := fileService.FileExists(importFile); exists {
|
||||
if err := store.Import(importFile); err != nil {
|
||||
logrus.WithError(err).Debugf("import %s failed", importFile)
|
||||
logrus.WithError(err).Debugf("Import %s failed", importFile)
|
||||
|
||||
// TODO: should really rollback on failure, but then we have nothing.
|
||||
} else {
|
||||
|
|
@ -23,7 +23,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
|
|||
// I also suspect that everything from "Init to Init" is potentially a migration
|
||||
err := store.Init()
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing data store: %v", err)
|
||||
log.Fatalf("Failed initializing data store: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,18 +1,41 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
type portainerFormatter struct {
|
||||
logrus.TextFormatter
|
||||
}
|
||||
|
||||
func (f *portainerFormatter) Format(entry *logrus.Entry) ([]byte, error) {
|
||||
var levelColor int
|
||||
switch entry.Level {
|
||||
case logrus.DebugLevel, logrus.TraceLevel:
|
||||
levelColor = 31 // gray
|
||||
case logrus.WarnLevel:
|
||||
levelColor = 33 // yellow
|
||||
case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
|
||||
levelColor = 31 // red
|
||||
default:
|
||||
levelColor = 36 // blue
|
||||
}
|
||||
return []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m %s %s\n", levelColor, strings.ToUpper(entry.Level.String()), entry.Time.Format(f.TimestampFormat), entry.Message)), nil
|
||||
}
|
||||
|
||||
func configureLogger() {
|
||||
logger := logrus.New() // logger is to implicitly substitute stdlib's log
|
||||
log.SetOutput(logger.Writer())
|
||||
|
||||
formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
|
||||
formatterLogrus := &portainerFormatter{logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true, TimestampFormat: "2006/01/02 15:04:05", FullTimestamp: true}}
|
||||
|
||||
logger.SetFormatter(formatter)
|
||||
logrus.SetFormatter(formatter)
|
||||
logrus.SetFormatter(formatterLogrus)
|
||||
|
||||
logger.SetLevel(logrus.DebugLevel)
|
||||
logrus.SetLevel(logrus.DebugLevel)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
|
@ -47,12 +48,12 @@ func initCLI() *portainer.CLIFlags {
|
|||
var cliService portainer.CLIService = &cli.Service{}
|
||||
flags, err := cliService.ParseFlags(portainer.APIVersion)
|
||||
if err != nil {
|
||||
log.Fatalf("failed parsing flags: %v", err)
|
||||
log.Fatalf("Failed parsing flags: %v", err)
|
||||
}
|
||||
|
||||
err = cliService.ValidateFlags(flags)
|
||||
if err != nil {
|
||||
log.Fatalf("failed validating flags:%v", err)
|
||||
log.Fatalf("Failed validating flags:%v", err)
|
||||
}
|
||||
return flags
|
||||
}
|
||||
|
|
@ -60,26 +61,26 @@ func initCLI() *portainer.CLIFlags {
|
|||
func initFileService(dataStorePath string) portainer.FileService {
|
||||
fileService, err := filesystem.NewService(dataStorePath, "")
|
||||
if err != nil {
|
||||
log.Fatalf("failed creating file service: %v", err)
|
||||
log.Fatalf("Failed creating file service: %v", err)
|
||||
}
|
||||
return fileService
|
||||
}
|
||||
|
||||
func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
|
||||
connection, err := database.NewDatabase("boltdb", *flags.Data)
|
||||
func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
|
||||
connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
panic(err.Error())
|
||||
}
|
||||
store := datastore.NewStore(*flags.Data, fileService, connection)
|
||||
isNew, err := store.Open()
|
||||
if err != nil {
|
||||
log.Fatalf("failed opening store: %v", err)
|
||||
log.Fatalf("Failed opening store: %v", err)
|
||||
}
|
||||
|
||||
if *flags.Rollback {
|
||||
err := store.Rollback(false)
|
||||
if err != nil {
|
||||
log.Fatalf("failed rolling back: %s", err)
|
||||
log.Fatalf("Failed rolling back: %v", err)
|
||||
}
|
||||
|
||||
log.Println("Exiting rollback")
|
||||
|
|
@ -90,31 +91,27 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
|
|||
// Init sets some defaults - its basically a migration
|
||||
err = store.Init()
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing data store: %v", err)
|
||||
log.Fatalf("Failed initializing data store: %v", err)
|
||||
}
|
||||
|
||||
if isNew {
|
||||
// from MigrateData
|
||||
store.VersionService.StoreDBVersion(portainer.DBVersion)
|
||||
|
||||
// Disabled for now. Can't use feature flags due to the way that works
|
||||
// EXPERIMENTAL, will only activate if `/data/import.json` exists
|
||||
//importFromJson(fileService, store)
|
||||
|
||||
err := updateSettingsFromFlags(store, flags)
|
||||
if err != nil {
|
||||
log.Fatalf("failed updating settings from flags: %v", err)
|
||||
log.Fatalf("Failed updating settings from flags: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
storedVersion, err := store.VersionService.DBVersion()
|
||||
if err != nil {
|
||||
log.Fatalf("Something failed during creation of new database: %v", err)
|
||||
}
|
||||
if storedVersion != portainer.DBVersion {
|
||||
err = store.MigrateData()
|
||||
} else {
|
||||
storedVersion, err := store.VersionService.DBVersion()
|
||||
if err != nil {
|
||||
log.Fatalf("failed migration: %v", err)
|
||||
log.Fatalf("Something Failed during creation of new database: %v", err)
|
||||
}
|
||||
if storedVersion != portainer.DBVersion {
|
||||
err = store.MigrateData()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed migration: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -127,7 +124,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
|
|||
|
||||
err := store.Export(exportFilename)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
|
||||
logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
|
||||
} else {
|
||||
logrus.Debugf("exported to %s", exportFilename)
|
||||
}
|
||||
|
|
@ -139,7 +136,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
|
|||
func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
|
||||
composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
|
||||
if err != nil {
|
||||
log.Fatalf("failed creating compose manager: %s", err)
|
||||
log.Fatalf("Failed creating compose manager: %v", err)
|
||||
}
|
||||
|
||||
return composeWrapper
|
||||
|
|
@ -347,7 +344,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
|
|||
func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
|
||||
existingKeyPair, err := fileService.KeyPairFilesExist()
|
||||
if err != nil {
|
||||
log.Fatalf("failed checking for existing key pair: %v", err)
|
||||
log.Fatalf("Failed checking for existing key pair: %v", err)
|
||||
}
|
||||
|
||||
if existingKeyPair {
|
||||
|
|
@ -491,19 +488,40 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
|
|||
return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
|
||||
}
|
||||
|
||||
func loadEncryptionSecretKey(keyfilename string) []byte {
|
||||
content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
log.Printf("Encryption key file `%s` not present", keyfilename)
|
||||
} else {
|
||||
log.Printf("Error reading encryption key file: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// return a 32 byte hash of the secret (required for AES)
|
||||
hash := sha256.Sum256(content)
|
||||
return hash[:]
|
||||
}
|
||||
|
||||
func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
||||
shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
|
||||
|
||||
fileService := initFileService(*flags.Data)
|
||||
encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
|
||||
if encryptionKey == nil {
|
||||
log.Println("proceeding without encryption key")
|
||||
}
|
||||
|
||||
dataStore := initDataStore(flags, fileService, shutdownCtx)
|
||||
dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx)
|
||||
|
||||
if err := dataStore.CheckCurrentEdition(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
instanceID, err := dataStore.Version().InstanceID()
|
||||
if err != nil {
|
||||
log.Fatalf("failed getting instance id: %v", err)
|
||||
log.Fatalf("Failed getting instance id: %v", err)
|
||||
}
|
||||
|
||||
apiKeyService := initAPIKeyService(dataStore)
|
||||
|
|
@ -514,12 +532,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
}
|
||||
jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing JWT service: %v", err)
|
||||
log.Fatalf("Failed initializing JWT service: %v", err)
|
||||
}
|
||||
|
||||
err = enableFeaturesFromFlags(dataStore, flags)
|
||||
if err != nil {
|
||||
log.Fatalf("failed enabling feature flag: %v", err)
|
||||
log.Fatalf("Failed enabling feature flag: %v", err)
|
||||
}
|
||||
|
||||
ldapService := initLDAPService()
|
||||
|
|
@ -538,12 +556,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
|
||||
sslSettings, err := sslService.GetSSLSettings()
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get ssl settings: %s", err)
|
||||
log.Fatalf("Failed to get ssl settings: %s", err)
|
||||
}
|
||||
|
||||
err = initKeyPair(fileService, digitalSignatureService)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing key pair: %v", err)
|
||||
log.Fatalf("Failed initializing key pair: %v", err)
|
||||
}
|
||||
|
||||
reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
|
||||
|
|
@ -553,7 +571,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
|
||||
snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing snapshot service: %v", err)
|
||||
log.Fatalf("Failed initializing snapshot service: %v", err)
|
||||
}
|
||||
snapshotService.Start()
|
||||
|
||||
|
|
@ -574,37 +592,37 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
|
||||
swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing swarm stack manager: %s", err)
|
||||
log.Fatalf("Failed initializing swarm stack manager: %v", err)
|
||||
}
|
||||
|
||||
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
|
||||
|
||||
helmPackageManager, err := initHelmPackageManager(*flags.Assets)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing helm package manager: %s", err)
|
||||
log.Fatalf("Failed initializing helm package manager: %v", err)
|
||||
}
|
||||
|
||||
err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
|
||||
if err != nil {
|
||||
log.Fatalf("failed loading edge jobs from database: %v", err)
|
||||
log.Fatalf("Failed loading edge jobs from database: %v", err)
|
||||
}
|
||||
|
||||
applicationStatus := initStatus(instanceID)
|
||||
|
||||
err = initEndpoint(flags, dataStore, snapshotService)
|
||||
if err != nil {
|
||||
log.Fatalf("failed initializing environment: %v", err)
|
||||
log.Fatalf("Failed initializing environment: %v", err)
|
||||
}
|
||||
|
||||
adminPasswordHash := ""
|
||||
if *flags.AdminPasswordFile != "" {
|
||||
content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
|
||||
if err != nil {
|
||||
log.Fatalf("failed getting admin password file: %v", err)
|
||||
log.Fatalf("Failed getting admin password file: %v", err)
|
||||
}
|
||||
adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
|
||||
if err != nil {
|
||||
log.Fatalf("failed hashing admin password: %v", err)
|
||||
log.Fatalf("Failed hashing admin password: %v", err)
|
||||
}
|
||||
} else if *flags.AdminPassword != "" {
|
||||
adminPasswordHash = *flags.AdminPassword
|
||||
|
|
@ -613,7 +631,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
if adminPasswordHash != "" {
|
||||
users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
|
||||
if err != nil {
|
||||
log.Fatalf("failed getting admin user: %v", err)
|
||||
log.Fatalf("Failed getting admin user: %v", err)
|
||||
}
|
||||
|
||||
if len(users) == 0 {
|
||||
|
|
@ -625,7 +643,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
}
|
||||
err := dataStore.User().Create(user)
|
||||
if err != nil {
|
||||
log.Fatalf("failed creating admin user: %v", err)
|
||||
log.Fatalf("Failed creating admin user: %v", err)
|
||||
}
|
||||
} else {
|
||||
log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
|
||||
|
|
@ -634,12 +652,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
|
|||
|
||||
err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
|
||||
if err != nil {
|
||||
log.Fatalf("failed starting tunnel server: %s", err)
|
||||
log.Fatalf("Failed starting tunnel server: %v", err)
|
||||
}
|
||||
|
||||
sslDBSettings, err := dataStore.SSLSettings().Settings()
|
||||
if err != nil {
|
||||
log.Fatalf("failed to fetch ssl settings from DB")
|
||||
log.Fatalf("Failed to fetch ssl settings from DB")
|
||||
}
|
||||
|
||||
scheduler := scheduler.NewScheduler(shutdownCtx)
|
||||
|
|
@ -692,6 +710,6 @@ func main() {
|
|||
server := buildServer(flags)
|
||||
log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
|
||||
err := server.Start()
|
||||
log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
|
||||
log.Printf("[INFO] [cmd,main] Http server exited: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,14 +11,16 @@ type Connection interface {
|
|||
// write the db contents to filename as json (the schema needs defining)
|
||||
ExportRaw(filename string) error
|
||||
|
||||
//Rollback(force bool) error
|
||||
//MigrateData(migratorParams *database.MigratorParameters, force bool) error
|
||||
|
||||
// TODO: this one is very database specific atm
|
||||
BackupTo(w io.Writer) error
|
||||
GetDatabaseFilename() string
|
||||
GetDatabaseFileName() string
|
||||
GetDatabaseFilePath() string
|
||||
GetStorePath() string
|
||||
|
||||
IsEncryptedStore() bool
|
||||
NeedsEncryptionMigration() bool
|
||||
SetEncrypted(encrypted bool)
|
||||
|
||||
SetServiceName(bucketName string) error
|
||||
GetObject(bucketName string, key []byte, object interface{}) error
|
||||
UpdateObject(bucketName string, key []byte, object interface{}) error
|
||||
|
|
|
|||
|
|
@ -11,39 +11,78 @@ import (
|
|||
|
||||
"github.com/boltdb/bolt"
|
||||
"github.com/portainer/portainer/api/dataservices/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
DatabaseFileName = "portainer.db"
|
||||
DatabaseFileName = "portainer.db"
|
||||
EncryptedDatabaseFileName = "portainer.edb"
|
||||
)
|
||||
|
||||
type DbConnection struct {
|
||||
Path string
|
||||
Path string
|
||||
EncryptionKey []byte
|
||||
isEncrypted bool
|
||||
|
||||
*bolt.DB
|
||||
}
|
||||
|
||||
func (connection *DbConnection) GetDatabaseFilename() string {
|
||||
// GetDatabaseFileName get the database filename
|
||||
func (connection *DbConnection) GetDatabaseFileName() string {
|
||||
if connection.IsEncryptedStore() {
|
||||
return EncryptedDatabaseFileName
|
||||
}
|
||||
|
||||
return DatabaseFileName
|
||||
}
|
||||
|
||||
// GetDataseFilePath get the path + filename for the database file
|
||||
func (connection *DbConnection) GetDatabaseFilePath() string {
|
||||
if connection.IsEncryptedStore() {
|
||||
return path.Join(connection.Path, EncryptedDatabaseFileName)
|
||||
}
|
||||
|
||||
return path.Join(connection.Path, DatabaseFileName)
|
||||
}
|
||||
|
||||
// GetStorePath get the filename and path for the database file
|
||||
func (connection *DbConnection) GetStorePath() string {
|
||||
return connection.Path
|
||||
}
|
||||
|
||||
func (connection *DbConnection) SetEncrypted(flag bool) {
|
||||
connection.isEncrypted = flag
|
||||
}
|
||||
|
||||
// Return true if the database is encrypted
|
||||
func (connection *DbConnection) IsEncryptedStore() bool {
|
||||
return connection.getEncryptionKey() != nil
|
||||
}
|
||||
|
||||
// NeedsEncryptionMigration returns true if database encryption is enabled and
|
||||
// we have an un-encrypted DB that requires migration to an encrypted DB
|
||||
func (connection *DbConnection) NeedsEncryptionMigration() bool {
|
||||
if connection.EncryptionKey != nil {
|
||||
dbFile := path.Join(connection.Path, DatabaseFileName)
|
||||
if _, err := os.Stat(dbFile); err == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
// This is an existing encrypted store or a new store.
|
||||
// A new store will open encrypted from the outset
|
||||
connection.SetEncrypted(true)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Open opens and initializes the BoltDB database.
|
||||
func (connection *DbConnection) Open() error {
|
||||
|
||||
// Disabled for now. Can't use feature flags due to the way that works
|
||||
// databaseExportPath := path.Join(connection.Path, fmt.Sprintf("raw-%s-%d.json", DatabaseFileName, time.Now().Unix()))
|
||||
// if err := connection.ExportRaw(databaseExportPath); err != nil {
|
||||
// log.Printf("raw export to %s error: %s", databaseExportPath, err)
|
||||
// } else {
|
||||
// log.Printf("raw export to %s success", databaseExportPath)
|
||||
// }
|
||||
|
||||
databasePath := path.Join(connection.Path, DatabaseFileName)
|
||||
logrus.Infof("Loading PortainerDB: %s", connection.GetDatabaseFileName())
|
||||
|
||||
// Now we open the db
|
||||
databasePath := connection.GetDatabaseFilePath()
|
||||
db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -71,12 +110,12 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
|
|||
}
|
||||
|
||||
func (connection *DbConnection) ExportRaw(filename string) error {
|
||||
databasePath := path.Join(connection.Path, DatabaseFileName)
|
||||
databasePath := connection.GetDatabaseFilePath()
|
||||
if _, err := os.Stat(databasePath); err != nil {
|
||||
return fmt.Errorf("stat on %s failed: %s", databasePath, err)
|
||||
}
|
||||
|
||||
b, err := exportJson(databasePath)
|
||||
b, err := connection.exportJson(databasePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -124,7 +163,15 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
|
|||
return err
|
||||
}
|
||||
|
||||
return UnmarshalObject(data, object)
|
||||
return connection.UnmarshalObject(data, object)
|
||||
}
|
||||
|
||||
func (connection *DbConnection) getEncryptionKey() []byte {
|
||||
if !connection.isEncrypted {
|
||||
return nil
|
||||
}
|
||||
|
||||
return connection.EncryptionKey
|
||||
}
|
||||
|
||||
// UpdateObject is a generic function used to update an object inside a database database.
|
||||
|
|
@ -132,7 +179,7 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje
|
|||
return connection.Update(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(bucketName))
|
||||
|
||||
data, err := MarshalObject(object)
|
||||
data, err := connection.MarshalObject(object)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -163,7 +210,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun
|
|||
cursor := bucket.Cursor()
|
||||
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
|
||||
var obj interface{}
|
||||
err := UnmarshalObject(v, &obj)
|
||||
err := connection.UnmarshalObject(v, &obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -205,7 +252,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
|
|||
seqId, _ := bucket.NextSequence()
|
||||
id, obj := fn(seqId)
|
||||
|
||||
data, err := MarshalObject(obj)
|
||||
data, err := connection.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -218,8 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
|
|||
func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
|
||||
return connection.Update(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(bucketName))
|
||||
|
||||
data, err := MarshalObject(obj)
|
||||
data, err := connection.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -240,7 +286,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
|
|||
return err
|
||||
}
|
||||
|
||||
data, err := MarshalObject(obj)
|
||||
data, err := connection.MarshalObject(obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -252,10 +298,9 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
|
|||
func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
|
||||
err := connection.View(func(tx *bolt.Tx) error {
|
||||
bucket := tx.Bucket([]byte(bucketName))
|
||||
|
||||
cursor := bucket.Cursor()
|
||||
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
|
||||
err := UnmarshalObject(v, obj)
|
||||
err := connection.UnmarshalObject(v, obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -277,7 +322,7 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
|
|||
|
||||
cursor := bucket.Cursor()
|
||||
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
|
||||
err := UnmarshalObjectWithJsoniter(v, obj)
|
||||
err := connection.UnmarshalObjectWithJsoniter(v, obj)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,8 +10,9 @@ import (
|
|||
|
||||
// inspired by github.com/konoui/boltdb-exporter (which has no license)
|
||||
// but very much simplified, based on how we use boltdb
|
||||
func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
|
||||
logrus.WithField("databasePath", databasePath).Infof("exportJson")
|
||||
|
||||
func exportJson(databasePath string) ([]byte, error) {
|
||||
connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
|
||||
if err != nil {
|
||||
return []byte("{}"), err
|
||||
|
|
@ -31,7 +32,7 @@ func exportJson(databasePath string) ([]byte, error) {
|
|||
continue
|
||||
}
|
||||
var obj interface{}
|
||||
err := UnmarshalObject(v, &obj)
|
||||
err := c.UnmarshalObject(v, &obj)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v))
|
||||
obj = v
|
||||
|
|
|
|||
|
|
@ -1,42 +1,123 @@
|
|||
package boltdb
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
|
||||
|
||||
// MarshalObject encodes an object to binary format
|
||||
func MarshalObject(object interface{}) ([]byte, error) {
|
||||
func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
|
||||
// Special case for the VERSION bucket. Here we're not using json
|
||||
if v, ok := object.(string); ok {
|
||||
return []byte(v), nil
|
||||
data = []byte(v)
|
||||
} else {
|
||||
data, err = json.Marshal(object)
|
||||
if err != nil {
|
||||
return data, err
|
||||
}
|
||||
}
|
||||
|
||||
return json.Marshal(object)
|
||||
if connection.getEncryptionKey() == nil {
|
||||
return data, nil
|
||||
}
|
||||
return encrypt(data, connection.getEncryptionKey())
|
||||
}
|
||||
|
||||
// UnmarshalObject decodes an object from binary data
|
||||
func UnmarshalObject(data []byte, object interface{}) error {
|
||||
// Special case for the VERSION bucket. Here we're not using json
|
||||
// So we need to return it as a string
|
||||
err := json.Unmarshal(data, object)
|
||||
if err != nil {
|
||||
if s, ok := object.(*string); ok {
|
||||
*s = string(data)
|
||||
return nil
|
||||
func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
|
||||
var err error
|
||||
if connection.getEncryptionKey() != nil {
|
||||
data, err = decrypt(data, connection.getEncryptionKey())
|
||||
if err != nil {
|
||||
errors.Wrapf(err, "Failed decrypting object")
|
||||
}
|
||||
}
|
||||
e := json.Unmarshal(data, object)
|
||||
if e != nil {
|
||||
// Special case for the VERSION bucket. Here we're not using json
|
||||
// So we need to return it as a string
|
||||
s, ok := object.(*string)
|
||||
if !ok {
|
||||
return errors.Wrap(err, e.Error())
|
||||
}
|
||||
|
||||
return err
|
||||
*s = string(data)
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
|
||||
// UnmarshalObjectWithJsoniter decodes an object from binary data
|
||||
// using the jsoniter library. It is mainly used to accelerate environment(endpoint)
|
||||
// decoding at the moment.
|
||||
func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
|
||||
func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
|
||||
if connection.getEncryptionKey() != nil {
|
||||
var err error
|
||||
data, err = decrypt(data, connection.getEncryptionKey())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
return jsoni.Unmarshal(data, &object)
|
||||
}
|
||||
|
||||
// mmm, don't have a KMS .... aes GCM seems the most likely from
|
||||
// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
|
||||
|
||||
func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
|
||||
block, _ := aes.NewCipher(passphrase)
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return encrypted, err
|
||||
}
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return encrypted, err
|
||||
}
|
||||
ciphertextByte := gcm.Seal(
|
||||
nonce,
|
||||
nonce,
|
||||
plaintext,
|
||||
nil)
|
||||
return ciphertextByte, nil
|
||||
}
|
||||
|
||||
func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
|
||||
if string(encrypted) == "false" {
|
||||
return []byte("false"), nil
|
||||
}
|
||||
block, err := aes.NewCipher(passphrase)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error creating cypher block")
|
||||
}
|
||||
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error creating GCM")
|
||||
}
|
||||
|
||||
nonceSize := gcm.NonceSize()
|
||||
if len(encrypted) < nonceSize {
|
||||
return encrypted, errEncryptedStringTooShort
|
||||
}
|
||||
|
||||
nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
|
||||
plaintextByte, err = gcm.Open(
|
||||
nil,
|
||||
nonce,
|
||||
ciphertextByteClean,
|
||||
nil)
|
||||
if err != nil {
|
||||
return encrypted, errors.Wrap(err, "Error decrypting text")
|
||||
}
|
||||
|
||||
return plaintextByte, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package boltdb
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
|
|
@ -8,9 +9,17 @@ import (
|
|||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
|
||||
const (
|
||||
jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
|
||||
passphrase = "my secret key"
|
||||
)
|
||||
|
||||
func Test_MarshalObject(t *testing.T) {
|
||||
func secretToEncryptionKey(passphrase string) []byte {
|
||||
hash := sha256.Sum256([]byte(passphrase))
|
||||
return hash[:]
|
||||
}
|
||||
|
||||
func Test_MarshalObjectUnencrypted(t *testing.T) {
|
||||
is := assert.New(t)
|
||||
|
||||
uuid := uuid.Must(uuid.NewV4())
|
||||
|
|
@ -73,16 +82,18 @@ func Test_MarshalObject(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
conn := DbConnection{}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
data, err := MarshalObject(test.object)
|
||||
data, err := conn.MarshalObject(test.object)
|
||||
is.NoError(err)
|
||||
is.Equal(test.expected, string(data))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_UnMarshalObject(t *testing.T) {
|
||||
func Test_UnMarshalObjectUnencrypted(t *testing.T) {
|
||||
is := assert.New(t)
|
||||
|
||||
// Based on actual data entering and what we expect out of the function
|
||||
|
|
@ -105,18 +116,62 @@ func Test_UnMarshalObject(t *testing.T) {
|
|||
expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
|
||||
},
|
||||
{
|
||||
// An unmarshalled json object string should return the same as a string without error also
|
||||
// An un-marshalled json object string should return the same as a string without error also
|
||||
object: []byte(jsonobject),
|
||||
expected: jsonobject,
|
||||
},
|
||||
}
|
||||
|
||||
conn := DbConnection{}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
var object string
|
||||
err := UnmarshalObject(test.object, &object)
|
||||
err := conn.UnmarshalObject(test.object, &object)
|
||||
is.NoError(err)
|
||||
is.Equal(test.expected, string(object))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_ObjectMarshallingEncrypted(t *testing.T) {
|
||||
is := assert.New(t)
|
||||
|
||||
// Based on actual data entering and what we expect out of the function
|
||||
|
||||
tests := []struct {
|
||||
object []byte
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
object: []byte(""),
|
||||
},
|
||||
{
|
||||
object: []byte("35"),
|
||||
},
|
||||
{
|
||||
// An unmarshalled byte string should return the same without error
|
||||
object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
|
||||
},
|
||||
{
|
||||
// An un-marshalled json object string should return the same as a string without error also
|
||||
object: []byte(jsonobject),
|
||||
},
|
||||
}
|
||||
|
||||
key := secretToEncryptionKey(passphrase)
|
||||
conn := DbConnection{EncryptionKey: key}
|
||||
for _, test := range tests {
|
||||
t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
|
||||
|
||||
data, err := conn.MarshalObject(test.object)
|
||||
is.NoError(err)
|
||||
|
||||
var object []byte
|
||||
err = conn.UnmarshalObject(data, &object)
|
||||
|
||||
is.NoError(err)
|
||||
is.Equal(test.object, object)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,15 +2,19 @@ package database
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/database/boltdb"
|
||||
)
|
||||
|
||||
// NewDatabase should use config options to return a connection to the requested database
|
||||
func NewDatabase(storeType, storePath string) (connection portainer.Connection, err error) {
|
||||
func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
|
||||
switch storeType {
|
||||
case "boltdb":
|
||||
return &boltdb.DbConnection{Path: storePath}, nil
|
||||
return &boltdb.DbConnection{
|
||||
Path: storePath,
|
||||
EncryptionKey: encryptionKey,
|
||||
}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("Unknown storage database: %s", storeType)
|
||||
return nil, fmt.Errorf("unknown storage database: %s", storeType)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
|
|||
endpoint, ok := obj.(*portainer.Endpoint)
|
||||
if !ok {
|
||||
logrus.WithField("obj", obj).Errorf("Failed to convert to Endpoint object")
|
||||
return nil, fmt.Errorf("Failed to convert to Endpoint object: %s", obj)
|
||||
return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
|
||||
}
|
||||
endpoints = append(endpoints, *endpoint)
|
||||
return &portainer.Endpoint{}, nil
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import "errors"
|
|||
|
||||
var (
|
||||
// TODO: i'm pretty sure this needs wrapping at several levels
|
||||
ErrObjectNotFound = errors.New("Object not found inside the database")
|
||||
ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
|
||||
ErrObjectNotFound = errors.New("object not found inside the database")
|
||||
ErrWrongDBEdition = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
|
||||
ErrDBImportFailed = errors.New("importing backup failed")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
|
|||
}
|
||||
|
||||
//HelmUserRepository returns an array of all HelmUserRepository
|
||||
func (service *Service) HelmUserRepositorys() ([]portainer.HelmUserRepository, error) {
|
||||
func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
|
||||
var repos = make([]portainer.HelmUserRepository, 0)
|
||||
|
||||
err := service.connection.GetAll(
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ type (
|
|||
|
||||
// HelmUserRepositoryService represents a service to manage HelmUserRepositories
|
||||
HelmUserRepositoryService interface {
|
||||
HelmUserRepositorys() ([]portainer.HelmUserRepository, error)
|
||||
HelmUserRepositories() ([]portainer.HelmUserRepository, error)
|
||||
HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
|
||||
Create(record *portainer.HelmUserRepository) error
|
||||
UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ func (store *Store) createBackupFolders() {
|
|||
}
|
||||
|
||||
func (store *Store) databasePath() string {
|
||||
return path.Join(store.connection.GetStorePath(), store.connection.GetDatabaseFilename())
|
||||
return store.connection.GetDatabaseFilePath()
|
||||
}
|
||||
|
||||
func (store *Store) commonBackupDir() string {
|
||||
|
|
@ -84,7 +84,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
|
|||
options.BackupDir = store.commonBackupDir()
|
||||
}
|
||||
if options.BackupFileName == "" {
|
||||
options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFilename(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
|
||||
options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
|
||||
}
|
||||
if options.BackupPath == "" {
|
||||
options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ func TestBackup(t *testing.T) {
|
|||
store.VersionService.StoreDBVersion(portainer.DBVersion)
|
||||
store.backupWithOptions(nil)
|
||||
|
||||
backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
|
||||
backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
|
||||
if !isFileExist(backupFileName) {
|
||||
t.Errorf("Expect backup file to be created %s", backupFileName)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,15 @@
|
|||
package datastore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
portainer "github.com/portainer/portainer/api"
|
||||
"github.com/portainer/portainer/api/dataservices/errors"
|
||||
portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
func (store *Store) version() (int, error) {
|
||||
|
|
@ -34,6 +39,12 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
|
|||
// Open opens and initializes the BoltDB database.
|
||||
func (store *Store) Open() (newStore bool, err error) {
|
||||
newStore = true
|
||||
|
||||
encryptionReq := store.connection.NeedsEncryptionMigration()
|
||||
if encryptionReq {
|
||||
store.encryptDB()
|
||||
}
|
||||
|
||||
err = store.connection.Open()
|
||||
if err != nil {
|
||||
return newStore, err
|
||||
|
|
@ -45,8 +56,18 @@ func (store *Store) Open() (newStore bool, err error) {
|
|||
}
|
||||
|
||||
// if we have DBVersion in the database then ensure we flag this as NOT a new store
|
||||
if _, err := store.VersionService.DBVersion(); err == nil {
|
||||
newStore = false
|
||||
version, err := store.VersionService.DBVersion()
|
||||
if err != nil {
|
||||
if store.IsErrObjectNotFound(err) {
|
||||
return newStore, nil
|
||||
}
|
||||
|
||||
return newStore, err
|
||||
}
|
||||
|
||||
if version > 0 {
|
||||
logrus.WithField("version", version).Infof("Opened existing store")
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return newStore, nil
|
||||
|
|
@ -65,16 +86,81 @@ func (store *Store) BackupTo(w io.Writer) error {
|
|||
// CheckCurrentEdition checks if current edition is community edition
|
||||
func (store *Store) CheckCurrentEdition() error {
|
||||
if store.edition() != portainer.PortainerCE {
|
||||
return errors.ErrWrongDBEdition
|
||||
return portainerErrors.ErrWrongDBEdition
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: move the use of this to dataservices.IsErrObjectNotFound()?
|
||||
func (store *Store) IsErrObjectNotFound(e error) bool {
|
||||
return e == errors.ErrObjectNotFound
|
||||
return e == portainerErrors.ErrObjectNotFound
|
||||
}
|
||||
|
||||
func (store *Store) Rollback(force bool) error {
|
||||
return store.connectionRollback(force)
|
||||
}
|
||||
|
||||
func (store *Store) encryptDB() error {
|
||||
store.connection.SetEncrypted(false)
|
||||
err := store.connection.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = store.initServices()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The DB is not currently encrypted. First save the encrypted db filename
|
||||
oldFilename := store.connection.GetDatabaseFilePath()
|
||||
logrus.Infof("Encrypting database")
|
||||
|
||||
// export file path for backup
|
||||
exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
|
||||
|
||||
logrus.Infof("Exporting database backup to %s", exportFilename)
|
||||
err = store.Export(exportFilename)
|
||||
if err != nil {
|
||||
logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
|
||||
return err
|
||||
}
|
||||
|
||||
logrus.Infof("Database backup exported")
|
||||
|
||||
// Close existing un-encrypted db so that we can delete the file later
|
||||
store.connection.Close()
|
||||
|
||||
// Tell the db layer to create an encrypted db when opened
|
||||
store.connection.SetEncrypted(true)
|
||||
store.connection.Open()
|
||||
|
||||
// We have to init services before import
|
||||
err = store.initServices()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = store.Import(exportFilename)
|
||||
if err != nil {
|
||||
// Remove the new encrypted file that we failed to import
|
||||
os.Remove(store.connection.GetDatabaseFilePath())
|
||||
logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
|
||||
}
|
||||
|
||||
err = os.Remove(oldFilename)
|
||||
if err != nil {
|
||||
logrus.Errorf("Failed to remove the un-encrypted db file")
|
||||
}
|
||||
|
||||
err = os.Remove(exportFilename)
|
||||
if err != nil {
|
||||
logrus.Errorf("Failed to remove the json backup file")
|
||||
}
|
||||
|
||||
// Close db connection
|
||||
store.connection.Close()
|
||||
|
||||
logrus.Info("Database successfully encrypted")
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -363,118 +363,184 @@ func (store *Store) Export(filename string) (err error) {
|
|||
backup := storeExport{}
|
||||
|
||||
if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Custom Templates")
|
||||
}
|
||||
} else {
|
||||
backup.CustomTemplate = c
|
||||
}
|
||||
|
||||
if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Edge Groups")
|
||||
}
|
||||
} else {
|
||||
backup.EdgeGroup = e
|
||||
}
|
||||
|
||||
if e, err := store.EdgeJob().EdgeJobs(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Edge Jobs")
|
||||
}
|
||||
} else {
|
||||
backup.EdgeJob = e
|
||||
}
|
||||
|
||||
if e, err := store.EdgeStack().EdgeStacks(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Edge Stacks")
|
||||
}
|
||||
} else {
|
||||
backup.EdgeStack = e
|
||||
}
|
||||
|
||||
if e, err := store.Endpoint().Endpoints(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Endpoints")
|
||||
}
|
||||
} else {
|
||||
backup.Endpoint = e
|
||||
}
|
||||
|
||||
if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Endpoint Groups")
|
||||
}
|
||||
} else {
|
||||
backup.EndpointGroup = e
|
||||
}
|
||||
|
||||
if r, err := store.EndpointRelation().EndpointRelations(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Endpoint Relations")
|
||||
}
|
||||
} else {
|
||||
backup.EndpointRelation = r
|
||||
}
|
||||
|
||||
if r, err := store.ExtensionService.Extensions(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Extensions")
|
||||
}
|
||||
} else {
|
||||
backup.Extensions = r
|
||||
}
|
||||
if r, err := store.HelmUserRepository().HelmUserRepositorys(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
|
||||
if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Helm User Repositories")
|
||||
}
|
||||
} else {
|
||||
backup.HelmUserRepository = r
|
||||
}
|
||||
|
||||
if r, err := store.Registry().Registries(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Registries")
|
||||
}
|
||||
} else {
|
||||
backup.Registry = r
|
||||
}
|
||||
|
||||
if c, err := store.ResourceControl().ResourceControls(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Resource Controls")
|
||||
}
|
||||
} else {
|
||||
backup.ResourceControl = c
|
||||
}
|
||||
|
||||
if role, err := store.Role().Roles(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Roles")
|
||||
}
|
||||
} else {
|
||||
backup.Role = role
|
||||
}
|
||||
|
||||
if r, err := store.ScheduleService.Schedules(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Schedules")
|
||||
}
|
||||
} else {
|
||||
backup.Schedules = r
|
||||
}
|
||||
|
||||
if settings, err := store.Settings().Settings(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Settings")
|
||||
}
|
||||
} else {
|
||||
backup.Settings = *settings
|
||||
}
|
||||
|
||||
if settings, err := store.SSLSettings().Settings(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting SSL Settings")
|
||||
}
|
||||
} else {
|
||||
backup.SSLSettings = *settings
|
||||
}
|
||||
|
||||
if t, err := store.Stack().Stacks(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Stacks")
|
||||
}
|
||||
} else {
|
||||
backup.Stack = t
|
||||
}
|
||||
|
||||
if t, err := store.Tag().Tags(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Tags")
|
||||
}
|
||||
} else {
|
||||
backup.Tag = t
|
||||
}
|
||||
|
||||
if t, err := store.TeamMembership().TeamMemberships(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Team Memberships")
|
||||
}
|
||||
} else {
|
||||
backup.TeamMembership = t
|
||||
}
|
||||
|
||||
if t, err := store.Team().Teams(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Teams")
|
||||
}
|
||||
} else {
|
||||
backup.Team = t
|
||||
}
|
||||
|
||||
if info, err := store.TunnelServer().Info(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Tunnel Server")
|
||||
}
|
||||
} else {
|
||||
backup.TunnelServer = *info
|
||||
}
|
||||
|
||||
if users, err := store.User().Users(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Users")
|
||||
}
|
||||
} else {
|
||||
backup.User = users
|
||||
}
|
||||
|
||||
if webhooks, err := store.Webhook().Webhooks(); err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting Webhooks")
|
||||
}
|
||||
} else {
|
||||
backup.Webhook = webhooks
|
||||
}
|
||||
|
||||
v, err := store.Version().DBVersion()
|
||||
if err != nil {
|
||||
logrus.WithError(err).Debugf("Export boom")
|
||||
if err != nil && !store.IsErrObjectNotFound(err) {
|
||||
logrus.WithError(err).Errorf("Exporting DB version")
|
||||
}
|
||||
instance, _ := store.Version().InstanceID()
|
||||
backup.Version = map[string]string{
|
||||
|
|
@ -518,50 +584,66 @@ func (store *Store) Import(filename string) (err error) {
|
|||
for _, v := range backup.CustomTemplate {
|
||||
store.CustomTemplate().UpdateCustomTemplate(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.EdgeGroup {
|
||||
store.EdgeGroup().UpdateEdgeGroup(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.EdgeJob {
|
||||
store.EdgeJob().UpdateEdgeJob(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.EdgeStack {
|
||||
store.EdgeStack().UpdateEdgeStack(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.Endpoint {
|
||||
store.Endpoint().UpdateEndpoint(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.EndpointGroup {
|
||||
store.EndpointGroup().UpdateEndpointGroup(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.EndpointRelation {
|
||||
store.EndpointRelation().UpdateEndpointRelation(v.EndpointID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.HelmUserRepository {
|
||||
store.HelmUserRepository().UpdateHelmUserRepository(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.Registry {
|
||||
store.Registry().UpdateRegistry(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.ResourceControl {
|
||||
store.ResourceControl().UpdateResourceControl(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.Role {
|
||||
store.Role().UpdateRole(v.ID, &v)
|
||||
}
|
||||
|
||||
store.Settings().UpdateSettings(&backup.Settings)
|
||||
store.SSLSettings().UpdateSettings(&backup.SSLSettings)
|
||||
|
||||
for _, v := range backup.Stack {
|
||||
store.Stack().UpdateStack(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.Tag {
|
||||
store.Tag().UpdateTag(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.TeamMembership {
|
||||
store.TeamMembership().UpdateTeamMembership(v.ID, &v)
|
||||
}
|
||||
|
||||
for _, v := range backup.Team {
|
||||
store.Team().UpdateTeam(v.ID, &v)
|
||||
}
|
||||
|
||||
store.TunnelServer().UpdateInfo(&backup.TunnelServer)
|
||||
|
||||
for _, user := range backup.User {
|
||||
|
|
@ -570,10 +652,9 @@ func (store *Store) Import(filename string) (err error) {
|
|||
}
|
||||
}
|
||||
|
||||
// backup[store.Webhook().BucketName()], err = store.Webhook().Webhooks()
|
||||
// if err != nil {
|
||||
// logrus.WithError(err).Debugf("Export boom")
|
||||
// }
|
||||
for _, v := range backup.Webhook {
|
||||
store.Webhook().UpdateWebhook(v.ID, &v)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ func NewTestStore(init bool) (bool, *Store, func(), error) {
|
|||
return false, nil, nil, err
|
||||
}
|
||||
|
||||
connection, err := database.NewDatabase("boltdb", storePath)
|
||||
connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -97,6 +97,7 @@ type (
|
|||
Rollback *bool
|
||||
SnapshotInterval *string
|
||||
BaseURL *string
|
||||
SecretKeyName *string
|
||||
}
|
||||
|
||||
// CustomTemplate represents a custom template
|
||||
|
|
|
|||
Loading…
Reference in New Issue