feat(database): add encryption support EE-1983 (#6316)

* bootstrap encryption key

* secret key message change in cli and secret key file content trimmed

* Migrate encryption code to latest version

* pull in newer code

* tidying up

* working data encryption layer

* fix tests

* remove stray comment

* fix a few minor issues and improve the comments

* split out databasefilename with param to two methods to be more obvious

* DB encryption integration (#6374)

* json methods moved under DBConnection

* store encryption fixed

* cleaned

* review comments addressed

* newstore value fixed

* backup test updated

* logrus format config updated

* Fix for newStore

Co-authored-by: Matt Hook <hookenz@gmail.com>

* Minor improvements

* Improve the export code.  Add missing webhook for import

* rename HelmUserRepositorys to HelmUserRepositories

* fix logging messages

* when starting portainer with a key (first use) http is disabled by default.  But when starting fresh without a key, http is enabled?

* Fix bug for default settings on new installs

Co-authored-by: Prabhat Khera <prabhat.khera@portainer.io>
Co-authored-by: Prabhat Khera <91852476+prabhat-org@users.noreply.github.com>

api/cli/cli.go

@@ -57,6 +57,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		Logo:                      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
 		Templates:                 kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
 		BaseURL:                   kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(),
+		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 	}
 
 	kingpin.Parse()

api/cli/defaults.go

@@ -16,10 +16,11 @@ const (
 	defaultTLSCertPath         = "/certs/cert.pem"
 	defaultTLSKeyPath          = "/certs/key.pem"
 	defaultHTTPDisabled        = "false"
-	defaultHTTPEnabled         = "false"
+	defaultHTTPEnabled         = "true"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "/certs/portainer.crt"
 	defaultSSLKeyPath          = "/certs/portainer.key"
 	defaultSnapshotInterval    = "5m"
 	defaultBaseURL             = "/"
+	defaultSecretKeyName       = "portainer"
 )

api/cli/defaults_windows.go

@@ -13,10 +13,11 @@ const (
 	defaultTLSCertPath         = "C:\\certs\\cert.pem"
 	defaultTLSKeyPath          = "C:\\certs\\key.pem"
 	defaultHTTPDisabled        = "false"
-	defaultHTTPEnabled         = "false"
+	defaultHTTPEnabled         = "true"
 	defaultSSL                 = "false"
 	defaultSSLCertPath         = "C:\\certs\\portainer.crt"
 	defaultSSLKeyPath          = "C:\\certs\\portainer.key"
 	defaultSnapshotInterval    = "5m"
 	defaultBaseURL             = "/"
+	defaultSecretKeyName       = "portainer"
 )

api/cmd/portainer/import.go

@@ -13,7 +13,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
 	importFile := "/data/import.json"
 	if exists, _ := fileService.FileExists(importFile); exists {
 		if err := store.Import(importFile); err != nil {
-			logrus.WithError(err).Debugf("import %s failed", importFile)
+			logrus.WithError(err).Debugf("Import %s failed", importFile)
 
 			// TODO: should really rollback on failure, but then we have nothing.
 		} else {
@@ -23,7 +23,7 @@ func importFromJson(fileService portainer.FileService, store *datastore.Store) {
 		// I also suspect that everything from "Init to Init" is potentially a migration
 		err := store.Init()
 		if err != nil {
-			log.Fatalf("failed initializing data store: %v", err)
+			log.Fatalf("Failed initializing data store: %v", err)
 		}
 	}
 }

api/cmd/portainer/log.go

@@ -1,18 +1,41 @@
 package main
 
 import (
+	"fmt"
 	"log"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 )
 
+type portainerFormatter struct {
+	logrus.TextFormatter
+}
+
+func (f *portainerFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	var levelColor int
+	switch entry.Level {
+	case logrus.DebugLevel, logrus.TraceLevel:
+		levelColor = 31 // gray
+	case logrus.WarnLevel:
+		levelColor = 33 // yellow
+	case logrus.ErrorLevel, logrus.FatalLevel, logrus.PanicLevel:
+		levelColor = 31 // red
+	default:
+		levelColor = 36 // blue
+	}
+	return []byte(fmt.Sprintf("\x1b[%dm%s\x1b[0m %s %s\n", levelColor, strings.ToUpper(entry.Level.String()), entry.Time.Format(f.TimestampFormat), entry.Message)), nil
+}
+
 func configureLogger() {
 	logger := logrus.New() // logger is to implicitly substitute stdlib's log
 	log.SetOutput(logger.Writer())
 
 	formatter := &logrus.TextFormatter{DisableTimestamp: true, DisableLevelTruncation: true}
+	formatterLogrus := &portainerFormatter{logrus.TextFormatter{DisableTimestamp: false, DisableLevelTruncation: true, TimestampFormat: "2006/01/02 15:04:05", FullTimestamp: true}}
+
 	logger.SetFormatter(formatter)
-	logrus.SetFormatter(formatter)
+	logrus.SetFormatter(formatterLogrus)
 
 	logger.SetLevel(logrus.DebugLevel)
 	logrus.SetLevel(logrus.DebugLevel)

api/cmd/portainer/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"log"
 	"os"
@@ -47,12 +48,12 @@ func initCLI() *portainer.CLIFlags {
 	var cliService portainer.CLIService = &cli.Service{}
 	flags, err := cliService.ParseFlags(portainer.APIVersion)
 	if err != nil {
-		log.Fatalf("failed parsing flags: %v", err)
+		log.Fatalf("Failed parsing flags: %v", err)
 	}
 
 	err = cliService.ValidateFlags(flags)
 	if err != nil {
-		log.Fatalf("failed validating flags:%v", err)
+		log.Fatalf("Failed validating flags:%v", err)
 	}
 	return flags
 }
@@ -60,26 +61,26 @@ func initCLI() *portainer.CLIFlags {
 func initFileService(dataStorePath string) portainer.FileService {
 	fileService, err := filesystem.NewService(dataStorePath, "")
 	if err != nil {
-		log.Fatalf("failed creating file service: %v", err)
+		log.Fatalf("Failed creating file service: %v", err)
 	}
 	return fileService
 }
 
-func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
+func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
-	connection, err := database.NewDatabase("boltdb", *flags.Data)
+	connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
 	if err != nil {
-		panic(err)
+		panic(err.Error())
 	}
 	store := datastore.NewStore(*flags.Data, fileService, connection)
 	isNew, err := store.Open()
 	if err != nil {
-		log.Fatalf("failed opening store: %v", err)
+		log.Fatalf("Failed opening store: %v", err)
 	}
 
 	if *flags.Rollback {
 		err := store.Rollback(false)
 		if err != nil {
-			log.Fatalf("failed rolling back: %s", err)
+			log.Fatalf("Failed rolling back: %v", err)
 		}
 
 		log.Println("Exiting rollback")
@@ -90,31 +91,27 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
 	// Init sets some defaults - its basically a migration
 	err = store.Init()
 	if err != nil {
-		log.Fatalf("failed initializing data store: %v", err)
+		log.Fatalf("Failed initializing data store: %v", err)
 	}
 
 	if isNew {
 		// from MigrateData
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
 
-		// Disabled for now.  Can't use feature flags due to the way that works
-		// EXPERIMENTAL, will only activate if `/data/import.json` exists
-		//importFromJson(fileService, store)
-
 		err := updateSettingsFromFlags(store, flags)
 		if err != nil {
-			log.Fatalf("failed updating settings from flags: %v", err)
+			log.Fatalf("Failed updating settings from flags: %v", err)
 		}
-	}
+	} else {
-
+		storedVersion, err := store.VersionService.DBVersion()
-	storedVersion, err := store.VersionService.DBVersion()
-	if err != nil {
-		log.Fatalf("Something failed during creation of new database: %v", err)
-	}
-	if storedVersion != portainer.DBVersion {
-		err = store.MigrateData()
 		if err != nil {
-			log.Fatalf("failed migration: %v", err)
+			log.Fatalf("Something Failed during creation of new database: %v", err)
+		}
+		if storedVersion != portainer.DBVersion {
+			err = store.MigrateData()
+			if err != nil {
+				log.Fatalf("Failed migration: %v", err)
+			}
 		}
 	}
 
@@ -127,7 +124,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
 
 		err := store.Export(exportFilename)
 		if err != nil {
-			logrus.WithError(err).Debugf("failed to export to %s", exportFilename)
+			logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
 		} else {
 			logrus.Debugf("exported to %s", exportFilename)
 		}
@@ -139,7 +136,7 @@ func initDataStore(flags *portainer.CLIFlags, fileService portainer.FileService,
 func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
 	composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
 	if err != nil {
-		log.Fatalf("failed creating compose manager: %s", err)
+		log.Fatalf("Failed creating compose manager: %v", err)
 	}
 
 	return composeWrapper
@@ -347,7 +344,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
 func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
 	existingKeyPair, err := fileService.KeyPairFilesExist()
 	if err != nil {
-		log.Fatalf("failed checking for existing key pair: %v", err)
+		log.Fatalf("Failed checking for existing key pair: %v", err)
 	}
 
 	if existingKeyPair {
@@ -491,19 +488,40 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
 	return createUnsecuredEndpoint(*flags.EndpointURL, dataStore, snapshotService)
 }
 
+func loadEncryptionSecretKey(keyfilename string) []byte {
+	content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("Encryption key file `%s` not present", keyfilename)
+		} else {
+			log.Printf("Error reading encryption key file: %v", err)
+		}
+
+		return nil
+	}
+
+	// return a 32 byte hash of the secret (required for AES)
+	hash := sha256.Sum256(content)
+	return hash[:]
+}
+
 func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	shutdownCtx, shutdownTrigger := context.WithCancel(context.Background())
 
 	fileService := initFileService(*flags.Data)
+	encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
+	if encryptionKey == nil {
+		log.Println("proceeding without encryption key")
+	}
 
-	dataStore := initDataStore(flags, fileService, shutdownCtx)
+	dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx)
 
 	if err := dataStore.CheckCurrentEdition(); err != nil {
 		log.Fatal(err)
 	}
 	instanceID, err := dataStore.Version().InstanceID()
 	if err != nil {
-		log.Fatalf("failed getting instance id: %v", err)
+		log.Fatalf("Failed getting instance id: %v", err)
 	}
 
 	apiKeyService := initAPIKeyService(dataStore)
@@ -514,12 +532,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	}
 	jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore)
 	if err != nil {
-		log.Fatalf("failed initializing JWT service: %v", err)
+		log.Fatalf("Failed initializing JWT service: %v", err)
 	}
 
 	err = enableFeaturesFromFlags(dataStore, flags)
 	if err != nil {
-		log.Fatalf("failed enabling feature flag: %v", err)
+		log.Fatalf("Failed enabling feature flag: %v", err)
 	}
 
 	ldapService := initLDAPService()
@@ -538,12 +556,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	sslSettings, err := sslService.GetSSLSettings()
 	if err != nil {
-		log.Fatalf("failed to get ssl settings: %s", err)
+		log.Fatalf("Failed to get ssl settings: %s", err)
 	}
 
 	err = initKeyPair(fileService, digitalSignatureService)
 	if err != nil {
-		log.Fatalf("failed initializing key pair: %v", err)
+		log.Fatalf("Failed initializing key pair: %v", err)
 	}
 
 	reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
@@ -553,7 +571,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
 	if err != nil {
-		log.Fatalf("failed initializing snapshot service: %v", err)
+		log.Fatalf("Failed initializing snapshot service: %v", err)
 	}
 	snapshotService.Start()
 
@@ -574,37 +592,37 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
 	if err != nil {
-		log.Fatalf("failed initializing swarm stack manager: %s", err)
+		log.Fatalf("Failed initializing swarm stack manager: %v", err)
 	}
 
 	kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
 
 	helmPackageManager, err := initHelmPackageManager(*flags.Assets)
 	if err != nil {
-		log.Fatalf("failed initializing helm package manager: %s", err)
+		log.Fatalf("Failed initializing helm package manager: %v", err)
 	}
 
 	err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
 	if err != nil {
-		log.Fatalf("failed loading edge jobs from database: %v", err)
+		log.Fatalf("Failed loading edge jobs from database: %v", err)
 	}
 
 	applicationStatus := initStatus(instanceID)
 
 	err = initEndpoint(flags, dataStore, snapshotService)
 	if err != nil {
-		log.Fatalf("failed initializing environment: %v", err)
+		log.Fatalf("Failed initializing environment: %v", err)
 	}
 
 	adminPasswordHash := ""
 	if *flags.AdminPasswordFile != "" {
 		content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
 		if err != nil {
-			log.Fatalf("failed getting admin password file: %v", err)
+			log.Fatalf("Failed getting admin password file: %v", err)
 		}
 		adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
 		if err != nil {
-			log.Fatalf("failed hashing admin password: %v", err)
+			log.Fatalf("Failed hashing admin password: %v", err)
 		}
 	} else if *flags.AdminPassword != "" {
 		adminPasswordHash = *flags.AdminPassword
@@ -613,7 +631,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 	if adminPasswordHash != "" {
 		users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
 		if err != nil {
-			log.Fatalf("failed getting admin user: %v", err)
+			log.Fatalf("Failed getting admin user: %v", err)
 		}
 
 		if len(users) == 0 {
@@ -625,7 +643,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 			}
 			err := dataStore.User().Create(user)
 			if err != nil {
-				log.Fatalf("failed creating admin user: %v", err)
+				log.Fatalf("Failed creating admin user: %v", err)
 			}
 		} else {
 			log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
@@ -634,12 +652,12 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
 
 	err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
 	if err != nil {
-		log.Fatalf("failed starting tunnel server: %s", err)
+		log.Fatalf("Failed starting tunnel server: %v", err)
 	}
 
 	sslDBSettings, err := dataStore.SSLSettings().Settings()
 	if err != nil {
-		log.Fatalf("failed to fetch ssl settings from DB")
+		log.Fatalf("Failed to fetch ssl settings from DB")
 	}
 
 	scheduler := scheduler.NewScheduler(shutdownCtx)
@@ -692,6 +710,6 @@ func main() {
 		server := buildServer(flags)
 		log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
 		err := server.Start()
-		log.Printf("[INFO] [cmd,main] Http server exited: %s\n", err)
+		log.Printf("[INFO] [cmd,main] Http server exited: %v\n", err)
 	}
 }

api/connection.go

@@ -11,14 +11,16 @@ type Connection interface {
 	// write the db contents to filename as json (the schema needs defining)
 	ExportRaw(filename string) error
 
-	//Rollback(force bool) error
-	//MigrateData(migratorParams *database.MigratorParameters, force bool) error
-
 	// TODO: this one is very database specific atm
 	BackupTo(w io.Writer) error
-	GetDatabaseFilename() string
+	GetDatabaseFileName() string
+	GetDatabaseFilePath() string
 	GetStorePath() string
 
+	IsEncryptedStore() bool
+	NeedsEncryptionMigration() bool
+	SetEncrypted(encrypted bool)
+
 	SetServiceName(bucketName string) error
 	GetObject(bucketName string, key []byte, object interface{}) error
 	UpdateObject(bucketName string, key []byte, object interface{}) error

api/database/boltdb/db.go

@@ -11,39 +11,78 @@ import (
 
 	"github.com/boltdb/bolt"
 	"github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
-	DatabaseFileName = "portainer.db"
+	DatabaseFileName          = "portainer.db"
+	EncryptedDatabaseFileName = "portainer.edb"
 )
 
 type DbConnection struct {
-	Path string
+	Path          string
+	EncryptionKey []byte
+	isEncrypted   bool
 
 	*bolt.DB
 }
 
-func (connection *DbConnection) GetDatabaseFilename() string {
+// GetDatabaseFileName get the database filename
+func (connection *DbConnection) GetDatabaseFileName() string {
+	if connection.IsEncryptedStore() {
+		return EncryptedDatabaseFileName
+	}
+
 	return DatabaseFileName
 }
 
+// GetDataseFilePath get the path + filename for the database file
+func (connection *DbConnection) GetDatabaseFilePath() string {
+	if connection.IsEncryptedStore() {
+		return path.Join(connection.Path, EncryptedDatabaseFileName)
+	}
+
+	return path.Join(connection.Path, DatabaseFileName)
+}
+
+// GetStorePath get the filename and path for the database file
 func (connection *DbConnection) GetStorePath() string {
 	return connection.Path
 }
 
+func (connection *DbConnection) SetEncrypted(flag bool) {
+	connection.isEncrypted = flag
+}
+
+// Return true if the database is encrypted
+func (connection *DbConnection) IsEncryptedStore() bool {
+	return connection.getEncryptionKey() != nil
+}
+
+// NeedsEncryptionMigration returns true if database encryption is enabled and
+// we have an un-encrypted DB that requires migration to an encrypted DB
+func (connection *DbConnection) NeedsEncryptionMigration() bool {
+	if connection.EncryptionKey != nil {
+		dbFile := path.Join(connection.Path, DatabaseFileName)
+		if _, err := os.Stat(dbFile); err == nil {
+			return true
+		}
+
+		// This is an existing encrypted store or a new store.
+		// A new store will open encrypted from the outset
+		connection.SetEncrypted(true)
+	}
+
+	return false
+}
+
 // Open opens and initializes the BoltDB database.
 func (connection *DbConnection) Open() error {
 
-	// Disabled for now.  Can't use feature flags due to the way that works
+	logrus.Infof("Loading PortainerDB: %s", connection.GetDatabaseFileName())
-	// databaseExportPath := path.Join(connection.Path, fmt.Sprintf("raw-%s-%d.json", DatabaseFileName, time.Now().Unix()))
-	// if err := connection.ExportRaw(databaseExportPath); err != nil {
-	// 	log.Printf("raw export to %s error: %s", databaseExportPath, err)
-	// } else {
-	// 	log.Printf("raw export to %s success", databaseExportPath)
-	// }
-
-	databasePath := path.Join(connection.Path, DatabaseFileName)
 
+	// Now we open the db
+	databasePath := connection.GetDatabaseFilePath()
 	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
 	if err != nil {
 		return err
@@ -71,12 +110,12 @@ func (connection *DbConnection) BackupTo(w io.Writer) error {
 }
 
 func (connection *DbConnection) ExportRaw(filename string) error {
-	databasePath := path.Join(connection.Path, DatabaseFileName)
+	databasePath := connection.GetDatabaseFilePath()
 	if _, err := os.Stat(databasePath); err != nil {
 		return fmt.Errorf("stat on %s failed: %s", databasePath, err)
 	}
 
-	b, err := exportJson(databasePath)
+	b, err := connection.exportJson(databasePath)
 	if err != nil {
 		return err
 	}
@@ -124,7 +163,15 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
 		return err
 	}
 
-	return UnmarshalObject(data, object)
+	return connection.UnmarshalObject(data, object)
+}
+
+func (connection *DbConnection) getEncryptionKey() []byte {
+	if !connection.isEncrypted {
+		return nil
+	}
+
+	return connection.EncryptionKey
 }
 
 // UpdateObject is a generic function used to update an object inside a database database.
@@ -132,7 +179,7 @@ func (connection *DbConnection) UpdateObject(bucketName string, key []byte, obje
 	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
-		data, err := MarshalObject(object)
+		data, err := connection.MarshalObject(object)
 		if err != nil {
 			return err
 		}
@@ -163,7 +210,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
 			var obj interface{}
-			err := UnmarshalObject(v, &obj)
+			err := connection.UnmarshalObject(v, &obj)
 			if err != nil {
 				return err
 			}
@@ -205,7 +252,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
 		seqId, _ := bucket.NextSequence()
 		id, obj := fn(seqId)
 
-		data, err := MarshalObject(obj)
+		data, err := connection.MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -218,8 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
 func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
 	return connection.Update(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
-
+		data, err := connection.MarshalObject(obj)
-		data, err := MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -240,7 +286,7 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
 			return err
 		}
 
-		data, err := MarshalObject(obj)
+		data, err := connection.MarshalObject(obj)
 		if err != nil {
 			return err
 		}
@@ -252,10 +298,9 @@ func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, i
 func (connection *DbConnection) GetAll(bucketName string, obj interface{}, append func(o interface{}) (interface{}, error)) error {
 	err := connection.View(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
-
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := UnmarshalObject(v, obj)
+			err := connection.UnmarshalObject(v, obj)
 			if err != nil {
 				return err
 			}
@@ -277,7 +322,7 @@ func (connection *DbConnection) GetAllWithJsoniter(bucketName string, obj interf
 
 		cursor := bucket.Cursor()
 		for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
-			err := UnmarshalObjectWithJsoniter(v, obj)
+			err := connection.UnmarshalObjectWithJsoniter(v, obj)
 			if err != nil {
 				return err
 			}

api/database/boltdb/export.go

@@ -10,8 +10,9 @@ import (
 
 // inspired by github.com/konoui/boltdb-exporter (which has no license)
 // but very much simplified, based on how we use boltdb
+func (c *DbConnection) exportJson(databasePath string) ([]byte, error) {
+	logrus.WithField("databasePath", databasePath).Infof("exportJson")
 
-func exportJson(databasePath string) ([]byte, error) {
 	connection, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second, ReadOnly: true})
 	if err != nil {
 		return []byte("{}"), err
@@ -31,7 +32,7 @@ func exportJson(databasePath string) ([]byte, error) {
 					continue
 				}
 				var obj interface{}
-				err := UnmarshalObject(v, &obj)
+				err := c.UnmarshalObject(v, &obj)
 				if err != nil {
 					logrus.WithError(err).Errorf("Failed to unmarshal (bucket %s): %v", bucketName, string(v))
 					obj = v

api/database/boltdb/json.go

@@ -1,42 +1,123 @@
 package boltdb
 
 import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
 	"encoding/json"
+	"fmt"
+	"io"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/pkg/errors"
 )
 
+var errEncryptedStringTooShort = fmt.Errorf("encrypted string too short")
+
 // MarshalObject encodes an object to binary format
-func MarshalObject(object interface{}) ([]byte, error) {
+func (connection *DbConnection) MarshalObject(object interface{}) (data []byte, err error) {
 	// Special case for the VERSION bucket. Here we're not using json
 	if v, ok := object.(string); ok {
-		return []byte(v), nil
+		data = []byte(v)
+	} else {
+		data, err = json.Marshal(object)
+		if err != nil {
+			return data, err
+		}
 	}
-
+	if connection.getEncryptionKey() == nil {
-	return json.Marshal(object)
+		return data, nil
+	}
+	return encrypt(data, connection.getEncryptionKey())
 }
 
 // UnmarshalObject decodes an object from binary data
-func UnmarshalObject(data []byte, object interface{}) error {
+func (connection *DbConnection) UnmarshalObject(data []byte, object interface{}) error {
-	// Special case for the VERSION bucket. Here we're not using json
+	var err error
-	// So we need to return it as a string
+	if connection.getEncryptionKey() != nil {
-	err := json.Unmarshal(data, object)
+		data, err = decrypt(data, connection.getEncryptionKey())
-	if err != nil {
+		if err != nil {
-		if s, ok := object.(*string); ok {
+			errors.Wrapf(err, "Failed decrypting object")
-			*s = string(data)
+		}
-			return nil
+	}
+	e := json.Unmarshal(data, object)
+	if e != nil {
+		// Special case for the VERSION bucket. Here we're not using json
+		// So we need to return it as a string
+		s, ok := object.(*string)
+		if !ok {
+			return errors.Wrap(err, e.Error())
 		}
 
-		return err
+		*s = string(data)
 	}
-
+	return err
-	return nil
 }
 
 // UnmarshalObjectWithJsoniter decodes an object from binary data
 // using the jsoniter library. It is mainly used to accelerate environment(endpoint)
 // decoding at the moment.
-func UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object interface{}) error {
+	if connection.getEncryptionKey() != nil {
+		var err error
+		data, err = decrypt(data, connection.getEncryptionKey())
+		if err != nil {
+			return err
+		}
+	}
 	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
 	return jsoni.Unmarshal(data, &object)
 }
+
+// mmm, don't have a KMS .... aes GCM seems the most likely from
+// https://gist.github.com/atoponce/07d8d4c833873be2f68c34f9afc5a78a#symmetric-encryption
+
+func encrypt(plaintext []byte, passphrase []byte) (encrypted []byte, err error) {
+	block, _ := aes.NewCipher(passphrase)
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, err
+	}
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return encrypted, err
+	}
+	ciphertextByte := gcm.Seal(
+		nonce,
+		nonce,
+		plaintext,
+		nil)
+	return ciphertextByte, nil
+}
+
+func decrypt(encrypted []byte, passphrase []byte) (plaintextByte []byte, err error) {
+	if string(encrypted) == "false" {
+		return []byte("false"), nil
+	}
+	block, err := aes.NewCipher(passphrase)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating cypher block")
+	}
+
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error creating GCM")
+	}
+
+	nonceSize := gcm.NonceSize()
+	if len(encrypted) < nonceSize {
+		return encrypted, errEncryptedStringTooShort
+	}
+
+	nonce, ciphertextByteClean := encrypted[:nonceSize], encrypted[nonceSize:]
+	plaintextByte, err = gcm.Open(
+		nil,
+		nonce,
+		ciphertextByteClean,
+		nil)
+	if err != nil {
+		return encrypted, errors.Wrap(err, "Error decrypting text")
+	}
+
+	return plaintextByte, err
+}

api/database/boltdb/json_test.go

@@ -1,6 +1,7 @@
 package boltdb
 
 import (
+	"crypto/sha256"
 	"fmt"
 	"testing"
 
@@ -8,9 +9,17 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-const jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
+const (
+	jsonobject = `{"LogoURL":"","BlackListedLabels":[],"AuthenticationMethod":1,"LDAPSettings":{"AnonymousMode":true,"ReaderDN":"","URL":"","TLSConfig":{"TLS":false,"TLSSkipVerify":false},"StartTLS":false,"SearchSettings":[{"BaseDN":"","Filter":"","UserNameAttribute":""}],"GroupSearchSettings":[{"GroupBaseDN":"","GroupFilter":"","GroupAttribute":""}],"AutoCreateUsers":true},"OAuthSettings":{"ClientID":"","AccessTokenURI":"","AuthorizationURI":"","ResourceURI":"","RedirectURI":"","UserIdentifier":"","Scopes":"","OAuthAutoCreateUsers":false,"DefaultTeamID":0,"SSO":true,"LogoutURI":"","KubeSecretKey":"j0zLVtY/lAWBk62ByyF0uP80SOXaitsABP0TTJX8MhI="},"OpenAMTConfiguration":{"Enabled":false,"MPSServer":"","Credentials":{"MPSUser":"","MPSPassword":"","MPSToken":""},"DomainConfiguration":{"CertFileText":"","CertPassword":"","DomainName":""},"WirelessConfiguration":null},"FeatureFlagSettings":{},"SnapshotInterval":"5m","TemplatesURL":"https://raw.githubusercontent.com/portainer/templates/master/templates-2.0.json","EdgeAgentCheckinInterval":5,"EnableEdgeComputeFeatures":false,"UserSessionTimeout":"8h","KubeconfigExpiry":"0","EnableTelemetry":true,"HelmRepositoryURL":"https://charts.bitnami.com/bitnami","KubectlShellImage":"portainer/kubectl-shell","DisplayDonationHeader":false,"DisplayExternalContributors":false,"EnableHostManagementFeatures":false,"AllowVolumeBrowserForRegularUsers":false,"AllowBindMountsForRegularUsers":false,"AllowPrivilegedModeForRegularUsers":false,"AllowHostNamespaceForRegularUsers":false,"AllowStackManagementForRegularUsers":false,"AllowDeviceMappingForRegularUsers":false,"AllowContainerCapabilitiesForRegularUsers":false}`
+	passphrase = "my secret key"
+)
 
-func Test_MarshalObject(t *testing.T) {
+func secretToEncryptionKey(passphrase string) []byte {
+	hash := sha256.Sum256([]byte(passphrase))
+	return hash[:]
+}
+
+func Test_MarshalObjectUnencrypted(t *testing.T) {
 	is := assert.New(t)
 
 	uuid := uuid.Must(uuid.NewV4())
@@ -73,16 +82,18 @@ func Test_MarshalObject(t *testing.T) {
 		},
 	}
 
+	conn := DbConnection{}
+
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
-			data, err := MarshalObject(test.object)
+			data, err := conn.MarshalObject(test.object)
 			is.NoError(err)
 			is.Equal(test.expected, string(data))
 		})
 	}
 }
 
-func Test_UnMarshalObject(t *testing.T) {
+func Test_UnMarshalObjectUnencrypted(t *testing.T) {
 	is := assert.New(t)
 
 	// Based on actual data entering and what we expect out of the function
@@ -105,18 +116,62 @@ func Test_UnMarshalObject(t *testing.T) {
 			expected: "9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6",
 		},
 		{
-			// An unmarshalled json object string should return the same as a string without error also
+			// An un-marshalled json object string should return the same as a string without error also
 			object:   []byte(jsonobject),
 			expected: jsonobject,
 		},
 	}
 
+	conn := DbConnection{}
+
 	for _, test := range tests {
 		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
 			var object string
-			err := UnmarshalObject(test.object, &object)
+			err := conn.UnmarshalObject(test.object, &object)
 			is.NoError(err)
 			is.Equal(test.expected, string(object))
 		})
 	}
 }
+
+func Test_ObjectMarshallingEncrypted(t *testing.T) {
+	is := assert.New(t)
+
+	// Based on actual data entering and what we expect out of the function
+
+	tests := []struct {
+		object   []byte
+		expected string
+	}{
+		{
+			object: []byte(""),
+		},
+		{
+			object: []byte("35"),
+		},
+		{
+			// An unmarshalled byte string should return the same without error
+			object: []byte("9ca4a1dd-a439-4593-b386-a7dfdc2e9fc6"),
+		},
+		{
+			// An un-marshalled json object string should return the same as a string without error also
+			object: []byte(jsonobject),
+		},
+	}
+
+	key := secretToEncryptionKey(passphrase)
+	conn := DbConnection{EncryptionKey: key}
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("%s -> %s", test.object, test.expected), func(t *testing.T) {
+
+			data, err := conn.MarshalObject(test.object)
+			is.NoError(err)
+
+			var object []byte
+			err = conn.UnmarshalObject(data, &object)
+
+			is.NoError(err)
+			is.Equal(test.object, object)
+		})
+	}
+}

api/database/database.go

@@ -2,15 +2,19 @@ package database
 
 import (
 	"fmt"
+
 	portainer "github.com/portainer/portainer/api"
 	"github.com/portainer/portainer/api/database/boltdb"
 )
 
 // NewDatabase should use config options to return a connection to the requested database
-func NewDatabase(storeType, storePath string) (connection portainer.Connection, err error) {
+func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) {
 	switch storeType {
 	case "boltdb":
-		return &boltdb.DbConnection{Path: storePath}, nil
+		return &boltdb.DbConnection{
+			Path:          storePath,
+			EncryptionKey: encryptionKey,
+		}, nil
 	}
-	return nil, fmt.Errorf("Unknown storage database: %s", storeType)
+	return nil, fmt.Errorf("unknown storage database: %s", storeType)
 }

api/dataservices/endpoint/endpoint.go

@@ -69,7 +69,7 @@ func (service *Service) Endpoints() ([]portainer.Endpoint, error) {
 			endpoint, ok := obj.(*portainer.Endpoint)
 			if !ok {
 				logrus.WithField("obj", obj).Errorf("Failed to convert to Endpoint object")
-				return nil, fmt.Errorf("Failed to convert to Endpoint object: %s", obj)
+				return nil, fmt.Errorf("failed to convert to Endpoint object: %s", obj)
 			}
 			endpoints = append(endpoints, *endpoint)
 			return &portainer.Endpoint{}, nil

api/dataservices/errors/errors.go

@@ -4,6 +4,7 @@ import "errors"
 
 var (
 	// TODO: i'm pretty sure this needs wrapping at several levels
-	ErrObjectNotFound = errors.New("Object not found inside the database")
+	ErrObjectNotFound = errors.New("object not found inside the database")
-	ErrWrongDBEdition = errors.New("The Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrWrongDBEdition = errors.New("the Portainer database is set for Portainer Business Edition, please follow the instructions in our documentation to downgrade it: https://documentation.portainer.io/v2.0-be/downgrade/be-to-ce/")
+	ErrDBImportFailed = errors.New("importing backup failed")
 )

api/dataservices/helmuserrepository/helmuserrepository.go

@@ -34,7 +34,7 @@ func NewService(connection portainer.Connection) (*Service, error) {
 }
 
 //HelmUserRepository returns an array of all HelmUserRepository
-func (service *Service) HelmUserRepositorys() ([]portainer.HelmUserRepository, error) {
+func (service *Service) HelmUserRepositories() ([]portainer.HelmUserRepository, error) {
 	var repos = make([]portainer.HelmUserRepository, 0)
 
 	err := service.connection.GetAll(

api/dataservices/interface.go

@@ -124,7 +124,7 @@ type (
 
 	// HelmUserRepositoryService represents a service to manage HelmUserRepositories
 	HelmUserRepositoryService interface {
-		HelmUserRepositorys() ([]portainer.HelmUserRepository, error)
+		HelmUserRepositories() ([]portainer.HelmUserRepository, error)
 		HelmUserRepositoryByUserID(userID portainer.UserID) ([]portainer.HelmUserRepository, error)
 		Create(record *portainer.HelmUserRepository) error
 		UpdateHelmUserRepository(ID portainer.HelmUserRepositoryID, repository *portainer.HelmUserRepository) error

api/datastore/backup.go

@@ -35,7 +35,7 @@ func (store *Store) createBackupFolders() {
 }
 
 func (store *Store) databasePath() string {
-	return path.Join(store.connection.GetStorePath(), store.connection.GetDatabaseFilename())
+	return store.connection.GetDatabaseFilePath()
 }
 
 func (store *Store) commonBackupDir() string {
@@ -84,7 +84,7 @@ func (store *Store) setupOptions(options *BackupOptions) *BackupOptions {
 		options.BackupDir = store.commonBackupDir()
 	}
 	if options.BackupFileName == "" {
-		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFilename(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
+		options.BackupFileName = fmt.Sprintf("%s.%s.%s", store.connection.GetDatabaseFileName(), fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405"))
 	}
 	if options.BackupPath == "" {
 		options.BackupPath = path.Join(options.BackupDir, options.BackupFileName)

api/datastore/backup_test.go

@@ -48,7 +48,7 @@ func TestBackup(t *testing.T) {
 		store.VersionService.StoreDBVersion(portainer.DBVersion)
 		store.backupWithOptions(nil)
 
-		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion))
+		backupFileName := path.Join(connection.GetStorePath(), "backups", "common", fmt.Sprintf("portainer.edb.%03d.*", portainer.DBVersion))
 		if !isFileExist(backupFileName) {
 			t.Errorf("Expect backup file to be created %s", backupFileName)
 		}

api/datastore/datastore.go

@@ -1,10 +1,15 @@
 package datastore
 
 import (
+	"fmt"
 	"io"
+	"os"
+	"path"
+	"time"
 
 	portainer "github.com/portainer/portainer/api"
-	"github.com/portainer/portainer/api/dataservices/errors"
+	portainerErrors "github.com/portainer/portainer/api/dataservices/errors"
+	"github.com/sirupsen/logrus"
 )
 
 func (store *Store) version() (int, error) {
@@ -34,6 +39,12 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
 // Open opens and initializes the BoltDB database.
 func (store *Store) Open() (newStore bool, err error) {
 	newStore = true
+
+	encryptionReq := store.connection.NeedsEncryptionMigration()
+	if encryptionReq {
+		store.encryptDB()
+	}
+
 	err = store.connection.Open()
 	if err != nil {
 		return newStore, err
@@ -45,8 +56,18 @@ func (store *Store) Open() (newStore bool, err error) {
 	}
 
 	// if we have DBVersion in the database then ensure we flag this as NOT a new store
-	if _, err := store.VersionService.DBVersion(); err == nil {
+	version, err := store.VersionService.DBVersion()
-		newStore = false
+	if err != nil {
+		if store.IsErrObjectNotFound(err) {
+			return newStore, nil
+		}
+
+		return newStore, err
+	}
+
+	if version > 0 {
+		logrus.WithField("version", version).Infof("Opened existing store")
+		return false, nil
 	}
 
 	return newStore, nil
@@ -65,16 +86,81 @@ func (store *Store) BackupTo(w io.Writer) error {
 // CheckCurrentEdition checks if current edition is community edition
 func (store *Store) CheckCurrentEdition() error {
 	if store.edition() != portainer.PortainerCE {
-		return errors.ErrWrongDBEdition
+		return portainerErrors.ErrWrongDBEdition
 	}
 	return nil
 }
 
 // TODO: move the use of this to dataservices.IsErrObjectNotFound()?
 func (store *Store) IsErrObjectNotFound(e error) bool {
-	return e == errors.ErrObjectNotFound
+	return e == portainerErrors.ErrObjectNotFound
 }
 
 func (store *Store) Rollback(force bool) error {
 	return store.connectionRollback(force)
 }
+
+func (store *Store) encryptDB() error {
+	store.connection.SetEncrypted(false)
+	err := store.connection.Open()
+	if err != nil {
+		return err
+	}
+
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	// The DB is not currently encrypted.  First save the encrypted db filename
+	oldFilename := store.connection.GetDatabaseFilePath()
+	logrus.Infof("Encrypting database")
+
+	// export file path for backup
+	exportFilename := path.Join(store.databasePath() + "." + fmt.Sprintf("backup-%d.json", time.Now().Unix()))
+
+	logrus.Infof("Exporting database backup to %s", exportFilename)
+	err = store.Export(exportFilename)
+	if err != nil {
+		logrus.WithError(err).Debugf("Failed to export to %s", exportFilename)
+		return err
+	}
+
+	logrus.Infof("Database backup exported")
+
+	// Close existing un-encrypted db so that we can delete the file later
+	store.connection.Close()
+
+	// Tell the db layer to create an encrypted db when opened
+	store.connection.SetEncrypted(true)
+	store.connection.Open()
+
+	// We have to init services before import
+	err = store.initServices()
+	if err != nil {
+		return err
+	}
+
+	err = store.Import(exportFilename)
+	if err != nil {
+		// Remove the new encrypted file that we failed to import
+		os.Remove(store.connection.GetDatabaseFilePath())
+		logrus.Fatal(portainerErrors.ErrDBImportFailed.Error())
+	}
+
+	err = os.Remove(oldFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the un-encrypted db file")
+	}
+
+	err = os.Remove(exportFilename)
+	if err != nil {
+		logrus.Errorf("Failed to remove the json backup file")
+	}
+
+	// Close db connection
+	store.connection.Close()
+
+	logrus.Info("Database successfully encrypted")
+	return nil
+}

api/datastore/services.go

@@ -363,118 +363,184 @@ func (store *Store) Export(filename string) (err error) {
 	backup := storeExport{}
 
 	if c, err := store.CustomTemplate().CustomTemplates(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Custom Templates")
+		}
 	} else {
 		backup.CustomTemplate = c
 	}
+
 	if e, err := store.EdgeGroup().EdgeGroups(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Groups")
+		}
 	} else {
 		backup.EdgeGroup = e
 	}
+
 	if e, err := store.EdgeJob().EdgeJobs(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Jobs")
+		}
 	} else {
 		backup.EdgeJob = e
 	}
+
 	if e, err := store.EdgeStack().EdgeStacks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Edge Stacks")
+		}
 	} else {
 		backup.EdgeStack = e
 	}
+
 	if e, err := store.Endpoint().Endpoints(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoints")
+		}
 	} else {
 		backup.Endpoint = e
 	}
+
 	if e, err := store.EndpointGroup().EndpointGroups(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoint Groups")
+		}
 	} else {
 		backup.EndpointGroup = e
 	}
+
 	if r, err := store.EndpointRelation().EndpointRelations(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Endpoint Relations")
+		}
 	} else {
 		backup.EndpointRelation = r
 	}
+
 	if r, err := store.ExtensionService.Extensions(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Extensions")
+		}
 	} else {
 		backup.Extensions = r
 	}
-	if r, err := store.HelmUserRepository().HelmUserRepositorys(); err != nil {
+
-		logrus.WithError(err).Debugf("Export boom")
+	if r, err := store.HelmUserRepository().HelmUserRepositories(); err != nil {
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Helm User Repositories")
+		}
 	} else {
 		backup.HelmUserRepository = r
 	}
+
 	if r, err := store.Registry().Registries(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Registries")
+		}
 	} else {
 		backup.Registry = r
 	}
+
 	if c, err := store.ResourceControl().ResourceControls(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Resource Controls")
+		}
 	} else {
 		backup.ResourceControl = c
 	}
+
 	if role, err := store.Role().Roles(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Roles")
+		}
 	} else {
 		backup.Role = role
 	}
+
 	if r, err := store.ScheduleService.Schedules(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Schedules")
+		}
 	} else {
 		backup.Schedules = r
 	}
+
 	if settings, err := store.Settings().Settings(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Settings")
+		}
 	} else {
 		backup.Settings = *settings
 	}
+
 	if settings, err := store.SSLSettings().Settings(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting SSL Settings")
+		}
 	} else {
 		backup.SSLSettings = *settings
 	}
+
 	if t, err := store.Stack().Stacks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Stacks")
+		}
 	} else {
 		backup.Stack = t
 	}
+
 	if t, err := store.Tag().Tags(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Tags")
+		}
 	} else {
 		backup.Tag = t
 	}
+
 	if t, err := store.TeamMembership().TeamMemberships(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Team Memberships")
+		}
 	} else {
 		backup.TeamMembership = t
 	}
+
 	if t, err := store.Team().Teams(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Teams")
+		}
 	} else {
 		backup.Team = t
 	}
+
 	if info, err := store.TunnelServer().Info(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Tunnel Server")
+		}
 	} else {
 		backup.TunnelServer = *info
 	}
+
 	if users, err := store.User().Users(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Users")
+		}
 	} else {
 		backup.User = users
 	}
+
 	if webhooks, err := store.Webhook().Webhooks(); err != nil {
-		logrus.WithError(err).Debugf("Export boom")
+		if !store.IsErrObjectNotFound(err) {
+			logrus.WithError(err).Errorf("Exporting Webhooks")
+		}
 	} else {
 		backup.Webhook = webhooks
 	}
+
 	v, err := store.Version().DBVersion()
-	if err != nil {
+	if err != nil && !store.IsErrObjectNotFound(err) {
-		logrus.WithError(err).Debugf("Export boom")
+		logrus.WithError(err).Errorf("Exporting DB version")
 	}
 	instance, _ := store.Version().InstanceID()
 	backup.Version = map[string]string{
@@ -518,50 +584,66 @@ func (store *Store) Import(filename string) (err error) {
 	for _, v := range backup.CustomTemplate {
 		store.CustomTemplate().UpdateCustomTemplate(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeGroup {
 		store.EdgeGroup().UpdateEdgeGroup(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeJob {
 		store.EdgeJob().UpdateEdgeJob(v.ID, &v)
 	}
+
 	for _, v := range backup.EdgeStack {
 		store.EdgeStack().UpdateEdgeStack(v.ID, &v)
 	}
+
 	for _, v := range backup.Endpoint {
 		store.Endpoint().UpdateEndpoint(v.ID, &v)
 	}
+
 	for _, v := range backup.EndpointGroup {
 		store.EndpointGroup().UpdateEndpointGroup(v.ID, &v)
 	}
+
 	for _, v := range backup.EndpointRelation {
 		store.EndpointRelation().UpdateEndpointRelation(v.EndpointID, &v)
 	}
+
 	for _, v := range backup.HelmUserRepository {
 		store.HelmUserRepository().UpdateHelmUserRepository(v.ID, &v)
 	}
+
 	for _, v := range backup.Registry {
 		store.Registry().UpdateRegistry(v.ID, &v)
 	}
+
 	for _, v := range backup.ResourceControl {
 		store.ResourceControl().UpdateResourceControl(v.ID, &v)
 	}
+
 	for _, v := range backup.Role {
 		store.Role().UpdateRole(v.ID, &v)
 	}
+
 	store.Settings().UpdateSettings(&backup.Settings)
 	store.SSLSettings().UpdateSettings(&backup.SSLSettings)
+
 	for _, v := range backup.Stack {
 		store.Stack().UpdateStack(v.ID, &v)
 	}
+
 	for _, v := range backup.Tag {
 		store.Tag().UpdateTag(v.ID, &v)
 	}
+
 	for _, v := range backup.TeamMembership {
 		store.TeamMembership().UpdateTeamMembership(v.ID, &v)
 	}
+
 	for _, v := range backup.Team {
 		store.Team().UpdateTeam(v.ID, &v)
 	}
+
 	store.TunnelServer().UpdateInfo(&backup.TunnelServer)
 
 	for _, user := range backup.User {
@@ -570,10 +652,9 @@ func (store *Store) Import(filename string) (err error) {
 		}
 	}
 
-	// backup[store.Webhook().BucketName()], err = store.Webhook().Webhooks()
+	for _, v := range backup.Webhook {
-	// if err != nil {
+		store.Webhook().UpdateWebhook(v.ID, &v)
-	// 	logrus.WithError(err).Debugf("Export boom")
+	}
-	// }
 
 	return nil
 }

api/datastore/teststore.go

@@ -42,7 +42,7 @@ func NewTestStore(init bool) (bool, *Store, func(), error) {
 		return false, nil, nil, err
 	}
 
-	connection, err := database.NewDatabase("boltdb", storePath)
+	connection, err := database.NewDatabase("boltdb", storePath, []byte("apassphrasewhichneedstobe32bytes"))
 	if err != nil {
 		panic(err)
 	}

api/portainer.go

@@ -97,6 +97,7 @@ type (
 		Rollback                  *bool
 		SnapshotInterval          *string
 		BaseURL                   *string
+		SecretKeyName             *string
 	}
 
 	// CustomTemplate represents a custom template