github
/
portainer
mirror of https://github.com/portainer/portainer
1
0
Fork 0
Code Issues Releases Wiki Activity

Fix(db): needs encryption migration function fixed EE-2414 (#6494)

Browse Source 
* fix(db) NeedsEncryptionMigration EE-2414
* fix for case where we started encrypted and restore unencrypted.  We don't want to have two databases
* fix(db): handle decryption error EE-2466

Co-authored-by: Matt Hook <hookenz@gmail.com>
Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>
pull/6514/head
Prabhat Khera 3 years ago committed by GitHub
parent ad7f87122d
commit a8d3cda3fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 250 additions and 60 deletions
Show Stats Download Patch File Download Diff File

18
api/backup/restore.go
Unescape View File

@ -10,6 +10,7 @@ import (
"github.com/pkg/errors"
"github.com/portainer/portainer/api/archive"
"github.com/portainer/portainer/api/crypto"
"github.com/portainer/portainer/api/database/boltdb"
"github.com/portainer/portainer/api/dataservices"
"github.com/portainer/portainer/api/filesystem"
"github.com/portainer/portainer/api/http/offlinegate"
@ -65,5 +66,20 @@ func restoreFiles(srcDir string, destinationDir string) error {
return err
}
}
return nil
// TODO: This is very boltdb module specific once again due to the filename. Move to bolt module? Refactor for another day
// Prevent the possibility of having both databases. Remove any default new instance
os.Remove(filepath.Join(destinationDir, boltdb.DatabaseFileName))
os.Remove(filepath.Join(destinationDir, boltdb.EncryptedDatabaseFileName))
// Now copy the database. It'll be either portainer.db or portainer.edb
// Note: CopyPath does not return an error if the source file doesn't exist
err := filesystem.CopyPath(filepath.Join(srcDir, boltdb.EncryptedDatabaseFileName), destinationDir)
if err != nil {
return err
}
return filesystem.CopyPath(filepath.Join(srcDir, boltdb.DatabaseFileName), destinationDir)
}

93
api/cmd/portainer/main.go
Unescape View File

@ -49,12 +49,12 @@ func initCLI() *portainer.CLIFlags {
var cliService portainer.CLIService = &cli.Service{}
flags, err := cliService.ParseFlags(portainer.APIVersion)
if err != nil {
log.Fatalf("Failed parsing flags: %v", err)
logrus.Fatalf("Failed parsing flags: %v", err)
}
err = cliService.ValidateFlags(flags)
if err != nil {
log.Fatalf("Failed validating flags:%v", err)
logrus.Fatalf("Failed validating flags:%v", err)
}
return flags
}
@ -62,7 +62,7 @@ func initCLI() *portainer.CLIFlags {
func initFileService(dataStorePath string) portainer.FileService {
fileService, err := filesystem.NewService(dataStorePath, "")
if err != nil {
log.Fatalf("Failed creating file service: %v", err)
logrus.Fatalf("Failed creating file service: %v", err)
}
return fileService
}
@ -70,7 +70,7 @@ func initFileService(dataStorePath string) portainer.FileService {
func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
if err != nil {
log.Fatalf("failed creating database connection: %s", err)
logrus.Fatalf("failed creating database connection: %s", err)
}
if bconn, ok := connection.(*boltdb.DbConnection); ok {
@ -78,22 +78,22 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
bconn.MaxBatchDelay = *flags.MaxBatchDelay
bconn.InitialMmapSize = *flags.InitialMmapSize
} else {
log.Fatalf("failed creating database connection: expecting a boltdb database type but a different one was received")
logrus.Fatalf("failed creating database connection: expecting a boltdb database type but a different one was received")
}
store := datastore.NewStore(*flags.Data, fileService, connection)
isNew, err := store.Open()
if err != nil {
log.Fatalf("Failed opening store: %v", err)
logrus.Fatalf("Failed opening store: %v", err)
}
if *flags.Rollback {
err := store.Rollback(false)
if err != nil {
log.Fatalf("Failed rolling back: %v", err)
logrus.Fatalf("Failed rolling back: %v", err)
}
log.Println("Exiting rollback")
logrus.Println("Exiting rollback")
os.Exit(0)
return nil
}
@ -101,21 +101,26 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
// Init sets some defaults - it's basically a migration
err = store.Init()
if err != nil {
log.Fatalf("Failed initializing data store: %v", err)
logrus.Fatalf("Failed initializing data store: %v", err)
}
if isNew {
// from MigrateData
store.VersionService.StoreDBVersion(portainer.DBVersion)
err := updateSettingsFromFlags(store, flags)
if err != nil {
logrus.Fatalf("Failed updating settings from flags: %v", err)
}
} else {
storedVersion, err := store.VersionService.DBVersion()
if err != nil {
log.Fatalf("Something Failed during creation of new database: %v", err)
logrus.Fatalf("Something Failed during creation of new database: %v", err)
}
if storedVersion != portainer.DBVersion {
err = store.MigrateData()
if err != nil {
log.Fatalf("Failed migration: %v", err)
logrus.Fatalf("Failed migration: %v", err)
}
}
}
@ -146,7 +151,7 @@ func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService port
func initComposeStackManager(assetsPath string, configPath string, reverseTunnelService portainer.ReverseTunnelService, proxyManager *proxy.Manager) portainer.ComposeStackManager {
composeWrapper, err := exec.NewComposeStackManager(assetsPath, configPath, proxyManager)
if err != nil {
log.Fatalf("Failed creating compose manager: %v", err)
logrus.Fatalf("Failed creating compose manager: %v", err)
}
return composeWrapper
@ -329,9 +334,9 @@ func enableFeaturesFromFlags(dataStore dataservices.DataStore, flags *portainer.
}
if featureState {
log.Printf("Feature %v : on", *correspondingFeature)
logrus.Printf("Feature %v : on", *correspondingFeature)
} else {
log.Printf("Feature %v : off", *correspondingFeature)
logrus.Printf("Feature %v : off", *correspondingFeature)
}
settings.FeatureFlagSettings[*correspondingFeature] = featureState
@ -360,7 +365,7 @@ func generateAndStoreKeyPair(fileService portainer.FileService, signatureService
func initKeyPair(fileService portainer.FileService, signatureService portainer.DigitalSignatureService) error {
existingKeyPair, err := fileService.KeyPairFilesExist()
if err != nil {
log.Fatalf("Failed checking for existing key pair: %v", err)
logrus.Fatalf("Failed checking for existing key pair: %v", err)
}
if existingKeyPair {
@ -431,7 +436,7 @@ func createTLSSecuredEndpoint(flags *portainer.CLIFlags, dataStore dataservices.
err := snapshotService.SnapshotEndpoint(endpoint)
if err != nil {
log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
logrus.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
}
return dataStore.Endpoint().Create(endpoint)
@ -477,7 +482,7 @@ func createUnsecuredEndpoint(endpointURL string, dataStore dataservices.DataStor
err := snapshotService.SnapshotEndpoint(endpoint)
if err != nil {
log.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
logrus.Printf("http error: environment snapshot error (environment=%s, URL=%s) (err=%s)\n", endpoint.Name, endpoint.URL, err)
}
return dataStore.Endpoint().Create(endpoint)
@ -494,7 +499,7 @@ func initEndpoint(flags *portainer.CLIFlags, dataStore dataservices.DataStore, s
}
if len(endpoints) > 0 {
log.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
logrus.Println("Instance already has defined environments. Skipping the environment defined via CLI.")
return nil
}
@ -508,9 +513,9 @@ func loadEncryptionSecretKey(keyfilename string) []byte {
content, err := os.ReadFile(path.Join("/run/secrets", keyfilename))
if err != nil {
if os.IsNotExist(err) {
log.Printf("Encryption key file `%s` not present", keyfilename)
logrus.Printf("Encryption key file `%s` not present", keyfilename)
} else {
log.Printf("Error reading encryption key file: %v", err)
logrus.Printf("Error reading encryption key file: %v", err)
}
return nil
@ -527,33 +532,33 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
fileService := initFileService(*flags.Data)
encryptionKey := loadEncryptionSecretKey(*flags.SecretKeyName)
if encryptionKey == nil {
log.Println("proceeding without encryption key")
logrus.Println("Proceeding without encryption key")
}
dataStore := initDataStore(flags, encryptionKey, fileService, shutdownCtx)
if err := dataStore.CheckCurrentEdition(); err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
instanceID, err := dataStore.Version().InstanceID()
if err != nil {
log.Fatalf("Failed getting instance id: %v", err)
logrus.Fatalf("Failed getting instance id: %v", err)
}
apiKeyService := initAPIKeyService(dataStore)
settings, err := dataStore.Settings().Settings()
if err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
jwtService, err := initJWTService(settings.UserSessionTimeout, dataStore)
if err != nil {
log.Fatalf("Failed initializing JWT service: %v", err)
logrus.Fatalf("Failed initializing JWT service: %v", err)
}
err = enableFeaturesFromFlags(dataStore, flags)
if err != nil {
log.Fatalf("Failed enabling feature flag: %v", err)
logrus.Fatalf("Failed enabling feature flag: %v", err)
}
ldapService := initLDAPService()
@ -567,17 +572,17 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
sslService, err := initSSLService(*flags.AddrHTTPS, *flags.Data, *flags.SSLCert, *flags.SSLKey, fileService, dataStore, shutdownTrigger)
if err != nil {
log.Fatal(err)
logrus.Fatal(err)
}
sslSettings, err := sslService.GetSSLSettings()
if err != nil {
log.Fatalf("Failed to get ssl settings: %s", err)
logrus.Fatalf("Failed to get ssl settings: %s", err)
}
err = initKeyPair(fileService, digitalSignatureService)
if err != nil {
log.Fatalf("Failed initializing key pair: %v", err)
logrus.Fatalf("Failed initializing key pair: %v", err)
}
reverseTunnelService := chisel.NewService(dataStore, shutdownCtx)
@ -587,7 +592,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
snapshotService, err := initSnapshotService(*flags.SnapshotInterval, dataStore, dockerClientFactory, kubernetesClientFactory, shutdownCtx)
if err != nil {
log.Fatalf("Failed initializing snapshot service: %v", err)
logrus.Fatalf("Failed initializing snapshot service: %v", err)
}
snapshotService.Start()
@ -608,37 +613,37 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
swarmStackManager, err := initSwarmStackManager(*flags.Assets, dockerConfigPath, digitalSignatureService, fileService, reverseTunnelService, dataStore)
if err != nil {
log.Fatalf("Failed initializing swarm stack manager: %v", err)
logrus.Fatalf("Failed initializing swarm stack manager: %v", err)
}
kubernetesDeployer := initKubernetesDeployer(kubernetesTokenCacheManager, kubernetesClientFactory, dataStore, reverseTunnelService, digitalSignatureService, proxyManager, *flags.Assets)
helmPackageManager, err := initHelmPackageManager(*flags.Assets)
if err != nil {
log.Fatalf("Failed initializing helm package manager: %v", err)
logrus.Fatalf("Failed initializing helm package manager: %v", err)
}
err = edge.LoadEdgeJobs(dataStore, reverseTunnelService)
if err != nil {
log.Fatalf("Failed loading edge jobs from database: %v", err)
logrus.Fatalf("Failed loading edge jobs from database: %v", err)
}
applicationStatus := initStatus(instanceID)
err = initEndpoint(flags, dataStore, snapshotService)
if err != nil {
log.Fatalf("Failed initializing environment: %v", err)
logrus.Fatalf("Failed initializing environment: %v", err)
}
adminPasswordHash := ""
if *flags.AdminPasswordFile != "" {
content, err := fileService.GetFileContent(*flags.AdminPasswordFile, "")
if err != nil {
log.Fatalf("Failed getting admin password file: %v", err)
logrus.Fatalf("Failed getting admin password file: %v", err)
}
adminPasswordHash, err = cryptoService.Hash(strings.TrimSuffix(string(content), "\n"))
if err != nil {
log.Fatalf("Failed hashing admin password: %v", err)
logrus.Fatalf("Failed hashing admin password: %v", err)
}
} else if *flags.AdminPassword != "" {
adminPasswordHash = *flags.AdminPassword
@ -647,11 +652,11 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
if adminPasswordHash != "" {
users, err := dataStore.User().UsersByRole(portainer.AdministratorRole)
if err != nil {
log.Fatalf("Failed getting admin user: %v", err)
logrus.Fatalf("Failed getting admin user: %v", err)
}
if len(users) == 0 {
log.Println("Created admin user with the given password.")
logrus.Println("Created admin user with the given password.")
user := &portainer.User{
Username: "admin",
Role: portainer.AdministratorRole,
@ -659,21 +664,21 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server {
}
err := dataStore.User().Create(user)
if err != nil {
log.Fatalf("Failed creating admin user: %v", err)
logrus.Fatalf("Failed creating admin user: %v", err)
}
} else {
log.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
logrus.Println("Instance already has an administrator user defined. Skipping admin password related flags.")
}
}
err = reverseTunnelService.StartTunnelServer(*flags.TunnelAddr, *flags.TunnelPort, snapshotService)
if err != nil {
log.Fatalf("Failed starting tunnel server: %v", err)
logrus.Fatalf("Failed starting tunnel server: %v", err)
}
sslDBSettings, err := dataStore.SSLSettings().Settings()
if err != nil {
log.Fatalf("Failed to fetch ssl settings from DB")
logrus.Fatalf("Failed to fetch ssl settings from DB")
}
scheduler := scheduler.NewScheduler(shutdownCtx)
@ -724,8 +729,8 @@ func main() {
for {
server := buildServer(flags)
log.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
logrus.Printf("[INFO] [cmd,main] Starting Portainer version %s\n", portainer.APIVersion)
err := server.Start()
log.Printf("[INFO] [cmd,main] Http server exited: %v\n", err)
logrus.Printf("[INFO] [cmd,main] Http server exited: %v\n", err)
}
}

2
api/connection.go
Unescape View File

@ -18,7 +18,7 @@ type Connection interface {
GetStorePath() string
IsEncryptedStore() bool
NeedsEncryptionMigration() bool
NeedsEncryptionMigration() (bool, error)
SetEncrypted(encrypted bool)
SetServiceName(bucketName string) error

60
api/database/boltdb/db.go
Unescape View File

@ -2,6 +2,7 @@ package boltdb
import (
"encoding/binary"
"errors"
"fmt"
"io"
"io/ioutil"
@ -10,7 +11,7 @@ import (
"time"
"github.com/boltdb/bolt"
"github.com/portainer/portainer/api/dataservices/errors"
dserrors "github.com/portainer/portainer/api/dataservices/errors"
"github.com/sirupsen/logrus"
)
@ -19,6 +20,11 @@ const (
EncryptedDatabaseFileName = "portainer.edb"
)
var (
ErrHaveEncryptedAndUnencrypted = errors.New("Portainer has detected both an encrypted and un-encrypted database and cannot start. Only one database should exist")
ErrHaveEncryptedWithNoKey = errors.New("The portainer database is encrypted, but no secret was loaded")
)
type DbConnection struct {
Path string
MaxBatchSize int
@ -64,19 +70,51 @@ func (connection *DbConnection) IsEncryptedStore() bool {
// NeedsEncryptionMigration returns true if database encryption is enabled and
// we have an un-encrypted DB that requires migration to an encrypted DB
func (connection *DbConnection) NeedsEncryptionMigration() bool {
if connection.EncryptionKey != nil {
dbFile := path.Join(connection.Path, DatabaseFileName)
if _, err := os.Stat(dbFile); err == nil {
return true
}
func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
// This is an existing encrypted store or a new store.
// A new store will open encrypted from the outset
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
// 1) portainer.edb + key => False
// 2) portainer.edb + no key => ERROR Fatal!
// 3) portainer.db + key => True (needs migration)
// 4) portainer.db + no key => False
// 5) NoDB (new) + key => False
// 6) NoDB (new) + no key => False
// 7) portainer.db & portainer.edb => ERROR Fatal!
// If we have a loaded encryption key, always set encrypted
if connection.EncryptionKey != nil {
connection.SetEncrypted(true)
}
return false
// Check for portainer.db
dbFile := path.Join(connection.Path, DatabaseFileName)
_, err := os.Stat(dbFile)
haveDbFile := err == nil
// Check for portainer.edb
edbFile := path.Join(connection.Path, EncryptedDatabaseFileName)
_, err = os.Stat(edbFile)
haveEdbFile := err == nil
if haveDbFile && haveEdbFile {
// 7 - encrypted and unencrypted db?
return false, ErrHaveEncryptedAndUnencrypted
}
if haveDbFile && connection.EncryptionKey != nil {
// 3 - needs migration
return true, nil
}
if haveEdbFile && connection.EncryptionKey == nil {
// 2 - encrypted db, but no key?
return false, ErrHaveEncryptedWithNoKey
}
// 1, 4, 5, 6
return false, nil
}
// Open opens and initializes the BoltDB database.
@ -159,7 +197,7 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
value := bucket.Get(key)
if value == nil {
return errors.ErrObjectNotFound
return dserrors.ErrObjectNotFound
}
data = make([]byte, len(value))

124
api/database/boltdb/db_test.go
Unescape View File

@ -0,0 +1,124 @@
package boltdb
import (
"os"
"path"
"testing"
"github.com/stretchr/testify/assert"
)
func Test_NeedsEncryptionMigration(t *testing.T) {
// Test the specific scenarios mentioned in NeedsEncryptionMigration
// i.e.
// Cases: Note, we need to check both portainer.db and portainer.edb
// to determine if it's a new store. We only need to differentiate between cases 2,3 and 5
// 1) portainer.edb + key => False
// 2) portainer.edb + no key => ERROR Fatal!
// 3) portainer.db + key => True (needs migration)
// 4) portainer.db + no key => False
// 5) NoDB (new) + key => False
// 6) NoDB (new) + no key => False
// 7) portainer.db & portainer.edb (key not important) => ERROR Fatal!
is := assert.New(t)
dir := t.TempDir()
cases := []struct {
name string
dbname string
key bool
expectError error
expectResult bool
}{
{
name: "portainer.edb + key",
dbname: EncryptedDatabaseFileName,
key: true,
expectError: nil,
expectResult: false,
},
{
name: "portainer.db + key (migration needed)",
dbname: DatabaseFileName,
key: true,
expectError: nil,
expectResult: true,
},
{
name: "portainer.db + no key",
dbname: DatabaseFileName,
key: false,
expectError: nil,
expectResult: false,
},
{
name: "NoDB (new) + key",
dbname: "",
key: false,
expectError: nil,
expectResult: false,
},
{
name: "NoDB (new) + no key",
dbname: "",
key: false,
expectError: nil,
expectResult: false,
},
// error tests
{
name: "portainer.edb + no key",
dbname: EncryptedDatabaseFileName,
key: false,
expectError: ErrHaveEncryptedWithNoKey,
expectResult: false,
},
{
name: "portainer.db & portainer.edb",
dbname: "both",
key: true,
expectError: ErrHaveEncryptedAndUnencrypted,
expectResult: false,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
connection := DbConnection{Path: dir}
if tc.dbname == "both" {
// Special case. If portainer.db and portainer.edb exist.
dbFile1 := path.Join(connection.Path, DatabaseFileName)
f, _ := os.Create(dbFile1)
f.Close()
defer os.Remove(dbFile1)
dbFile2 := path.Join(connection.Path, EncryptedDatabaseFileName)
f, _ = os.Create(dbFile2)
f.Close()
defer os.Remove(dbFile2)
} else if tc.dbname != "" {
dbFile := path.Join(connection.Path, tc.dbname)
f, _ := os.Create(dbFile)
f.Close()
defer os.Remove(dbFile)
}
if tc.key {
connection.EncryptionKey = []byte("secret")
}
result, err := connection.NeedsEncryptionMigration()
is.Equal(tc.expectError, err, "Fatal Error failure. Test: %s", tc.name)
is.Equal(result, tc.expectResult, "Failed test: %s", tc.name)
})
}
}

11
api/datastore/datastore.go
Unescape View File

@ -40,9 +40,16 @@ func NewStore(storePath string, fileService portainer.FileService, connection po
func (store *Store) Open() (newStore bool, err error) {
newStore = true
encryptionReq := store.connection.NeedsEncryptionMigration()
encryptionReq, err := store.connection.NeedsEncryptionMigration()
if err != nil {
return false, err
}
if encryptionReq {
store.encryptDB()
err = store.encryptDB()
if err != nil {
return false, err
}
}
err = store.connection.Open()

2
yarn.lock
Unescape View File

@ -12592,7 +12592,7 @@ locate-path@^6.0.0:
dependencies:
p-locate "^5.0.0"
lodash-es@^4.17.15, lodash-es@^4.17.21:
lodash-es@^4.17.21:
version "4.17.21"
resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee"
integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==

Write Preview
Loading…
Cancel
Save