diff --git a/api/bolt/backup.go b/api/bolt/backup.go new file mode 100644 index 000000000..26ee078d0 --- /dev/null +++ b/api/bolt/backup.go @@ -0,0 +1,142 @@ +package bolt + +import ( + "fmt" + "os" + "path" + "time" + + plog "github.com/portainer/portainer/api/bolt/log" +) + +var backupDefaults = struct { + backupDir string + commonDir string + databaseFileName string +}{ + "backups", + "common", + databaseFileName, +} + +var backupLog = plog.NewScopedLog("bolt, backup") + +// +// Backup Helpers +// + +// createBackupFolders create initial folders for backups +func (store *Store) createBackupFolders() { + // create common dir + commonDir := store.commonBackupDir() + if exists, _ := store.fileService.FileExists(commonDir); !exists { + if err := os.MkdirAll(commonDir, 0700); err != nil { + backupLog.Error("Error while creating common backup folder", err) + } + } +} + +func (store *Store) databasePath() string { + return path.Join(store.path, databaseFileName) +} + +func (store *Store) commonBackupDir() string { + return path.Join(store.path, backupDefaults.backupDir, backupDefaults.commonDir) +} + +func (store *Store) copyDBFile(from string, to string) error { + backupLog.Info(fmt.Sprintf("Copying db file from %s to %s", from, to)) + err := store.fileService.Copy(from, to, true) + if err != nil { + backupLog.Error("Failed", err) + } + return err +} + +// BackupOptions provide a helper to inject backup options +type BackupOptions struct { + Version int + BackupDir string + BackupFileName string + BackupPath string +} + +func (store *Store) setupOptions(options *BackupOptions) *BackupOptions { + if options == nil { + options = &BackupOptions{} + } + if options.Version == 0 { + options.Version, _ = store.version() + } + if options.BackupDir == "" { + options.BackupDir = store.commonBackupDir() + } + if options.BackupFileName == "" { + options.BackupFileName = fmt.Sprintf("%s.%s.%s", backupDefaults.databaseFileName, fmt.Sprintf("%03d", options.Version), time.Now().Format("20060102150405")) + } + if options.BackupPath == "" { + options.BackupPath = path.Join(options.BackupDir, options.BackupFileName) + } + return options +} + +// BackupWithOptions backup current database with options +func (store *Store) BackupWithOptions(options *BackupOptions) (string, error) { + backupLog.Info("creating db backup") + store.createBackupFolders() + + options = store.setupOptions(options) + + return options.BackupPath, store.copyDBFile(store.databasePath(), options.BackupPath) +} + +// RestoreWithOptions previously saved backup for the current Edition with options +// Restore strategies: +// - default: restore latest from current edition +// - restore a specific +func (store *Store) RestoreWithOptions(options *BackupOptions) error { + options = store.setupOptions(options) + + // Check if backup file exist before restoring + _, err := os.Stat(options.BackupPath) + if os.IsNotExist(err) { + backupLog.Error(fmt.Sprintf("Backup file to restore does not exist %s", options.BackupPath), err) + return err + } + + err = store.Close() + if err != nil { + backupLog.Error("Error while closing store before restore", err) + return err + } + + backupLog.Info("Restoring db backup") + err = store.copyDBFile(options.BackupPath, store.databasePath()) + if err != nil { + return err + } + + return store.Open() +} + +// RemoveWithOptions removes backup database based on supplied options +func (store *Store) RemoveWithOptions(options *BackupOptions) error { + backupLog.Info("Removing db backup") + + options = store.setupOptions(options) + _, err := os.Stat(options.BackupPath) + + if os.IsNotExist(err) { + backupLog.Error(fmt.Sprintf("Backup file to remove does not exist %s", options.BackupPath), err) + return err + } + + backupLog.Info(fmt.Sprintf("Removing db file at %s", options.BackupPath)) + err = os.Remove(options.BackupPath) + if err != nil { + backupLog.Error("Failed", err) + return err + } + + return nil +} diff --git a/api/bolt/backup_test.go b/api/bolt/backup_test.go new file mode 100644 index 000000000..c2a61252e --- /dev/null +++ b/api/bolt/backup_test.go @@ -0,0 +1,116 @@ +package bolt + +import ( + "fmt" + "os" + "path" + "path/filepath" + "testing" + + portainer "github.com/portainer/portainer/api" +) + +// isFileExist is helper function to check for file existence +func isFileExist(path string) bool { + matches, err := filepath.Glob(path) + if err != nil { + return false + } + return len(matches) > 0 +} + +func TestCreateBackupFolders(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + + backupPath := path.Join(store.path, backupDefaults.backupDir) + + if isFileExist(backupPath) { + t.Error("Expect backups folder to not exist") + } + + store.createBackupFolders() + if !isFileExist(backupPath) { + t.Error("Expect backups folder to exist") + } +} + +func TestStoreCreation(t *testing.T) { + store, teardown := MustNewTestStore(true) + defer teardown() + + if store == nil { + t.Error("Expect to create a store") + } + + if store.edition() != portainer.PortainerCE { + t.Error("Expect to get CE Edition") + } +} + +func TestBackup(t *testing.T) { + store, teardown := MustNewTestStore(true) + defer teardown() + + t.Run("Backup should create default db backup", func(t *testing.T) { + store.VersionService.StoreDBVersion(portainer.DBVersion) + store.BackupWithOptions(nil) + + backupFileName := path.Join(store.path, "backups", "common", fmt.Sprintf("portainer.db.%03d.*", portainer.DBVersion)) + if !isFileExist(backupFileName) { + t.Errorf("Expect backup file to be created %s", backupFileName) + } + }) + + t.Run("BackupWithOption should create a name specific backup at common path", func(t *testing.T) { + store.BackupWithOptions(&BackupOptions{ + BackupFileName: beforePortainerVersionUpgradeBackup, + BackupDir: store.commonBackupDir(), + }) + backupFileName := path.Join(store.path, "backups", "common", beforePortainerVersionUpgradeBackup) + if !isFileExist(backupFileName) { + t.Errorf("Expect backup file to be created %s", backupFileName) + } + }) +} + +func TestRemoveWithOptions(t *testing.T) { + store, teardown := MustNewTestStore(true) + defer teardown() + + t.Run("successfully removes file if existent", func(t *testing.T) { + store.createBackupFolders() + options := &BackupOptions{ + BackupDir: store.commonBackupDir(), + BackupFileName: "test.txt", + } + + filePath := path.Join(options.BackupDir, options.BackupFileName) + f, err := os.Create(filePath) + if err != nil { + t.Fatalf("file should be created; err=%s", err) + } + f.Close() + + err = store.RemoveWithOptions(options) + if err != nil { + t.Errorf("RemoveWithOptions should successfully remove file; err=%w", err) + } + + if isFileExist(f.Name()) { + t.Errorf("RemoveWithOptions should successfully remove file; file=%s", f.Name()) + } + }) + + t.Run("fails to removes file if non-existent", func(t *testing.T) { + options := &BackupOptions{ + BackupDir: store.commonBackupDir(), + BackupFileName: "test.txt", + } + + err := store.RemoveWithOptions(options) + if err == nil { + t.Error("RemoveWithOptions should fail for non-existent file") + } + }) +} diff --git a/api/bolt/datastore.go b/api/bolt/datastore.go index da8197cd9..ecc4da880 100644 --- a/api/bolt/datastore.go +++ b/api/bolt/datastore.go @@ -2,7 +2,6 @@ package bolt import ( "io" - "log" "path" "time" @@ -21,7 +20,6 @@ import ( "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/bolt/extension" "github.com/portainer/portainer/api/bolt/internal" - "github.com/portainer/portainer/api/bolt/migrator" "github.com/portainer/portainer/api/bolt/registry" "github.com/portainer/portainer/api/bolt/resourcecontrol" "github.com/portainer/portainer/api/bolt/role" @@ -36,7 +34,6 @@ import ( "github.com/portainer/portainer/api/bolt/user" "github.com/portainer/portainer/api/bolt/version" "github.com/portainer/portainer/api/bolt/webhook" - "github.com/portainer/portainer/api/internal/authorization" ) const ( @@ -76,6 +73,14 @@ type Store struct { WebhookService *webhook.Service } +func (store *Store) version() (int, error) { + version, err := store.VersionService.DBVersion() + if err == errors.ErrObjectNotFound { + version = 0 + } + return version, err +} + func (store *Store) edition() portainer.SoftwareEdition { edition, err := store.VersionService.Edition() if err == errors.ErrObjectNotFound { @@ -133,64 +138,6 @@ func (store *Store) IsNew() bool { return store.isNew } -// CheckCurrentEdition checks if current edition is community edition -func (store *Store) CheckCurrentEdition() error { - if store.edition() != portainer.PortainerCE { - return errors.ErrWrongDBEdition - } - return nil -} - -// MigrateData automatically migrate the data based on the DBVersion. -// This process is only triggered on an existing database, not if the database was just created. -// if force is true, then migrate regardless. -func (store *Store) MigrateData(force bool) error { - if store.isNew && !force { - return store.VersionService.StoreDBVersion(portainer.DBVersion) - } - - version, err := store.VersionService.DBVersion() - if err == errors.ErrObjectNotFound { - version = 0 - } else if err != nil { - return err - } - - if version < portainer.DBVersion { - migratorParams := &migrator.Parameters{ - DB: store.connection.DB, - DatabaseVersion: version, - EndpointGroupService: store.EndpointGroupService, - EndpointService: store.EndpointService, - EndpointRelationService: store.EndpointRelationService, - ExtensionService: store.ExtensionService, - RegistryService: store.RegistryService, - ResourceControlService: store.ResourceControlService, - RoleService: store.RoleService, - ScheduleService: store.ScheduleService, - SettingsService: store.SettingsService, - StackService: store.StackService, - TagService: store.TagService, - TeamMembershipService: store.TeamMembershipService, - UserService: store.UserService, - VersionService: store.VersionService, - FileService: store.fileService, - DockerhubService: store.DockerHubService, - AuthorizationService: authorization.NewService(store), - } - migrator := migrator.NewMigrator(migratorParams) - - log.Printf("Migrating database from version %v to %v.\n", version, portainer.DBVersion) - err = migrator.Migrate() - if err != nil { - log.Printf("An error occurred during database migration: %s\n", err) - return err - } - } - - return nil -} - // BackupTo backs up db to a provided writer. // It does hot backup and doesn't block other database reads and writes func (store *Store) BackupTo(w io.Writer) error { @@ -199,3 +146,11 @@ func (store *Store) BackupTo(w io.Writer) error { return err }) } + +// CheckCurrentEdition checks if current edition is community edition +func (store *Store) CheckCurrentEdition() error { + if store.edition() != portainer.PortainerCE { + return errors.ErrWrongDBEdition + } + return nil +} diff --git a/api/bolt/migrate_data.go b/api/bolt/migrate_data.go new file mode 100644 index 000000000..a7cb3863d --- /dev/null +++ b/api/bolt/migrate_data.go @@ -0,0 +1,146 @@ +package bolt + +import ( + "fmt" + + "github.com/portainer/portainer/api/cli" + + werrors "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/bolt/errors" + plog "github.com/portainer/portainer/api/bolt/log" + "github.com/portainer/portainer/api/bolt/migrator" + "github.com/portainer/portainer/api/internal/authorization" +) + +const beforePortainerVersionUpgradeBackup = "portainer.db.bak" + +var migrateLog = plog.NewScopedLog("bolt, migrate") + +// FailSafeMigrate backup and restore DB if migration fail +func (store *Store) FailSafeMigrate(migrator *migrator.Migrator) error { + defer func() { + if err := recover(); err != nil { + migrateLog.Info(fmt.Sprintf("Error during migration, recovering [%v]", err)) + store.Rollback(true) + } + }() + return migrator.Migrate() +} + +// MigrateData automatically migrate the data based on the DBVersion. +// This process is only triggered on an existing database, not if the database was just created. +// if force is true, then migrate regardless. +func (store *Store) MigrateData(force bool) error { + if store.isNew && !force { + return store.VersionService.StoreDBVersion(portainer.DBVersion) + } + + migrator, err := store.newMigrator() + if err != nil { + return err + } + + // backup db file before upgrading DB to support rollback + isUpdating, err := store.VersionService.IsUpdating() + if err != nil && err != errors.ErrObjectNotFound { + return err + } + + if !isUpdating && migrator.Version() != portainer.DBVersion { + err = store.backupVersion(migrator) + if err != nil { + return werrors.Wrapf(err, "failed to backup database") + } + } + + if migrator.Version() < portainer.DBVersion { + migrateLog.Info(fmt.Sprintf("Migrating database from version %v to %v.\n", migrator.Version(), portainer.DBVersion)) + err = store.FailSafeMigrate(migrator) + if err != nil { + migrateLog.Error("An error occurred during database migration", err) + return err + } + } + + return nil +} + +func (store *Store) newMigrator() (*migrator.Migrator, error) { + version, err := store.version() + if err != nil { + return nil, err + } + + migratorParams := &migrator.Parameters{ + DB: store.connection.DB, + DatabaseVersion: version, + EndpointGroupService: store.EndpointGroupService, + EndpointService: store.EndpointService, + EndpointRelationService: store.EndpointRelationService, + ExtensionService: store.ExtensionService, + RegistryService: store.RegistryService, + ResourceControlService: store.ResourceControlService, + RoleService: store.RoleService, + ScheduleService: store.ScheduleService, + SettingsService: store.SettingsService, + StackService: store.StackService, + TagService: store.TagService, + TeamMembershipService: store.TeamMembershipService, + UserService: store.UserService, + VersionService: store.VersionService, + FileService: store.fileService, + DockerhubService: store.DockerHubService, + AuthorizationService: authorization.NewService(store), + } + return migrator.NewMigrator(migratorParams), nil +} + +// getBackupRestoreOptions returns options to store db at common backup dir location; used by: +// - db backup prior to version upgrade +// - db rollback +func getBackupRestoreOptions(store *Store) *BackupOptions { + return &BackupOptions{ + BackupDir: store.commonBackupDir(), + BackupFileName: beforePortainerVersionUpgradeBackup, + } +} + +// backupVersion will backup the database or panic if any errors occur +func (store *Store) backupVersion(migrator *migrator.Migrator) error { + migrateLog.Info("Backing up database prior to version upgrade...") + + options := getBackupRestoreOptions(store) + + _, err := store.BackupWithOptions(options) + if err != nil { + migrateLog.Error("An error occurred during database backup", err) + removalErr := store.RemoveWithOptions(options) + if removalErr != nil { + migrateLog.Error("An error occurred during store removal prior to backup", err) + } + return err + } + + return nil +} + +// Rollback to a pre-upgrade backup copy/snapshot of portainer.db +func (store *Store) Rollback(force bool) error { + + if !force { + confirmed, err := cli.Confirm("Are you sure you want to rollback your database to the previous backup?") + if err != nil || !confirmed { + return err + } + } + + options := getBackupRestoreOptions(store) + + err := store.RestoreWithOptions(options) + if err != nil { + return err + } + + return store.Close() +} diff --git a/api/bolt/migrate_data_test.go b/api/bolt/migrate_data_test.go new file mode 100644 index 000000000..895d8695e --- /dev/null +++ b/api/bolt/migrate_data_test.go @@ -0,0 +1,161 @@ +package bolt + +import ( + "fmt" + "log" + "strings" + "testing" + + portainer "github.com/portainer/portainer/api" +) + +// testVersion is a helper which tests current store version against wanted version +func testVersion(store *Store, versionWant int, t *testing.T) { + if v, _ := store.version(); v != versionWant { + t.Errorf("Expect store version to be %d but was %d", versionWant, v) + } +} + +func TestMigrateData(t *testing.T) { + t.Run("MigrateData for New Store", func(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + store.MigrateData(false) + testVersion(store, portainer.DBVersion, t) + store.Close() + }) + + tests := []struct { + version int + expectedVersion int + }{ + {version: 2, expectedVersion: portainer.DBVersion}, + {version: 21, expectedVersion: portainer.DBVersion}, + } + for _, tc := range tests { + store, teardown := MustNewTestStore(true) + defer teardown() + + // Setup data + store.VersionService.StoreDBVersion(tc.version) + + // Required roles by migrations 22.2 + store.RoleService.CreateRole(&portainer.Role{ID: 1}) + store.RoleService.CreateRole(&portainer.Role{ID: 2}) + store.RoleService.CreateRole(&portainer.Role{ID: 3}) + store.RoleService.CreateRole(&portainer.Role{ID: 4}) + + t.Run(fmt.Sprintf("MigrateData for version %d", tc.version), func(t *testing.T) { + store.MigrateData(true) + testVersion(store, tc.expectedVersion, t) + }) + + t.Run(fmt.Sprintf("Restoring DB after migrateData for version %d", tc.version), func(t *testing.T) { + store.Rollback(true) + store.Open() + testVersion(store, tc.version, t) + }) + } + + t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + + version := 2 + store.VersionService.StoreDBVersion(version) + + store.MigrateData(true) + + testVersion(store, version, t) + }) + + t.Run("MigrateData should create backup file upon update", func(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + store.VersionService.StoreDBVersion(0) + + store.MigrateData(true) + + options := store.setupOptions(getBackupRestoreOptions(store)) + + if !isFileExist(options.BackupPath) { + t.Errorf("Backup file should exist; file=%s", options.BackupPath) + } + }) + + t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + + store.VersionService.StoreIsUpdating(true) + + store.MigrateData(true) + + options := store.setupOptions(getBackupRestoreOptions(store)) + + if isFileExist(options.BackupPath) { + t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath) + } + }) + + t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + + store.MigrateData(true) + + options := store.setupOptions(getBackupRestoreOptions(store)) + + if isFileExist(options.BackupPath) { + t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath) + } + }) + +} + +func Test_getBackupRestoreOptions(t *testing.T) { + store, teardown := MustNewTestStore(false) + defer teardown() + + options := getBackupRestoreOptions(store) + + wantDir := store.commonBackupDir() + if !strings.HasSuffix(options.BackupDir, wantDir) { + log.Fatalf("incorrect backup dir; got=%s, want=%s", options.BackupDir, wantDir) + } + + wantFilename := "portainer.db.bak" + if options.BackupFileName != wantFilename { + log.Fatalf("incorrect backup file; got=%s, want=%s", options.BackupFileName, wantFilename) + } +} + +func TestRollback(t *testing.T) { + t.Run("Rollback should restore upgrade after backup", func(t *testing.T) { + version := 21 + store, teardown := MustNewTestStore(false) + defer teardown() + store.VersionService.StoreDBVersion(version) + + _, err := store.BackupWithOptions(getBackupRestoreOptions(store)) + if err != nil { + log.Fatal(err) + } + + // Change the current edition + err = store.VersionService.StoreDBVersion(version + 10) + if err != nil { + log.Fatal(err) + } + + err = store.Rollback(true) + if err != nil { + t.Logf("Rollback failed: %s", err) + t.Fail() + return + } + + store.Open() + testVersion(store, version, t) + }) +} diff --git a/api/bolt/migrator/migrate_ce.go b/api/bolt/migrator/migrate_ce.go new file mode 100644 index 000000000..15c8885bb --- /dev/null +++ b/api/bolt/migrator/migrate_ce.go @@ -0,0 +1,318 @@ +package migrator + +import ( + "fmt" + + werrors "github.com/pkg/errors" + portainer "github.com/portainer/portainer/api" +) + +func migrationError(err error, context string) error { + return werrors.Wrap(err, "failed in "+context) +} + +// Migrate checks the database version and migrate the existing data to the most recent data model. +func (m *Migrator) Migrate() error { + // set DB to updating status + err := m.versionService.StoreIsUpdating(true) + if err != nil { + return migrationError(err, "StoreIsUpdating") + } + + // Portainer < 1.12 + if m.currentDBVersion < 1 { + err := m.updateAdminUserToDBVersion1() + if err != nil { + return migrationError(err, "updateAdminUserToDBVersion1") + } + } + + // Portainer 1.12.x + if m.currentDBVersion < 2 { + err := m.updateResourceControlsToDBVersion2() + if err != nil { + return migrationError(err, "updateResourceControlsToDBVersion2") + } + err = m.updateEndpointsToDBVersion2() + if err != nil { + return migrationError(err, "updateEndpointsToDBVersion2") + } + } + + // Portainer 1.13.x + if m.currentDBVersion < 3 { + err := m.updateSettingsToDBVersion3() + if err != nil { + return migrationError(err, "updateSettingsToDBVersion3") + } + } + + // Portainer 1.14.0 + if m.currentDBVersion < 4 { + err := m.updateEndpointsToDBVersion4() + if err != nil { + return migrationError(err, "updateEndpointsToDBVersion4") + } + } + + // https://github.com/portainer/portainer/issues/1235 + if m.currentDBVersion < 5 { + err := m.updateSettingsToVersion5() + if err != nil { + return migrationError(err, "updateSettingsToVersion5") + } + } + + // https://github.com/portainer/portainer/issues/1236 + if m.currentDBVersion < 6 { + err := m.updateSettingsToVersion6() + if err != nil { + return migrationError(err, "updateSettingsToVersion6") + } + } + + // https://github.com/portainer/portainer/issues/1449 + if m.currentDBVersion < 7 { + err := m.updateSettingsToVersion7() + if err != nil { + return migrationError(err, "updateSettingsToVersion7") + } + } + + if m.currentDBVersion < 8 { + err := m.updateEndpointsToVersion8() + if err != nil { + return migrationError(err, "updateEndpointsToVersion8") + } + } + + // https: //github.com/portainer/portainer/issues/1396 + if m.currentDBVersion < 9 { + err := m.updateEndpointsToVersion9() + if err != nil { + return migrationError(err, "updateEndpointsToVersion9") + } + } + + // https://github.com/portainer/portainer/issues/461 + if m.currentDBVersion < 10 { + err := m.updateEndpointsToVersion10() + if err != nil { + return migrationError(err, "updateEndpointsToVersion10") + } + } + + // https://github.com/portainer/portainer/issues/1906 + if m.currentDBVersion < 11 { + err := m.updateEndpointsToVersion11() + if err != nil { + return migrationError(err, "updateEndpointsToVersion11") + } + } + + // Portainer 1.18.0 + if m.currentDBVersion < 12 { + err := m.updateEndpointsToVersion12() + if err != nil { + return migrationError(err, "updateEndpointsToVersion12") + } + + err = m.updateEndpointGroupsToVersion12() + if err != nil { + return migrationError(err, "updateEndpointGroupsToVersion12") + } + + err = m.updateStacksToVersion12() + if err != nil { + return migrationError(err, "updateStacksToVersion12") + } + } + + // Portainer 1.19.0 + if m.currentDBVersion < 13 { + err := m.updateSettingsToVersion13() + if err != nil { + return migrationError(err, "updateSettingsToVersion13") + } + } + + // Portainer 1.19.2 + if m.currentDBVersion < 14 { + err := m.updateResourceControlsToDBVersion14() + if err != nil { + return migrationError(err, "updateResourceControlsToDBVersion14") + } + } + + // Portainer 1.20.0 + if m.currentDBVersion < 15 { + err := m.updateSettingsToDBVersion15() + if err != nil { + return migrationError(err, "updateSettingsToDBVersion15") + } + + err = m.updateTemplatesToVersion15() + if err != nil { + return migrationError(err, "updateTemplatesToVersion15") + } + } + + if m.currentDBVersion < 16 { + err := m.updateSettingsToDBVersion16() + if err != nil { + return migrationError(err, "updateSettingsToDBVersion16") + } + } + + // Portainer 1.20.1 + if m.currentDBVersion < 17 { + err := m.updateExtensionsToDBVersion17() + if err != nil { + return migrationError(err, "updateExtensionsToDBVersion17") + } + } + + // Portainer 1.21.0 + if m.currentDBVersion < 18 { + err := m.updateUsersToDBVersion18() + if err != nil { + return migrationError(err, "updateUsersToDBVersion18") + } + + err = m.updateEndpointsToDBVersion18() + if err != nil { + return migrationError(err, "updateEndpointsToDBVersion18") + } + + err = m.updateEndpointGroupsToDBVersion18() + if err != nil { + return migrationError(err, "updateEndpointGroupsToDBVersion18") + } + + err = m.updateRegistriesToDBVersion18() + if err != nil { + return migrationError(err, "updateRegistriesToDBVersion18") + } + } + + // Portainer 1.22.0 + if m.currentDBVersion < 19 { + err := m.updateSettingsToDBVersion19() + if err != nil { + return migrationError(err, "updateSettingsToDBVersion19") + } + } + + // Portainer 1.22.1 + if m.currentDBVersion < 20 { + err := m.updateUsersToDBVersion20() + if err != nil { + return migrationError(err, "updateUsersToDBVersion20") + } + + err = m.updateSettingsToDBVersion20() + if err != nil { + return migrationError(err, "updateSettingsToDBVersion20") + } + + err = m.updateSchedulesToDBVersion20() + if err != nil { + return migrationError(err, "updateSchedulesToDBVersion20") + } + } + + // Portainer 1.23.0 + // DBVersion 21 is missing as it was shipped as via hotfix 1.22.2 + if m.currentDBVersion < 22 { + err := m.updateResourceControlsToDBVersion22() + if err != nil { + return migrationError(err, "updateResourceControlsToDBVersion22") + } + + err = m.updateUsersAndRolesToDBVersion22() + if err != nil { + return migrationError(err, "updateUsersAndRolesToDBVersion22") + } + } + + // Portainer 1.24.0 + if m.currentDBVersion < 23 { + err := m.updateTagsToDBVersion23() + if err != nil { + return migrationError(err, "updateTagsToDBVersion23") + } + + err = m.updateEndpointsAndEndpointGroupsToDBVersion23() + if err != nil { + return migrationError(err, "updateEndpointsAndEndpointGroupsToDBVersion23") + } + } + + // Portainer 1.24.1 + if m.currentDBVersion < 24 { + err := m.updateSettingsToDB24() + if err != nil { + return migrationError(err, "updateSettingsToDB24") + } + } + + // Portainer 2.0.0 + if m.currentDBVersion < 25 { + err := m.updateSettingsToDB25() + if err != nil { + return migrationError(err, "updateSettingsToDB25") + } + + err = m.updateStacksToDB24() + if err != nil { + return migrationError(err, "updateStacksToDB24") + } + } + + // Portainer 2.1.0 + if m.currentDBVersion < 26 { + err := m.updateEndpointSettingsToDB25() + if err != nil { + return migrationError(err, "updateEndpointSettingsToDB25") + } + } + + // Portainer 2.2.0 + if m.currentDBVersion < 27 { + err := m.updateStackResourceControlToDB27() + if err != nil { + return migrationError(err, "updateStackResourceControlToDB27") + } + } + + // Portainer 2.6.0 + if m.currentDBVersion < 30 { + err := m.migrateDBVersionToDB30() + if err != nil { + return migrationError(err, "migrateDBVersionToDB30") + } + } + + // Portainer 2.9.0 + if m.currentDBVersion < 32 { + err := m.migrateDBVersionToDB32() + if err != nil { + return migrationError(err, "migrateDBVersionToDB32") + } + } + + if m.currentDBVersion < 33 { + if err := m.migrateDBVersionTo33(); err != nil { + return migrationError(err, "migrateDBVersionTo33") + } + } + + err = m.versionService.StoreDBVersion(portainer.DBVersion) + if err != nil { + return migrationError(err, "StoreDBVersion") + } + migrateLog.Info(fmt.Sprintf("Updated DB version to %d", portainer.DBVersion)) + + // reset DB updating status + return m.versionService.StoreIsUpdating(false) +} diff --git a/api/bolt/migrator/migrator.go b/api/bolt/migrator/migrator.go index df3ad0436..622db7e80 100644 --- a/api/bolt/migrator/migrator.go +++ b/api/bolt/migrator/migrator.go @@ -27,8 +27,9 @@ var migrateLog = plog.NewScopedLog("bolt, migrate") type ( // Migrator defines a service to migrate data after a Portainer version update. Migrator struct { - currentDBVersion int - db *bolt.DB + db *bolt.DB + currentDBVersion int + endpointGroupService *endpointgroup.Service endpointService *endpoint.Service endpointRelationService *endpointrelation.Service @@ -97,295 +98,7 @@ func NewMigrator(parameters *Parameters) *Migrator { } } -// Migrate checks the database version and migrate the existing data to the most recent data model. -func (m *Migrator) Migrate() error { - // Portainer < 1.12 - if m.currentDBVersion < 1 { - err := m.updateAdminUserToDBVersion1() - if err != nil { - return err - } - } - - // Portainer 1.12.x - if m.currentDBVersion < 2 { - err := m.updateResourceControlsToDBVersion2() - if err != nil { - return err - } - err = m.updateEndpointsToDBVersion2() - if err != nil { - return err - } - } - - // Portainer 1.13.x - if m.currentDBVersion < 3 { - err := m.updateSettingsToDBVersion3() - if err != nil { - return err - } - } - - // Portainer 1.14.0 - if m.currentDBVersion < 4 { - err := m.updateEndpointsToDBVersion4() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1235 - if m.currentDBVersion < 5 { - err := m.updateSettingsToVersion5() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1236 - if m.currentDBVersion < 6 { - err := m.updateSettingsToVersion6() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1449 - if m.currentDBVersion < 7 { - err := m.updateSettingsToVersion7() - if err != nil { - return err - } - } - - if m.currentDBVersion < 8 { - err := m.updateEndpointsToVersion8() - if err != nil { - return err - } - } - - // https: //github.com/portainer/portainer/issues/1396 - if m.currentDBVersion < 9 { - err := m.updateEndpointsToVersion9() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/461 - if m.currentDBVersion < 10 { - err := m.updateEndpointsToVersion10() - if err != nil { - return err - } - } - - // https://github.com/portainer/portainer/issues/1906 - if m.currentDBVersion < 11 { - err := m.updateEndpointsToVersion11() - if err != nil { - return err - } - } - - // Portainer 1.18.0 - if m.currentDBVersion < 12 { - err := m.updateEndpointsToVersion12() - if err != nil { - return err - } - - err = m.updateEndpointGroupsToVersion12() - if err != nil { - return err - } - - err = m.updateStacksToVersion12() - if err != nil { - return err - } - } - - // Portainer 1.19.0 - if m.currentDBVersion < 13 { - err := m.updateSettingsToVersion13() - if err != nil { - return err - } - } - - // Portainer 1.19.2 - if m.currentDBVersion < 14 { - err := m.updateResourceControlsToDBVersion14() - if err != nil { - return err - } - } - - // Portainer 1.20.0 - if m.currentDBVersion < 15 { - err := m.updateSettingsToDBVersion15() - if err != nil { - return err - } - - err = m.updateTemplatesToVersion15() - if err != nil { - return err - } - } - - if m.currentDBVersion < 16 { - err := m.updateSettingsToDBVersion16() - if err != nil { - return err - } - } - - // Portainer 1.20.1 - if m.currentDBVersion < 17 { - err := m.updateExtensionsToDBVersion17() - if err != nil { - return err - } - } - - // Portainer 1.21.0 - if m.currentDBVersion < 18 { - err := m.updateUsersToDBVersion18() - if err != nil { - return err - } - - err = m.updateEndpointsToDBVersion18() - if err != nil { - return err - } - - err = m.updateEndpointGroupsToDBVersion18() - if err != nil { - return err - } - - err = m.updateRegistriesToDBVersion18() - if err != nil { - return err - } - } - - // Portainer 1.22.0 - if m.currentDBVersion < 19 { - err := m.updateSettingsToDBVersion19() - if err != nil { - return err - } - } - - // Portainer 1.22.1 - if m.currentDBVersion < 20 { - err := m.updateUsersToDBVersion20() - if err != nil { - return err - } - - err = m.updateSettingsToDBVersion20() - if err != nil { - return err - } - - err = m.updateSchedulesToDBVersion20() - if err != nil { - return err - } - } - - // Portainer 1.23.0 - // DBVersion 21 is missing as it was shipped as via hotfix 1.22.2 - if m.currentDBVersion < 22 { - err := m.updateResourceControlsToDBVersion22() - if err != nil { - return err - } - - err = m.updateUsersAndRolesToDBVersion22() - if err != nil { - return err - } - } - - // Portainer 1.24.0 - if m.currentDBVersion < 23 { - err := m.updateTagsToDBVersion23() - if err != nil { - return err - } - - err = m.updateEndpointsAndEndpointGroupsToDBVersion23() - if err != nil { - return err - } - } - - // Portainer 1.24.1 - if m.currentDBVersion < 24 { - err := m.updateSettingsToDB24() - if err != nil { - return err - } - } - - // Portainer 2.0.0 - if m.currentDBVersion < 25 { - err := m.updateSettingsToDB25() - if err != nil { - return err - } - - err = m.updateStacksToDB24() - if err != nil { - return err - } - } - - // Portainer 2.1.0 - if m.currentDBVersion < 26 { - err := m.updateEndpointSettingsToDB25() - if err != nil { - return err - } - } - - // Portainer 2.2.0 - if m.currentDBVersion < 27 { - err := m.updateStackResourceControlToDB27() - if err != nil { - return err - } - } - - // Portainer 2.6.0 - if m.currentDBVersion < 30 { - err := m.migrateDBVersionToDB30() - if err != nil { - return err - } - } - - // Portainer 2.9.0 - if m.currentDBVersion < 32 { - err := m.migrateDBVersionToDB32() - if err != nil { - return err - } - } - - if m.currentDBVersion < 33 { - if err := m.migrateDBVersionTo33(); err != nil { - return err - } - } - - return m.versionService.StoreDBVersion(portainer.DBVersion) +// Version exposes version of database +func (migrator *Migrator) Version() int { + return migrator.currentDBVersion } diff --git a/api/bolt/stack/tests/stack_test.go b/api/bolt/stack/tests/stack_test.go index d0c66dadf..862291941 100644 --- a/api/bolt/stack/tests/stack_test.go +++ b/api/bolt/stack/tests/stack_test.go @@ -4,18 +4,12 @@ import ( "testing" "time" - "github.com/portainer/portainer/api/bolt" - - bolterrors "github.com/portainer/portainer/api/bolt/errors" - - "github.com/portainer/portainer/api/bolt/bolttest" - "github.com/gofrs/uuid" - - "github.com/stretchr/testify/assert" - portainer "github.com/portainer/portainer/api" + "github.com/portainer/portainer/api/bolt" + bolterrors "github.com/portainer/portainer/api/bolt/errors" "github.com/portainer/portainer/api/filesystem" + "github.com/stretchr/testify/assert" ) func newGuidString(t *testing.T) string { @@ -35,7 +29,7 @@ func TestService_StackByWebhookID(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode. Normally takes ~1s to run.") } - store, teardown := bolttest.MustNewTestStore(true) + store, teardown := bolt.MustNewTestStore(true) defer teardown() b := stackBuilder{t: t, store: store} @@ -93,7 +87,7 @@ func Test_RefreshableStacks(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode. Normally takes ~1s to run.") } - store, teardown := bolttest.MustNewTestStore(true) + store, teardown := bolt.MustNewTestStore(true) defer teardown() staticStack := portainer.Stack{ID: 1} diff --git a/api/bolt/bolttest/datastore.go b/api/bolt/teststore.go similarity index 78% rename from api/bolt/bolttest/datastore.go rename to api/bolt/teststore.go index ac0887b2c..bfec62acb 100644 --- a/api/bolt/bolttest/datastore.go +++ b/api/bolt/teststore.go @@ -1,4 +1,4 @@ -package bolttest +package bolt import ( "io/ioutil" @@ -6,13 +6,12 @@ import ( "os" "github.com/pkg/errors" - "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/filesystem" ) var errTempDir = errors.New("can't create a temp dir") -func MustNewTestStore(init bool) (*bolt.Store, func()) { +func MustNewTestStore(init bool) (*Store, func()) { store, teardown, err := NewTestStore(init) if err != nil { if !errors.Is(err, errTempDir) { @@ -24,7 +23,7 @@ func MustNewTestStore(init bool) (*bolt.Store, func()) { return store, teardown } -func NewTestStore(init bool) (*bolt.Store, func(), error) { +func NewTestStore(init bool) (*Store, func(), error) { // Creates unique temp directory in a concurrency friendly manner. dataStorePath, err := ioutil.TempDir("", "boltdb") if err != nil { @@ -36,7 +35,7 @@ func NewTestStore(init bool) (*bolt.Store, func(), error) { return nil, nil, err } - store, err := bolt.NewStore(dataStorePath, fileService) + store, err := NewStore(dataStorePath, fileService) if err != nil { return nil, nil, err } @@ -60,7 +59,7 @@ func NewTestStore(init bool) (*bolt.Store, func(), error) { return store, teardown, nil } -func teardown(store *bolt.Store, dataStorePath string) { +func teardown(store *Store, dataStorePath string) { err := store.Close() if err != nil { log.Fatalln(err) diff --git a/api/bolt/version/version.go b/api/bolt/version/version.go index c697173ff..d43be4d67 100644 --- a/api/bolt/version/version.go +++ b/api/bolt/version/version.go @@ -15,6 +15,7 @@ const ( versionKey = "DB_VERSION" instanceKey = "INSTANCE_ID" editionKey = "EDITION" + updatingKey = "DB_UPDATING" ) // Service represents a service to manage stored versions. @@ -83,6 +84,21 @@ func (service *Service) StoreDBVersion(version int) error { }) } +// IsUpdating retrieves the database updating status. +func (service *Service) IsUpdating() (bool, error) { + isUpdating, err := service.getKey(updatingKey) + if err != nil { + return false, err + } + + return strconv.ParseBool(string(isUpdating)) +} + +// StoreIsUpdating store the database updating status. +func (service *Service) StoreIsUpdating(isUpdating bool) error { + return service.setKey(updatingKey, strconv.FormatBool(isUpdating)) +} + // InstanceID retrieves the stored instance ID. func (service *Service) InstanceID() (string, error) { var data []byte diff --git a/api/cli/cli.go b/api/cli/cli.go index e5c984807..4518df71f 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -47,6 +47,7 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) { SSL: kingpin.Flag("ssl", "Secure Portainer instance using SSL (deprecated)").Default(defaultSSL).Bool(), SSLCert: kingpin.Flag("sslcert", "Path to the SSL certificate used to secure the Portainer instance").String(), SSLKey: kingpin.Flag("sslkey", "Path to the SSL key used to secure the Portainer instance").String(), + Rollback: kingpin.Flag("rollback", "Rollback the database store to the previous version").Bool(), SnapshotInterval: kingpin.Flag("snapshot-interval", "Duration between each environment snapshot job").Default(defaultSnapshotInterval).String(), AdminPassword: kingpin.Flag("admin-password", "Hashed admin password").String(), AdminPasswordFile: kingpin.Flag("admin-password-file", "Path to the file containing the password for the admin user").String(), diff --git a/api/cli/confirm.go b/api/cli/confirm.go new file mode 100644 index 000000000..90fff2354 --- /dev/null +++ b/api/cli/confirm.go @@ -0,0 +1,24 @@ +package cli + +import ( + "bufio" + "log" + "os" + "strings" +) + +// Confirm starts a rollback db cli application +func Confirm(message string) (bool, error) { + log.Printf("%s [y/N]", message) + + reader := bufio.NewReader(os.Stdin) + answer, err := reader.ReadString('\n') + if err != nil { + return false, err + } + answer = strings.Replace(answer, "\n", "", -1) + answer = strings.ToLower(answer) + + return answer == "y" || answer == "yes", nil + +} diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index 1954a8161..78ca69e17 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -56,7 +56,7 @@ func initFileService(dataStorePath string) portainer.FileService { return fileService } -func initDataStore(dataStorePath string, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore { +func initDataStore(dataStorePath string, rollback bool, fileService portainer.FileService, shutdownCtx context.Context) portainer.DataStore { store, err := bolt.NewStore(dataStorePath, fileService) if err != nil { log.Fatalf("failed creating data store: %v", err) @@ -67,6 +67,17 @@ func initDataStore(dataStorePath string, fileService portainer.FileService, shut log.Fatalf("failed opening store: %v", err) } + if rollback { + err := store.Rollback(false) + if err != nil { + log.Fatalf("failed rolling back: %s", err) + } + + log.Println("Exiting rollback") + os.Exit(0) + return nil + } + err = store.Init() if err != nil { log.Fatalf("failed initializing data store: %v", err) @@ -399,7 +410,7 @@ func buildServer(flags *portainer.CLIFlags) portainer.Server { fileService := initFileService(*flags.Data) - dataStore := initDataStore(*flags.Data, fileService, shutdownCtx) + dataStore := initDataStore(*flags.Data, *flags.Rollback, fileService, shutdownCtx) if err := dataStore.CheckCurrentEdition(); err != nil { log.Fatal(err) diff --git a/api/http/handler/helm/helm_delete_test.go b/api/http/handler/helm/helm_delete_test.go index 6c83b788c..618d7c969 100644 --- a/api/http/handler/helm/helm_delete_test.go +++ b/api/http/handler/helm/helm_delete_test.go @@ -13,7 +13,7 @@ import ( "github.com/portainer/portainer/api/kubernetes" "github.com/stretchr/testify/assert" - bolt "github.com/portainer/portainer/api/bolt/bolttest" + "github.com/portainer/portainer/api/bolt" helper "github.com/portainer/portainer/api/internal/testhelpers" ) diff --git a/api/http/handler/helm/helm_install_test.go b/api/http/handler/helm/helm_install_test.go index b98ecd32f..52fdb1b2e 100644 --- a/api/http/handler/helm/helm_install_test.go +++ b/api/http/handler/helm/helm_install_test.go @@ -12,7 +12,7 @@ import ( "github.com/portainer/libhelm/options" "github.com/portainer/libhelm/release" portainer "github.com/portainer/portainer/api" - bolt "github.com/portainer/portainer/api/bolt/bolttest" + "github.com/portainer/portainer/api/bolt" "github.com/portainer/portainer/api/http/security" helper "github.com/portainer/portainer/api/internal/testhelpers" "github.com/portainer/portainer/api/kubernetes" diff --git a/api/http/handler/helm/helm_list_test.go b/api/http/handler/helm/helm_list_test.go index cc50a24aa..459aad736 100644 --- a/api/http/handler/helm/helm_list_test.go +++ b/api/http/handler/helm/helm_list_test.go @@ -14,7 +14,7 @@ import ( "github.com/portainer/portainer/api/kubernetes" "github.com/stretchr/testify/assert" - bolt "github.com/portainer/portainer/api/bolt/bolttest" + "github.com/portainer/portainer/api/bolt" helper "github.com/portainer/portainer/api/internal/testhelpers" ) diff --git a/api/http/handler/stacks/webhook_invoke_test.go b/api/http/handler/stacks/webhook_invoke_test.go index cc6656519..cd4c3946f 100644 --- a/api/http/handler/stacks/webhook_invoke_test.go +++ b/api/http/handler/stacks/webhook_invoke_test.go @@ -6,15 +6,13 @@ import ( "testing" "github.com/gofrs/uuid" - "github.com/stretchr/testify/assert" - portainer "github.com/portainer/portainer/api" - - "github.com/portainer/portainer/api/bolt/bolttest" + "github.com/portainer/portainer/api/bolt" + "github.com/stretchr/testify/assert" ) func TestHandler_webhookInvoke(t *testing.T) { - store, teardown := bolttest.MustNewTestStore(true) + store, teardown := bolt.MustNewTestStore(true) defer teardown() webhookID := newGuidString(t) diff --git a/api/internal/testhelpers/datastore.go b/api/internal/testhelpers/datastore.go index f6d5786b1..15a1fa87d 100644 --- a/api/internal/testhelpers/datastore.go +++ b/api/internal/testhelpers/datastore.go @@ -38,7 +38,7 @@ func (d *datastore) Close() error { retur func (d *datastore) CheckCurrentEdition() error { return nil } func (d *datastore) IsNew() bool { return false } func (d *datastore) MigrateData(force bool) error { return nil } -func (d *datastore) RollbackToCE() error { return nil } +func (d *datastore) Rollback(force bool) error { return nil } func (d *datastore) CustomTemplate() portainer.CustomTemplateService { return d.customTemplate } func (d *datastore) EdgeGroup() portainer.EdgeGroupService { return d.edgeGroup } func (d *datastore) EdgeJob() portainer.EdgeJobService { return d.edgeJob } diff --git a/api/portainer.go b/api/portainer.go index eb0b0992c..0220fe444 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -66,6 +66,7 @@ type ( SSL *bool SSLCert *string SSLKey *string + Rollback *bool SnapshotInterval *string } @@ -1103,6 +1104,7 @@ type ( Close() error IsNew() bool MigrateData(force bool) error + Rollback(force bool) error CheckCurrentEdition() error BackupTo(w io.Writer) error @@ -1204,6 +1206,7 @@ type ( FileService interface { GetDockerConfigPath() string GetFileContent(filePath string) ([]byte, error) + Copy(fromFilePath string, toFilePath string, deleteIfExists bool) error Rename(oldPath, newPath string) error RemoveDirectory(directoryPath string) error StoreTLSFileFromBytes(folder string, fileType TLSFileType, data []byte) (string, error) diff --git a/api/stacks/deploy_test.go b/api/stacks/deploy_test.go index 6ccb9beea..dd0b3ff69 100644 --- a/api/stacks/deploy_test.go +++ b/api/stacks/deploy_test.go @@ -7,7 +7,7 @@ import ( "testing" portainer "github.com/portainer/portainer/api" - bolt "github.com/portainer/portainer/api/bolt/bolttest" + "github.com/portainer/portainer/api/bolt" gittypes "github.com/portainer/portainer/api/git/types" "github.com/stretchr/testify/assert" )