package datastore import ( "bytes" "encoding/json" "fmt" "io" "os" "path/filepath" "strings" "testing" "github.com/portainer/portainer/api/database/boltdb" "github.com/google/go-cmp/cmp" "github.com/portainer/portainer/api/database/models" "github.com/rs/zerolog/log" ) // testVersion is a helper which tests current store version against wanted version func testVersion(store *Store, versionWant string, t *testing.T) { v, err := store.VersionService.Version() if err != nil { t.Errorf("Expect store version to be %s but was %s with error: %s", versionWant, v.SchemaVersion, err) } if v.SchemaVersion != versionWant { t.Errorf("Expect store version to be %s but was %s", versionWant, v.SchemaVersion) } } func TestMigrateData(t *testing.T) { snapshotTests := []struct { testName string srcPath string wantPath string overrideInstanceId bool }{ { testName: "migrate version 24 to latest", srcPath: "test_data/input_24.json", wantPath: "test_data/output_24_to_latest.json", overrideInstanceId: true, }, } for _, test := range snapshotTests { t.Run(test.testName, func(t *testing.T) { err := migrateDBTestHelper(t, test.srcPath, test.wantPath, test.overrideInstanceId) if err != nil { t.Errorf( "Failed migrating mock database %v: %v", test.srcPath, err, ) } }) } // t.Run("MigrateData for New Store & Re-Open Check", func(t *testing.T) { // newStore, store, teardown := MustNewTestStore(t, true, false) // defer teardown() // if !newStore { // t.Error("Expect a new DB") // } // testVersion(store, portainer.APIVersion, t) // store.Close() // newStore, _ = store.Open() // if newStore { // t.Error("Expect store to NOT be new DB") // } // }) // tests := []struct { // version string // expectedVersion string // }{ // {version: "1.24.1", expectedVersion: portainer.APIVersion}, // {version: "2.0.0", expectedVersion: portainer.APIVersion}, // } // for _, tc := range tests { // _, store, teardown := MustNewTestStore(t, true, true) // defer teardown() // // Setup data // v := models.Version{SchemaVersion: tc.version} // store.VersionService.UpdateVersion(&v) // // Required roles by migrations 22.2 // store.RoleService.Create(&portainer.Role{ID: 1}) // store.RoleService.Create(&portainer.Role{ID: 2}) // store.RoleService.Create(&portainer.Role{ID: 3}) // store.RoleService.Create(&portainer.Role{ID: 4}) // t.Run(fmt.Sprintf("MigrateData for version %s", tc.version), func(t *testing.T) { // store.MigrateData() // testVersion(store, tc.expectedVersion, t) // }) // t.Run(fmt.Sprintf("Restoring DB after migrateData for version %s", tc.version), func(t *testing.T) { // store.Rollback(true) // store.Open() // testVersion(store, tc.version, t) // }) // } // t.Run("Error in MigrateData should restore backup before MigrateData", func(t *testing.T) { // _, store, teardown := MustNewTestStore(t, false, true) // defer teardown() // v := models.Version{SchemaVersion: "1.24.1"} // store.VersionService.UpdateVersion(&v) // store.MigrateData() // testVersion(store, v.SchemaVersion, t) // }) // t.Run("MigrateData should create backup file upon update", func(t *testing.T) { // _, store, teardown := MustNewTestStore(t, false, true) // defer teardown() // v := models.Version{SchemaVersion: "0.0.0"} // store.VersionService.UpdateVersion(&v) // store.MigrateData() // options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir())) // if !isFileExist(options.BackupPath) { // t.Errorf("Backup file should exist; file=%s", options.BackupPath) // } // }) // t.Run("MigrateData should fail to create backup if database file is set to updating", func(t *testing.T) { // _, store, teardown := MustNewTestStore(t, false, true) // defer teardown() // store.VersionService.StoreIsUpdating(true) // store.MigrateData() // options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir())) // if isFileExist(options.BackupPath) { // t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath) // } // }) // t.Run("MigrateData should not create backup on startup if portainer version matches db", func(t *testing.T) { // _, store, teardown := MustNewTestStore(t, false, true) // defer teardown() // store.MigrateData() // options := store.setupOptions(getBackupRestoreOptions(store.commonBackupDir())) // if isFileExist(options.BackupPath) { // t.Errorf("Backup file should not exist for dirty database; file=%s", options.BackupPath) // } // }) } func Test_getBackupRestoreOptions(t *testing.T) { _, store, teardown := MustNewTestStore(t, false, true) defer teardown() options := getBackupRestoreOptions(store.commonBackupDir()) wantDir := store.commonBackupDir() if !strings.HasSuffix(options.BackupDir, wantDir) { log.Fatal().Str("got", options.BackupDir).Str("want", wantDir).Msg("incorrect backup dir") } wantFilename := "portainer.db.bak" if options.BackupFileName != wantFilename { log.Fatal().Str("got", options.BackupFileName).Str("want", wantFilename).Msg("incorrect backup file") } } func TestRollback(t *testing.T) { t.Run("Rollback should restore upgrade after backup", func(t *testing.T) { version := models.Version{SchemaVersion: "2.4.0"} _, store, teardown := MustNewTestStore(t, true, false) defer teardown() err := store.VersionService.UpdateVersion(&version) if err != nil { t.Errorf("Failed updating version: %v", err) } _, err = store.backupWithOptions(getBackupRestoreOptions(store.commonBackupDir())) if err != nil { log.Fatal().Err(err).Msg("") } // Change the current version version2 := models.Version{SchemaVersion: "2.6.0"} err = store.VersionService.UpdateVersion(&version2) if err != nil { log.Fatal().Err(err).Msg("") } err = store.Rollback(true) if err != nil { t.Logf("Rollback failed: %s", err) t.Fail() return } _, err = store.Open() if err != nil { t.Logf("Open failed: %s", err) t.Fail() return } testVersion(store, version.SchemaVersion, t) }) } // isFileExist is helper function to check for file existence func isFileExist(path string) bool { matches, err := filepath.Glob(path) if err != nil { return false } return len(matches) > 0 } // migrateDBTestHelper loads a json representation of a bolt database from srcPath, // parses it into a database, runs a migration on that database, and then // compares it with an expected output database. func migrateDBTestHelper(t *testing.T, srcPath, wantPath string, overrideInstanceId bool) error { srcJSON, err := os.ReadFile(srcPath) if err != nil { t.Fatalf("failed loading source JSON file %v: %v", srcPath, err) } // Parse source json to db. // When we create a new test store, it sets its version field automatically to latest. _, store, _ := MustNewTestStore(t, true, false) fmt.Println("store.path=", store.GetConnection().GetDatabaseFilePath()) store.connection.DeleteObject("version", []byte("VERSION")) // defer teardown() err = importJSON(t, bytes.NewReader(srcJSON), store) if err != nil { return err } // Run the actual migrations on our input database. err = store.MigrateData() if err != nil { return err } if overrideInstanceId { // old versions of portainer did not have instance-id. Because this gets generated // we need to override the expected output to match the expected value to pass the test v, err := store.VersionService.Version() if err != nil { return err } v.InstanceID = "463d5c47-0ea5-4aca-85b1-405ceefee254" err = store.VersionService.UpdateVersion(v) if err != nil { return err } } // Assert that our database connection is using bolt so we can call // exportJson rather than ExportRaw. The exportJson function allows us to // strip out the metadata which we don't want for our tests. // TODO: update connection interface in CE to allow us to use ExportRaw and pass meta false err = store.connection.Close() if err != nil { t.Fatalf("err closing bolt connection: %v", err) } con, ok := store.connection.(*boltdb.DbConnection) if !ok { t.Fatalf("backing database is not using boltdb, but the migrations test requires it") } // Convert database back to json. databasePath := con.GetDatabaseFilePath() if _, err := os.Stat(databasePath); err != nil { return fmt.Errorf("stat on %s failed: %w", databasePath, err) } gotJSON, err := con.ExportJSON(databasePath, false) if err != nil { t.Logf( "failed re-exporting database %s to JSON: %v", databasePath, err, ) } wantJSON, err := os.ReadFile(wantPath) if err != nil { t.Fatalf("failed loading want JSON file %v: %v", wantPath, err) } // Compare the result we got with the one we wanted. if diff := cmp.Diff(wantJSON, gotJSON); diff != "" { gotPath := filepath.Join(os.TempDir(), "portainer-migrator-test-fail.json") os.WriteFile( gotPath, gotJSON, 0600, ) t.Errorf( "migrate data from %s to %s failed\nwrote migrated input to %s\nmismatch (-want +got):\n%s", srcPath, wantPath, gotPath, diff, ) } return nil } // importJSON reads input JSON and commits it to a portainer datastore.Store. // Errors are logged with the testing package. func importJSON(t *testing.T, r io.Reader, store *Store) error { objects := make(map[string]interface{}) // Parse json into map of objects. d := json.NewDecoder(r) d.UseNumber() err := d.Decode(&objects) if err != nil { return err } // Get database connection from store. con := store.connection for k, v := range objects { switch k { case "version": versions, ok := v.(map[string]interface{}) if !ok { t.Logf("failed casting %s to map[string]interface{}", k) } // New format db version, ok := versions["VERSION"] if ok { err := con.CreateObjectWithStringId( k, []byte("VERSION"), version, ) if err != nil { t.Logf("failed writing VERSION in %s: %v", k, err) } } // old format db dbVersion, ok := versions["DB_VERSION"] if ok { numDBVersion, ok := dbVersion.(json.Number) if !ok { t.Logf("failed parsing DB_VERSION as json number from %s", k) } intDBVersion, err := numDBVersion.Int64() if err != nil { t.Logf("failed casting %v to int: %v", numDBVersion, intDBVersion) } err = con.CreateObjectWithStringId( k, []byte("DB_VERSION"), int(intDBVersion), ) if err != nil { t.Logf("failed writing DB_VERSION in %s: %v", k, err) } } instanceID, ok := versions["INSTANCE_ID"] if ok { err = con.CreateObjectWithStringId( k, []byte("INSTANCE_ID"), instanceID, ) if err != nil { t.Logf("failed writing INSTANCE_ID in %s: %v", k, err) } } edition, ok := versions["EDITION"] if ok { err = con.CreateObjectWithStringId( k, []byte("EDITION"), edition, ) if err != nil { t.Logf("failed writing EDITION in %s: %v", k, err) } } case "dockerhub": obj, ok := v.([]interface{}) if !ok { t.Logf("failed to cast %s to []interface{}", k) } err := con.CreateObjectWithStringId( k, []byte("DOCKERHUB"), obj[0], ) if err != nil { t.Logf("failed writing DOCKERHUB in %s: %v", k, err) } case "ssl": obj, ok := v.(map[string]interface{}) if !ok { t.Logf("failed to case %s to map[string]interface{}", k) } err := con.CreateObjectWithStringId( k, []byte("SSL"), obj, ) if err != nil { t.Logf("failed writing SSL in %s: %v", k, err) } case "settings": obj, ok := v.(map[string]interface{}) if !ok { t.Logf("failed to case %s to map[string]interface{}", k) } err := con.CreateObjectWithStringId( k, []byte("SETTINGS"), obj, ) if err != nil { t.Logf("failed writing SETTINGS in %s: %v", k, err) } case "tunnel_server": obj, ok := v.(map[string]interface{}) if !ok { t.Logf("failed to case %s to map[string]interface{}", k) } err := con.CreateObjectWithStringId( k, []byte("INFO"), obj, ) if err != nil { t.Logf("failed writing INFO in %s: %v", k, err) } case "templates": continue default: objlist, ok := v.([]interface{}) if !ok { t.Logf("failed to cast %s to []interface{}", k) } for _, obj := range objlist { value, ok := obj.(map[string]interface{}) if !ok { t.Logf("failed to cast %v to map[string]interface{}", obj) } else { var ok bool var id interface{} switch k { case "endpoint_relations": // TODO: need to make into an int, then do that weird // stringification id, ok = value["EndpointID"] default: id, ok = value["Id"] } if !ok { // endpoint_relations: EndpointID t.Logf("missing Id field: %s", k) id = "error" } n, ok := id.(json.Number) if !ok { t.Logf("failed to cast %v to json.Number in %s", id, k) } else { key, err := n.Int64() if err != nil { t.Logf("failed to cast %v to int in %s", n, k) } else { err := con.CreateObjectWithId( k, int(key), value, ) if err != nil { t.Logf("failed writing %v in %s: %v", key, k, err) } } } } } } } return nil }