feat(boltdb): attempt to compact using a read-only database BE-12287 (#1267)

api/database/boltdb/db.go

@@ -136,10 +136,8 @@ func (connection *DbConnection) NeedsEncryptionMigration() (bool, error) {
 func (connection *DbConnection) Open() error {
 	log.Info().Str("filename", connection.GetDatabaseFileName()).Msg("loading PortainerDB")
 
-	// Now we open the db
 	databasePath := connection.GetDatabaseFilePath()
-
-	db, err := bolt.Open(databasePath, 0600, connection.boltOptions())
+	db, err := bolt.Open(databasePath, 0600, connection.boltOptions(connection.Compact))
 	if err != nil {
 		return err
 	}
@@ -152,6 +150,15 @@ func (connection *DbConnection) Open() error {
 		log.Info().Msg("compacting database")
 		if err := connection.compact(); err != nil {
 			log.Error().Err(err).Msg("failed to compact database")
+
+			// Close the read-only database and re-open in read-write mode
+			if err := connection.Close(); err != nil {
+				log.Warn().Err(err).Msg("failure to close the database after failed compaction")
+			}
+
+			connection.Compact = false
+
+			return connection.Open()
 		} else {
 			log.Info().Msg("database compaction completed")
 		}
@@ -424,9 +431,14 @@ func (connection *DbConnection) RestoreMetadata(s map[string]any) error {
 }
 
 // compact attempts to compact the database and replace it iff it succeeds
-func (connection *DbConnection) compact() error {
+func (connection *DbConnection) compact() (err error) {
 	compactedPath := connection.GetDatabaseFilePath() + compactedSuffix
-	compactedDB, err := bolt.Open(compactedPath, 0o600, connection.boltOptions())
+
+	if err := os.Remove(compactedPath); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failure to remove an existing compacted database: %w", err)
+	}
+
+	compactedDB, err := bolt.Open(compactedPath, 0o600, connection.boltOptions(false))
 	if err != nil {
 		return fmt.Errorf("failure to create the compacted database: %w", err)
 	}
@@ -453,11 +465,12 @@ func (connection *DbConnection) compact() error {
 	return nil
 }
 
-func (connection *DbConnection) boltOptions() *bolt.Options {
+func (connection *DbConnection) boltOptions(readOnly bool) *bolt.Options {
 	return &bolt.Options{
 		Timeout:         1 * time.Second,
 		InitialMmapSize: connection.InitialMmapSize,
 		FreelistType:    bolt.FreelistMapType,
 		NoFreelistSync:  true,
+		ReadOnly:        readOnly,
 	}
 }

api/database/boltdb/db_test.go

@@ -5,6 +5,8 @@ import (
 	"path"
 	"testing"
 
+	"github.com/portainer/portainer/api/filesystem"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.etcd.io/bbolt"
@@ -123,10 +125,7 @@ func Test_NeedsEncryptionMigration(t *testing.T) {
 }
 
 func TestDBCompaction(t *testing.T) {
-	db := &DbConnection{
-		Path:    t.TempDir(),
-		Compact: true,
-	}
+	db := &DbConnection{Path: t.TempDir()}
 
 	err := db.Open()
 	require.NoError(t, err)
@@ -147,6 +146,7 @@ func TestDBCompaction(t *testing.T) {
 	require.NoError(t, err)
 
 	// Reopen the DB to trigger compaction
+	db.Compact = true
 	err = db.Open()
 	require.NoError(t, err)
 
@@ -168,10 +168,14 @@ func TestDBCompaction(t *testing.T) {
 	require.NoError(t, err)
 
 	// Failures
-
-	err = os.Mkdir(db.GetDatabaseFilePath()+compactedSuffix, 0o755)
+	compactedPath := db.GetDatabaseFilePath() + compactedSuffix
+	err = os.Mkdir(compactedPath, 0o755)
 	require.NoError(t, err)
 
+	f, err := os.Create(filesystem.JoinPaths(compactedPath, "somefile"))
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
 	err = db.Open()
 	require.NoError(t, err)
 }