diff --git a/api/cli/cli.go b/api/cli/cli.go index 267f584d9..b4c0d6e10 100644 --- a/api/cli/cli.go +++ b/api/cli/cli.go @@ -56,6 +56,7 @@ func CLIFlags() *portainer.CLIFlags { PullLimitCheckDisabled: kingpin.Flag("pull-limit-check-disabled", "Pull limit check").Envar(portainer.PullLimitCheckDisabledEnvVar).Default(defaultPullLimitCheckDisabled).Bool(), TrustedOrigins: kingpin.Flag("trusted-origins", "List of trusted origins for CSRF protection. Separate multiple origins with a comma.").Envar(portainer.TrustedOriginsEnvVar).String(), CSP: kingpin.Flag("csp", "Content Security Policy (CSP) header").Envar(portainer.CSPEnvVar).Default("true").Bool(), + CompactDB: kingpin.Flag("compact-db", "Enable database compaction on startup").Envar(portainer.CompactDBEnvVar).Default("false").Bool(), } } diff --git a/api/cmd/portainer/main.go b/api/cmd/portainer/main.go index c569d0d67..691490eea 100644 --- a/api/cmd/portainer/main.go +++ b/api/cmd/portainer/main.go @@ -84,7 +84,7 @@ func initFileService(dataStorePath string) portainer.FileService { } func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore { - connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey) + connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey, *flags.CompactDB) if err != nil { log.Fatal().Err(err).Msg("failed creating database connection") } diff --git a/api/database/boltdb/db.go b/api/database/boltdb/db.go index 32a1b55c3..3ff8f19d6 100644 --- a/api/database/boltdb/db.go +++ b/api/database/boltdb/db.go @@ -21,6 +21,9 @@ import ( const ( DatabaseFileName = "portainer.db" EncryptedDatabaseFileName = "portainer.edb" + + txMaxSize = 65536 + compactedSuffix = ".compacted" ) var ( @@ -35,6 +38,7 @@ type DbConnection struct { InitialMmapSize int EncryptionKey []byte isEncrypted bool + Compact bool *bolt.DB } @@ -135,12 +139,7 @@ func (connection *DbConnection) Open() error { // Now we open the db databasePath := connection.GetDatabaseFilePath() - db, err := bolt.Open(databasePath, 0600, &bolt.Options{ - Timeout: 1 * time.Second, - InitialMmapSize: connection.InitialMmapSize, - FreelistType: bolt.FreelistMapType, - NoFreelistSync: true, - }) + db, err := bolt.Open(databasePath, 0600, connection.boltOptions()) if err != nil { return err } @@ -149,6 +148,15 @@ func (connection *DbConnection) Open() error { db.MaxBatchDelay = connection.MaxBatchDelay connection.DB = db + if connection.Compact { + log.Info().Msg("compacting database") + if err := connection.compact(); err != nil { + log.Error().Err(err).Msg("failed to compact database") + } else { + log.Info().Msg("database compaction completed") + } + } + return nil } @@ -414,3 +422,42 @@ func (connection *DbConnection) RestoreMetadata(s map[string]any) error { return err } + +// compact attempts to compact the database and replace it iff it succeeds +func (connection *DbConnection) compact() error { + compactedPath := connection.GetDatabaseFilePath() + compactedSuffix + compactedDB, err := bolt.Open(compactedPath, 0o600, connection.boltOptions()) + if err != nil { + return fmt.Errorf("failure to create the compacted database: %w", err) + } + + compactedDB.MaxBatchSize = connection.MaxBatchSize + compactedDB.MaxBatchDelay = connection.MaxBatchDelay + + if err := bolt.Compact(compactedDB, connection.DB, txMaxSize); err != nil { + return fmt.Errorf("failure to compact the database: %w", + errors.Join(err, compactedDB.Close(), os.Remove(compactedPath))) + } + + if err := os.Rename(compactedPath, connection.GetDatabaseFilePath()); err != nil { + return fmt.Errorf("failure to move the compacted database: %w", + errors.Join(err, compactedDB.Close(), os.Remove(compactedPath))) + } + + if err := connection.Close(); err != nil { + log.Warn().Err(err).Msg("failure to close the database after compaction") + } + + connection.DB = compactedDB + + return nil +} + +func (connection *DbConnection) boltOptions() *bolt.Options { + return &bolt.Options{ + Timeout: 1 * time.Second, + InitialMmapSize: connection.InitialMmapSize, + FreelistType: bolt.FreelistMapType, + NoFreelistSync: true, + } +} diff --git a/api/database/boltdb/db_test.go b/api/database/boltdb/db_test.go index 6e4639c68..1aba08fb8 100644 --- a/api/database/boltdb/db_test.go +++ b/api/database/boltdb/db_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" ) func Test_NeedsEncryptionMigration(t *testing.T) { @@ -119,3 +121,57 @@ func Test_NeedsEncryptionMigration(t *testing.T) { }) } } + +func TestDBCompaction(t *testing.T) { + db := &DbConnection{ + Path: t.TempDir(), + Compact: true, + } + + err := db.Open() + require.NoError(t, err) + + err = db.Update(func(tx *bbolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("testbucket")) + if err != nil { + return err + } + + b.Put([]byte("key"), []byte("value")) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + // Reopen the DB to trigger compaction + err = db.Open() + require.NoError(t, err) + + // Check that the data is still there + err = db.View(func(tx *bbolt.Tx) error { + b := tx.Bucket([]byte("testbucket")) + if b == nil { + return nil + } + + val := b.Get([]byte("key")) + require.Equal(t, []byte("value"), val) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + // Failures + + err = os.Mkdir(db.GetDatabaseFilePath()+compactedSuffix, 0o755) + require.NoError(t, err) + + err = db.Open() + require.NoError(t, err) +} diff --git a/api/database/database.go b/api/database/database.go index 492d56bcf..c3c3cc267 100644 --- a/api/database/database.go +++ b/api/database/database.go @@ -8,11 +8,12 @@ import ( ) // NewDatabase should use config options to return a connection to the requested database -func NewDatabase(storeType, storePath string, encryptionKey []byte) (connection portainer.Connection, err error) { +func NewDatabase(storeType, storePath string, encryptionKey []byte, compact bool) (connection portainer.Connection, err error) { if storeType == "boltdb" { return &boltdb.DbConnection{ Path: storePath, EncryptionKey: encryptionKey, + Compact: compact, }, nil } diff --git a/api/datastore/teststore.go b/api/datastore/teststore.go index 0945195b0..5d9573a61 100644 --- a/api/datastore/teststore.go +++ b/api/datastore/teststore.go @@ -44,7 +44,7 @@ func NewTestStore(t testing.TB, init, secure bool) (bool, *Store, func(), error) secretKey = nil } - connection, err := database.NewDatabase("boltdb", storePath, secretKey) + connection, err := database.NewDatabase("boltdb", storePath, secretKey, false) if err != nil { panic(err) } diff --git a/api/internal/testhelpers/datastore.go b/api/internal/testhelpers/datastore.go index d89b07531..0dc543cf9 100644 --- a/api/internal/testhelpers/datastore.go +++ b/api/internal/testhelpers/datastore.go @@ -113,7 +113,7 @@ type datastoreOption = func(d *testDatastore) // NewDatastore creates new instance of testDatastore. // Will apply options before returning, opts will be applied from left to right. func NewDatastore(options ...datastoreOption) *testDatastore { - conn, _ := database.NewDatabase("boltdb", "", nil) + conn, _ := database.NewDatabase("boltdb", "", nil, false) d := testDatastore{connection: conn} for _, o := range options { diff --git a/api/portainer.go b/api/portainer.go index c190465c9..f79ec976c 100644 --- a/api/portainer.go +++ b/api/portainer.go @@ -112,6 +112,7 @@ type ( AdminPasswordFile *string Assets *string CSP *bool + CompactDB *bool Data *string FeatureFlags *[]string EnableEdgeComputeFeatures *bool @@ -1846,6 +1847,8 @@ const ( TrustedOriginsEnvVar = "TRUSTED_ORIGINS" // CSPEnvVar is the environment variable used to enable/disable the Content Security Policy CSPEnvVar = "CSP" + // CompactDBEnvVar is the environment variable used to enable/disable the startup compaction of the database + CompactDBEnvVar = "COMPACT_DB" ) // List of supported features