feat(performance): add settings to tune the performance of the database EE-2363 (#6389)

* feat(performance): add settings to tune the performance of the database EE-2363

* Change panics to log.Fatals.

Co-authored-by: andres-portainer <andres-portainer@users.noreply.github.com>

api/cli/cli.go

@@ -57,6 +57,9 @@ func (*Service) ParseFlags(version string) (*portainer.CLIFlags, error) {
 		Logo:                      kingpin.Flag("logo", "URL for the logo displayed in the UI").String(),
 		Templates:                 kingpin.Flag("templates", "URL to the templates definitions.").Short('t').String(),
 		BaseURL:                   kingpin.Flag("base-url", "Base URL parameter such as portainer if running portainer as http://yourdomain.com/portainer/.").Short('b').Default(defaultBaseURL).String(),
+		InitialMmapSize:           kingpin.Flag("initial-mmap-size", "Initial mmap size of the database in bytes").Int(),
+		MaxBatchSize:              kingpin.Flag("max-batch-size", "Maximum size of a batch").Int(),
+		MaxBatchDelay:             kingpin.Flag("max-batch-delay", "Maximum delay before a batch starts").Duration(),
 		SecretKeyName:             kingpin.Flag("secret-key-name", "Secret key name for encryption and will be used as /run/secrets/<secret-key-name>.").Default(defaultSecretKeyName).String(),
 	}
 

api/cmd/portainer/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/portainer/portainer/api/cli"
 	"github.com/portainer/portainer/api/crypto"
 	"github.com/portainer/portainer/api/database"
+	"github.com/portainer/portainer/api/database/boltdb"
 	"github.com/portainer/portainer/api/dataservices"
 	"github.com/portainer/portainer/api/datastore"
 	"github.com/portainer/portainer/api/docker"
@@ -69,8 +70,17 @@ func initFileService(dataStorePath string) portainer.FileService {
 func initDataStore(flags *portainer.CLIFlags, secretKey []byte, fileService portainer.FileService, shutdownCtx context.Context) dataservices.DataStore {
 	connection, err := database.NewDatabase("boltdb", *flags.Data, secretKey)
 	if err != nil {
-		panic(err.Error())
+		log.Fatalf("failed creating database connection: %s", err)
 	}
+
+	if bconn, ok := connection.(*boltdb.DbConnection); ok {
+		bconn.MaxBatchSize = *flags.MaxBatchSize
+		bconn.MaxBatchDelay = *flags.MaxBatchDelay
+		bconn.InitialMmapSize = *flags.InitialMmapSize
+	} else {
+		log.Fatalf("failed creating database connection: expecting a boltdb database type but a different one was received")
+	}
+
 	store := datastore.NewStore(*flags.Data, fileService, connection)
 	isNew, err := store.Open()
 	if err != nil {

api/database/boltdb/db.go

@@ -20,9 +20,12 @@ const (
 )
 
 type DbConnection struct {
-	Path          string
-	EncryptionKey []byte
-	isEncrypted   bool
+	Path            string
+	MaxBatchSize    int
+	MaxBatchDelay   time.Duration
+	InitialMmapSize int
+	EncryptionKey   []byte
+	isEncrypted     bool
 
 	*bolt.DB
 }
@@ -83,10 +86,15 @@ func (connection *DbConnection) Open() error {
 
 	// Now we open the db
 	databasePath := connection.GetDatabaseFilePath()
-	db, err := bolt.Open(databasePath, 0600, &bolt.Options{Timeout: 1 * time.Second})
+	db, err := bolt.Open(databasePath, 0600, &bolt.Options{
+		Timeout:         1 * time.Second,
+		InitialMmapSize: connection.InitialMmapSize,
+	})
 	if err != nil {
 		return err
 	}
+	db.MaxBatchSize = connection.MaxBatchSize
+	db.MaxBatchDelay = connection.MaxBatchDelay
 	connection.DB = db
 	return nil
 }
@@ -133,7 +141,7 @@ func (connection *DbConnection) ConvertToKey(v int) []byte {
 
 // CreateBucket is a generic function used to create a bucket inside a database database.
 func (connection *DbConnection) SetServiceName(bucketName string) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		_, err := tx.CreateBucketIfNotExists([]byte(bucketName))
 		if err != nil {
 			return err
@@ -163,7 +171,7 @@ func (connection *DbConnection) GetObject(bucketName string, key []byte, object
 		return err
 	}
 
-	return connection.UnmarshalObject(data, object)
+	return connection.UnmarshalObjectWithJsoniter(data, object)
 }
 
 func (connection *DbConnection) getEncryptionKey() []byte {
@@ -176,26 +184,20 @@ func (connection *DbConnection) getEncryptionKey() []byte {
 
 // UpdateObject is a generic function used to update an object inside a database database.
 func (connection *DbConnection) UpdateObject(bucketName string, key []byte, object interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	data, err := connection.MarshalObject(object)
+	if err != nil {
+		return err
+	}
+
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
-
-		data, err := connection.MarshalObject(object)
-		if err != nil {
-			return err
-		}
-
-		err = bucket.Put(key, data)
-		if err != nil {
-			return err
-		}
-
-		return nil
+		return bucket.Put(key, data)
 	})
 }
 
 // DeleteObject is a generic function used to delete an object inside a database database.
 func (connection *DbConnection) DeleteObject(bucketName string, key []byte) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		return bucket.Delete(key)
 	})
@@ -204,7 +206,7 @@ func (connection *DbConnection) DeleteObject(bucketName string, key []byte) erro
 // DeleteAllObjects delete all objects where matching() returns (id, ok).
 // TODO: think about how to return the error inside (maybe change ok to type err, and use "notfound"?
 func (connection *DbConnection) DeleteAllObjects(bucketName string, matching func(o interface{}) (id int, ok bool)) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		cursor := bucket.Cursor()
@@ -231,7 +233,7 @@ func (connection *DbConnection) DeleteAllObjects(bucketName string, matching fun
 func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 	var identifier int
 
-	connection.Update(func(tx *bolt.Tx) error {
+	connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		id, err := bucket.NextSequence()
 		if err != nil {
@@ -246,7 +248,7 @@ func (connection *DbConnection) GetNextIdentifier(bucketName string) int {
 
 // CreateObject creates a new object in the bucket, using the next bucket sequence id
 func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64) (int, interface{})) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		seqId, _ := bucket.NextSequence()
@@ -263,7 +265,7 @@ func (connection *DbConnection) CreateObject(bucketName string, fn func(uint64)
 
 // CreateObjectWithId creates a new object in the bucket, using the specified id
 func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, obj interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 		data, err := connection.MarshalObject(obj)
 		if err != nil {
@@ -277,7 +279,7 @@ func (connection *DbConnection) CreateObjectWithId(bucketName string, id int, ob
 // CreateObjectWithSetSequence creates a new object in the bucket, using the specified id, and sets the bucket sequence
 // avoid this :)
 func (connection *DbConnection) CreateObjectWithSetSequence(bucketName string, id int, obj interface{}) error {
-	return connection.Update(func(tx *bolt.Tx) error {
+	return connection.Batch(func(tx *bolt.Tx) error {
 		bucket := tx.Bucket([]byte(bucketName))
 
 		// We manually manage sequences for schedules

api/database/boltdb/json.go

@@ -66,7 +66,17 @@ func (connection *DbConnection) UnmarshalObjectWithJsoniter(data []byte, object
 		}
 	}
 	var jsoni = jsoniter.ConfigCompatibleWithStandardLibrary
-	return jsoni.Unmarshal(data, &object)
+	err := jsoni.Unmarshal(data, &object)
+	if err != nil {
+		if s, ok := object.(*string); ok {
+			*s = string(data)
+			return nil
+		}
+
+		return err
+	}
+
+	return nil
 }
 
 // mmm, don't have a KMS .... aes GCM seems the most likely from

api/http/handler/endpoints/endpoint_status_inspect.go

@@ -103,6 +103,15 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 		}
 	}
 
+	if endpoint.EdgeCheckinInterval == 0 {
+		settings, err := handler.DataStore.Settings().Settings()
+		if err != nil {
+			return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
+		}
+
+		endpoint.EdgeCheckinInterval = settings.EdgeAgentCheckinInterval
+	}
+
 	endpoint.LastCheckInDate = time.Now().Unix()
 
 	err = handler.DataStore.Endpoint().UpdateEndpoint(endpoint.ID, endpoint)
@@ -110,18 +119,8 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to Unable to persist environment changes inside the database", err}
 	}
 
-	settings, err := handler.DataStore.Settings().Settings()
-	if err != nil {
-		return &httperror.HandlerError{http.StatusInternalServerError, "Unable to retrieve settings from the database", err}
-	}
-
 	tunnel := handler.ReverseTunnelService.GetTunnelDetails(endpoint.ID)
 
-	checkinInterval := settings.EdgeAgentCheckinInterval
-	if endpoint.EdgeCheckinInterval != 0 {
-		checkinInterval = endpoint.EdgeCheckinInterval
-	}
-
 	schedules := []edgeJobResponse{}
 	for _, job := range tunnel.Jobs {
 		schedule := edgeJobResponse{
@@ -146,7 +145,7 @@ func (handler *Handler) endpointStatusInspect(w http.ResponseWriter, r *http.Req
 		Status:          tunnel.Status,
 		Port:            tunnel.Port,
 		Schedules:       schedules,
-		CheckinInterval: checkinInterval,
+		CheckinInterval: endpoint.EdgeCheckinInterval,
 		Credentials:     tunnel.Credentials,
 	}
 

api/portainer.go

@@ -97,6 +97,9 @@ type (
 		Rollback                  *bool
 		SnapshotInterval          *string
 		BaseURL                   *string
+		InitialMmapSize           *int
+		MaxBatchSize              *int
+		MaxBatchDelay             *time.Duration
 		SecretKeyName             *string
 	}