feat: 监控数据使用独立数据库 (#4424)

backend/app/api/v1/monitor.go

@@ -28,7 +28,7 @@ func (b *BaseApi) LoadMonitor(c *gin.Context) {
 	var backdatas []dto.MonitorData
 	if req.Param == "all" || req.Param == "cpu" || req.Param == "memory" || req.Param == "load" {
 		var bases []model.MonitorBase
-		if err := global.DB.
+		if err := global.MonitorDB.
 			Where("created_at > ? AND created_at < ?", req.StartTime, req.EndTime).
 			Find(&bases).Error; err != nil {
 			helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
@@ -45,7 +45,7 @@ func (b *BaseApi) LoadMonitor(c *gin.Context) {
 	}
 	if req.Param == "all" || req.Param == "io" {
 		var bases []model.MonitorIO
-		if err := global.DB.
+		if err := global.MonitorDB.
 			Where("created_at > ? AND created_at < ?", req.StartTime, req.EndTime).
 			Find(&bases).Error; err != nil {
 			helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
@@ -62,7 +62,7 @@ func (b *BaseApi) LoadMonitor(c *gin.Context) {
 	}
 	if req.Param == "all" || req.Param == "network" {
 		var bases []model.MonitorNetwork
-		if err := global.DB.
+		if err := global.MonitorDB.
 			Where("name = ? AND created_at > ? AND created_at < ?", req.Info, req.StartTime, req.EndTime).
 			Find(&bases).Error; err != nil {
 			helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)

backend/app/repo/setting.go

@@ -68,20 +68,20 @@ func (u *SettingRepo) Update(key, value string) error {
 }
 
 func (u *SettingRepo) CreateMonitorBase(model model.MonitorBase) error {
-	return global.DB.Create(&model).Error
+	return global.MonitorDB.Create(&model).Error
 }
 func (u *SettingRepo) BatchCreateMonitorIO(ioList []model.MonitorIO) error {
-	return global.DB.CreateInBatches(ioList, len(ioList)).Error
+	return global.MonitorDB.CreateInBatches(ioList, len(ioList)).Error
 }
 func (u *SettingRepo) BatchCreateMonitorNet(ioList []model.MonitorNetwork) error {
-	return global.DB.CreateInBatches(ioList, len(ioList)).Error
+	return global.MonitorDB.CreateInBatches(ioList, len(ioList)).Error
 }
 func (u *SettingRepo) DelMonitorBase(timeForDelete time.Time) error {
-	return global.DB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorBase{}).Error
+	return global.MonitorDB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorBase{}).Error
 }
 func (u *SettingRepo) DelMonitorIO(timeForDelete time.Time) error {
-	return global.DB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorIO{}).Error
+	return global.MonitorDB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorIO{}).Error
 }
 func (u *SettingRepo) DelMonitorNet(timeForDelete time.Time) error {
-	return global.DB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorNetwork{}).Error
+	return global.MonitorDB.Where("created_at < ?", timeForDelete).Delete(&model.MonitorNetwork{}).Error
 }

backend/global/global.go

@@ -13,13 +13,14 @@ import (
 )
 
 var (
-	DB      *gorm.DB
+	DB        *gorm.DB
-	LOG     *logrus.Logger
+	MonitorDB *gorm.DB
-	CONF    configs.ServerConfig
+	LOG       *logrus.Logger
-	VALID   *validator.Validate
+	CONF      configs.ServerConfig
-	SESSION *psession.PSession
+	VALID     *validator.Validate
-	CACHE   *badger_db.Cache
+	SESSION   *psession.PSession
-	Viper   *viper.Viper
+	CACHE     *badger_db.Cache
+	Viper     *viper.Viper
 
 	Cron           *cron.Cron
 	MonitorCronID  cron.EntryID

backend/init/db/db.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"time"
 
 	"github.com/1Panel-dev/1Panel/backend/global"
@@ -34,6 +35,7 @@ func Init() {
 			Colorful:                  false,
 		},
 	)
+	initMonitorDB(newLogger)
 
 	db, err := gorm.Open(sqlite.Open(fullPath), &gorm.Config{
 		DisableForeignKeyConstraintWhenMigrating: true,
@@ -45,7 +47,7 @@ func Init() {
 	_ = db.Exec("PRAGMA journal_mode = WAL;")
 	sqlDB, dbError := db.DB()
 	if dbError != nil {
-		panic(err)
+		panic(dbError)
 	}
 	sqlDB.SetConnMaxIdleTime(10)
 	sqlDB.SetMaxOpenConns(100)
@@ -54,3 +56,35 @@ func Init() {
 	global.DB = db
 	global.LOG.Info("init db successfully")
 }
+
+func initMonitorDB(newLogger logger.Interface) {
+	if _, err := os.Stat(global.CONF.System.DbPath); err != nil {
+		if err := os.MkdirAll(global.CONF.System.DbPath, os.ModePerm); err != nil {
+			panic(fmt.Errorf("init db dir failed, err: %v", err))
+		}
+	}
+	fullPath := path.Join(global.CONF.System.DbPath, "monitor.db")
+	if _, err := os.Stat(fullPath); err != nil {
+		if _, err := os.Create(fullPath); err != nil {
+			panic(fmt.Errorf("init db file failed, err: %v", err))
+		}
+	}
+
+	db, err := gorm.Open(sqlite.Open(fullPath), &gorm.Config{
+		DisableForeignKeyConstraintWhenMigrating: true,
+		Logger:                                   newLogger,
+	})
+	if err != nil {
+		panic(err)
+	}
+	sqlDB, dbError := db.DB()
+	if dbError != nil {
+		panic(dbError)
+	}
+	sqlDB.SetConnMaxIdleTime(10)
+	sqlDB.SetMaxOpenConns(100)
+	sqlDB.SetConnMaxLifetime(time.Hour)
+
+	global.MonitorDB = db
+	global.LOG.Info("init monitor db successfully")
+}

backend/init/migration/migrate.go

@@ -77,6 +77,7 @@ func Init() {
 		migrations.AddDatabaseIsDelete,
 		migrations.AddXpackHideMenu,
 		migrations.AddCronjobCommand,
+		migrations.NewMonitorDB,
 	})
 	if err := m.Migrate(); err != nil {
 		global.LOG.Error(err)

backend/init/migration/migrations/v_1_10.go

@@ -2,6 +2,7 @@ package migrations
 
 import (
 	"github.com/1Panel-dev/1Panel/backend/app/model"
+	"github.com/1Panel-dev/1Panel/backend/global"
 	"github.com/go-gormigrate/gormigrate/v2"
 	"gorm.io/gorm"
 )
@@ -44,4 +45,66 @@ var AddCronjobCommand = &gormigrate.Migration{
 		}
 		return nil
 	},
-}
+}
+
+var NewMonitorDB = &gormigrate.Migration{
+	ID: "20240408-new-monitor-db",
+	Migrate: func(tx *gorm.DB) error {
+		var (
+			bases    []model.MonitorBase
+			ios      []model.MonitorIO
+			networks []model.MonitorNetwork
+		)
+		if err := tx.Find(&bases).Error; err != nil {
+			return err
+		}
+		if err := tx.Find(&ios).Error; err != nil {
+			return err
+		}
+		if err := tx.Find(&networks).Error; err != nil {
+			return err
+		}
+		if err := global.MonitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorIO{}, &model.MonitorNetwork{}); err != nil {
+			return err
+		}
+
+		_ = global.MonitorDB.Exec("DELETE FROM monitor_bases").Error
+		_ = global.MonitorDB.Exec("DELETE FROM monitor_ios").Error
+		_ = global.MonitorDB.Exec("DELETE FROM monitor_networks").Error
+
+		for i := 0; i <= len(bases)/200; i++ {
+			var itemData []model.MonitorBase
+			if 200*(i+1) <= len(bases) {
+				itemData = bases[200*i : 200*(i+1)]
+			} else {
+				itemData = bases[200*i:]
+			}
+			if err := global.MonitorDB.Create(&itemData).Error; err != nil {
+				return err
+			}
+		}
+		for i := 0; i <= len(ios)/200; i++ {
+			var itemData []model.MonitorIO
+			if 200*(i+1) <= len(ios) {
+				itemData = ios[200*i : 200*(i+1)]
+			} else {
+				itemData = ios[200*i:]
+			}
+			if err := global.MonitorDB.Create(&itemData).Error; err != nil {
+				return err
+			}
+		}
+		for i := 0; i <= len(networks)/200; i++ {
+			var itemData []model.MonitorNetwork
+			if 200*(i+1) <= len(networks) {
+				itemData = networks[200*i : 200*(i+1)]
+			} else {
+				itemData = networks[200*i:]
+			}
+			if err := global.MonitorDB.Create(&itemData).Error; err != nil {
+				return err
+			}
+		}
+		return nil
+	},
+}