From edd6b52f05e570629b23892ea3a82e0a920bac06 Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:41:56 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=AE=A1=E5=88=92=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=87=E4=BB=BD=E5=88=B0=E5=A4=9A=E4=B8=AA?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E8=B4=A6=E5=8F=B7=20(#3689)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/backup.go | 38 +- backend/app/dto/backup.go | 5 + backend/app/dto/cronjob.go | 75 +-- backend/app/dto/setting.go | 2 +- backend/app/model/backup.go | 2 + backend/app/model/cronjob.go | 7 +- backend/app/repo/backup.go | 8 + backend/app/repo/cronjob.go | 5 + backend/app/service/backup.go | 47 +- backend/app/service/backup_app.go | 6 +- backend/app/service/backup_mysql.go | 8 +- backend/app/service/backup_postgresql.go | 5 +- backend/app/service/backup_redis.go | 5 +- backend/app/service/backup_website.go | 5 +- backend/app/service/cornjob.go | 47 +- backend/app/service/cronjob_backup.go | 386 +++++++++++ backend/app/service/cronjob_helper.go | 617 ++++-------------- backend/app/service/snapshot.go | 39 +- backend/app/service/snapshot_create.go | 53 +- backend/constant/backup.go | 1 + backend/init/migration/migrate.go | 1 + backend/init/migration/migrations/v_1_9.go | 152 +++++ backend/router/ro_setting.go | 1 + backend/utils/cloud_storage/client/local.go | 69 ++ backend/utils/cloud_storage/client/sftp.go | 11 + .../cloud_storage/cloud_storage_client.go | 2 + backend/utils/cmd/cmd.go | 25 +- cmd/server/docs/docs.go | 76 ++- cmd/server/docs/swagger.json | 76 ++- cmd/server/docs/swagger.yaml | 50 +- frontend/src/api/interface/backup.ts | 3 + frontend/src/api/interface/cronjob.ts | 8 +- frontend/src/api/modules/setting.ts | 3 + frontend/src/lang/modules/en.ts | 1 + frontend/src/lang/modules/tw.ts | 1 + frontend/src/lang/modules/zh.ts | 1 + frontend/src/views/cronjob/backup/index.vue | 127 ++++ frontend/src/views/cronjob/helper.ts | 46 ++ frontend/src/views/cronjob/index.vue | 54 +- frontend/src/views/cronjob/operate/index.vue | 75 ++- frontend/src/views/cronjob/record/index.vue | 341 +--------- .../setting/backup-account/onedrive/index.vue | 2 +- 42 files changed, 1469 insertions(+), 1017 deletions(-) create mode 100644 backend/app/service/cronjob_backup.go create mode 100644 backend/utils/cloud_storage/client/local.go create mode 100644 frontend/src/views/cronjob/backup/index.vue diff --git a/backend/app/api/v1/backup.go b/backend/app/api/v1/backup.go index a0abfd476..9f3b82572 100644 --- a/backend/app/api/v1/backup.go +++ b/backend/app/api/v1/backup.go @@ -162,6 +162,32 @@ func (b *BaseApi) SearchBackupRecords(c *gin.Context) { }) } +// @Tags Backup Account +// @Summary Page backup records by cronjob +// @Description 通过计划任务获取备份记录列表分页 +// @Accept json +// @Param request body dto.RecordSearchByCronjob true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/backup/record/search/bycronjob [post] +func (b *BaseApi) SearchBackupRecordsByCronjob(c *gin.Context) { + var req dto.RecordSearchByCronjob + if err := helper.CheckBindAndValidate(&req, c); err != nil { + return + } + + total, list, err := backupService.SearchRecordsByCronjobWithPage(req) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + + helper.SuccessWithData(c, dto.PageResult{ + Items: list, + Total: total, + }) +} + // @Tags Backup Account // @Summary Download backup record // @Description 下载备份记录 @@ -345,14 +371,12 @@ func (b *BaseApi) Recover(c *gin.Context) { return } - if req.Source != "LOCAL" { - downloadPath, err := backupService.DownloadRecord(dto.DownloadRecord{Source: req.Source, FileDir: path.Dir(req.File), FileName: path.Base(req.File)}) - if err != nil { - helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, fmt.Errorf("download file failed, err: %v", err)) - return - } - req.File = downloadPath + downloadPath, err := backupService.DownloadRecord(dto.DownloadRecord{Source: req.Source, FileDir: path.Dir(req.File), FileName: path.Base(req.File)}) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, fmt.Errorf("download file failed, err: %v", err)) + return } + req.File = downloadPath switch req.Type { case "mysql", "mariadb": if err := backupService.MysqlRecover(req); err != nil { diff --git a/backend/app/dto/backup.go b/backend/app/dto/backup.go index cfe612f4b..f5ab3ebd4 100644 --- a/backend/app/dto/backup.go +++ b/backend/app/dto/backup.go @@ -51,6 +51,11 @@ type RecordSearch struct { DetailName string `json:"detailName"` } +type RecordSearchByCronjob struct { + PageInfo + CronjobID uint `json:"cronjobID" validate:"required"` +} + type BackupRecords struct { ID uint `json:"id"` CreatedAt time.Time `json:"createdAt"` diff --git a/backend/app/dto/cronjob.go b/backend/app/dto/cronjob.go index 6ae136a2c..7f2003bc9 100644 --- a/backend/app/dto/cronjob.go +++ b/backend/app/dto/cronjob.go @@ -7,18 +7,18 @@ type CronjobCreate struct { Type string `json:"type" validate:"required"` Spec string `json:"spec" validate:"required"` - Script string `json:"script"` - ContainerName string `json:"containerName"` - AppID string `json:"appID"` - Website string `json:"website"` - ExclusionRules string `json:"exclusionRules"` - DBType string `json:"dbType"` - DBName string `json:"dbName"` - URL string `json:"url"` - SourceDir string `json:"sourceDir"` - KeepLocal bool `json:"keepLocal"` - TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies" validate:"number,min=1"` + Script string `json:"script"` + ContainerName string `json:"containerName"` + AppID string `json:"appID"` + Website string `json:"website"` + ExclusionRules string `json:"exclusionRules"` + DBType string `json:"dbType"` + DBName string `json:"dbName"` + URL string `json:"url"` + SourceDir string `json:"sourceDir"` + TargetDirID int `json:"targetDirID"` + TargetAccountIDs string `json:"targetAccountIDs"` + RetainCopies int `json:"retainCopies" validate:"number,min=1"` } type CronjobUpdate struct { @@ -26,18 +26,18 @@ type CronjobUpdate struct { Name string `json:"name" validate:"required"` Spec string `json:"spec" validate:"required"` - Script string `json:"script"` - ContainerName string `json:"containerName"` - AppID string `json:"appID"` - Website string `json:"website"` - ExclusionRules string `json:"exclusionRules"` - DBType string `json:"dbType"` - DBName string `json:"dbName"` - URL string `json:"url"` - SourceDir string `json:"sourceDir"` - KeepLocal bool `json:"keepLocal"` - TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies" validate:"number,min=1"` + Script string `json:"script"` + ContainerName string `json:"containerName"` + AppID string `json:"appID"` + Website string `json:"website"` + ExclusionRules string `json:"exclusionRules"` + DBType string `json:"dbType"` + DBName string `json:"dbName"` + URL string `json:"url"` + SourceDir string `json:"sourceDir"` + TargetDirID int `json:"targetDirID"` + TargetAccountIDs string `json:"targetAccountIDs"` + RetainCopies int `json:"retainCopies" validate:"number,min=1"` } type CronjobUpdateStatus struct { @@ -66,19 +66,20 @@ type CronjobInfo struct { Type string `json:"type"` Spec string `json:"spec"` - Script string `json:"script"` - ContainerName string `json:"containerName"` - AppID string `json:"appID"` - Website string `json:"website"` - ExclusionRules string `json:"exclusionRules"` - DBType string `json:"dbType"` - DBName string `json:"dbName"` - URL string `json:"url"` - SourceDir string `json:"sourceDir"` - KeepLocal bool `json:"keepLocal"` - TargetDir string `json:"targetDir"` - TargetDirID int `json:"targetDirID"` - RetainCopies int `json:"retainCopies"` + Script string `json:"script"` + ContainerName string `json:"containerName"` + AppID string `json:"appID"` + Website string `json:"website"` + ExclusionRules string `json:"exclusionRules"` + DBType string `json:"dbType"` + DBName string `json:"dbName"` + URL string `json:"url"` + SourceDir string `json:"sourceDir"` + TargetDir string `json:"targetDir"` + TargetDirID int `json:"targetDirID"` + TargetAccounts string `json:"targetAccounts"` + TargetAccountIDs string `json:"targetAccountIDs"` + RetainCopies int `json:"retainCopies"` LastRecordTime string `json:"lastRecordTime"` Status string `json:"status"` diff --git a/backend/app/dto/setting.go b/backend/app/dto/setting.go index 9895b0eec..4a0d0d3a6 100644 --- a/backend/app/dto/setting.go +++ b/backend/app/dto/setting.go @@ -100,7 +100,7 @@ type SnapshotStatus struct { type SnapshotCreate struct { ID uint `json:"id"` - From string `json:"from" validate:"required,oneof=OSS S3 SFTP MINIO COS KODO OneDrive WebDAV"` + From string `json:"from" validate:"required"` Description string `json:"description" validate:"max=256"` } type SnapshotRecover struct { diff --git a/backend/app/model/backup.go b/backend/app/model/backup.go index 7c2fdf80b..50690dcaa 100644 --- a/backend/app/model/backup.go +++ b/backend/app/model/backup.go @@ -12,6 +12,8 @@ type BackupAccount struct { type BackupRecord struct { BaseModel + From string `gorm:"type:varchar(64)" json:"from"` + CronjobID uint `gorm:"type:decimal" json:"cronjobID"` Type string `gorm:"type:varchar(64);not null" json:"type"` Name string `gorm:"type:varchar(64);not null" json:"name"` DetailName string `gorm:"type:varchar(256)" json:"detailName"` diff --git a/backend/app/model/cronjob.go b/backend/app/model/cronjob.go index fb1432666..03b787090 100644 --- a/backend/app/model/cronjob.go +++ b/backend/app/model/cronjob.go @@ -19,9 +19,10 @@ type Cronjob struct { SourceDir string `gorm:"type:varchar(256)" json:"sourceDir"` ExclusionRules string `gorm:"longtext" json:"exclusionRules"` - KeepLocal bool `gorm:"type:varchar(64)" json:"keepLocal"` - TargetDirID uint64 `gorm:"type:decimal" json:"targetDirID"` - RetainCopies uint64 `gorm:"type:decimal" json:"retainCopies"` + KeepLocal bool `gorm:"type:varchar(64)" json:"keepLocal"` + TargetDirID uint64 `gorm:"type:decimal" json:"targetDirID"` + TargetAccountIDs string `gorm:"type:varchar(64)" json:"targetAccountIDs"` + RetainCopies uint64 `gorm:"type:decimal" json:"retainCopies"` Status string `gorm:"type:varchar(64)" json:"status"` EntryIDs string `gorm:"type:varchar(64)" json:"entryIDs"` diff --git a/backend/app/repo/backup.go b/backend/app/repo/backup.go index 2e4609b51..7548ba8ba 100644 --- a/backend/app/repo/backup.go +++ b/backend/app/repo/backup.go @@ -2,6 +2,7 @@ package repo import ( "context" + "github.com/1Panel-dev/1Panel/backend/app/model" "github.com/1Panel-dev/1Panel/backend/global" "gorm.io/gorm" @@ -23,6 +24,7 @@ type IBackupRepo interface { WithByDetailName(detailName string) DBOption WithByFileName(fileName string) DBOption WithByType(backupType string) DBOption + WithByCronID(cronjobID uint) DBOption } func NewIBackupRepo() IBackupRepo { @@ -125,3 +127,9 @@ func (u *BackupRepo) Delete(opts ...DBOption) error { func (u *BackupRepo) DeleteRecord(ctx context.Context, opts ...DBOption) error { return getTx(ctx, opts...).Delete(&model.BackupRecord{}).Error } + +func (u *BackupRepo) WithByCronID(cronjobID uint) DBOption { + return func(g *gorm.DB) *gorm.DB { + return g.Where("cronjob_id = ?", cronjobID) + } +} diff --git a/backend/app/repo/cronjob.go b/backend/app/repo/cronjob.go index 115b573b8..272c8984f 100644 --- a/backend/app/repo/cronjob.go +++ b/backend/app/repo/cronjob.go @@ -28,6 +28,7 @@ type ICronjobRepo interface { Delete(opts ...DBOption) error DeleteRecord(opts ...DBOption) error StartRecords(cronjobID uint, fromLocal bool, targetPath string) model.JobRecords + UpdateRecords(id uint, vars map[string]interface{}) error EndRecords(record model.JobRecords, status, message, records string) PageRecords(page, size int, opts ...DBOption) (int64, []model.JobRecords, error) } @@ -164,6 +165,10 @@ func (u *CronjobRepo) Update(id uint, vars map[string]interface{}) error { return global.DB.Model(&model.Cronjob{}).Where("id = ?", id).Updates(vars).Error } +func (u *CronjobRepo) UpdateRecords(id uint, vars map[string]interface{}) error { + return global.DB.Model(&model.JobRecords{}).Where("id = ?", id).Updates(vars).Error +} + func (u *CronjobRepo) Delete(opts ...DBOption) error { db := global.DB for _, opt := range opts { diff --git a/backend/app/service/backup.go b/backend/app/service/backup.go index a3b85e232..301a62cf0 100644 --- a/backend/app/service/backup.go +++ b/backend/app/service/backup.go @@ -28,6 +28,7 @@ type BackupService struct{} type IBackupService interface { List() ([]dto.BackupInfo, error) SearchRecordsWithPage(search dto.RecordSearch) (int64, []dto.BackupRecords, error) + SearchRecordsByCronjobWithPage(search dto.RecordSearchByCronjob) (int64, []dto.BackupRecords, error) LoadOneDriveInfo() (dto.OneDriveInfo, error) DownloadRecord(info dto.DownloadRecord) (string, error) Create(backupDto dto.BackupOperate) error @@ -94,14 +95,43 @@ func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, [ return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) } itemPath := path.Join(records[i].FileDir, records[i].FileName) - if records[i].Source == "LOCAL" { - fileInfo, err := os.Stat(itemPath) - if err == nil { - item.Size = fileInfo.Size() + if _, ok := clientMap[records[i].Source]; !ok { + backup, err := backupRepo.Get(commonRepo.WithByType(records[i].Source)) + if err != nil { + global.LOG.Errorf("load backup model %s from db failed, err: %v", records[i].Source, err) + return total, datas, err } + client, err := u.NewClient(&backup) + if err != nil { + global.LOG.Errorf("load backup client %s from db failed, err: %v", records[i].Source, err) + return total, datas, err + } + item.Size, _ = client.Size(path.Join(strings.TrimLeft(backup.BackupPath, "/"), itemPath)) datas = append(datas, item) + clientMap[records[i].Source] = loadSizeHelper{backupPath: strings.TrimLeft(backup.BackupPath, "/"), client: client} continue } + item.Size, _ = clientMap[records[i].Source].client.Size(path.Join(clientMap[records[i].Source].backupPath, itemPath)) + datas = append(datas, item) + } + return total, datas, err +} + +func (u *BackupService) SearchRecordsByCronjobWithPage(search dto.RecordSearchByCronjob) (int64, []dto.BackupRecords, error) { + total, records, err := backupRepo.PageRecord( + search.Page, search.PageSize, + commonRepo.WithOrderBy("created_at desc"), + backupRepo.WithByCronID(search.CronjobID), + ) + + var datas []dto.BackupRecords + clientMap := make(map[string]loadSizeHelper) + for i := 0; i < len(records); i++ { + var item dto.BackupRecords + if err := copier.Copy(&item, &records[i]); err != nil { + return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) + } + itemPath := path.Join(records[i].FileDir, records[i].FileName) if _, ok := clientMap[records[i].Source]; !ok { backup, err := backupRepo.Get(commonRepo.WithByType(records[i].Source)) if err != nil { @@ -156,7 +186,11 @@ func (u *BackupService) LoadOneDriveInfo() (dto.OneDriveInfo, error) { func (u *BackupService) DownloadRecord(info dto.DownloadRecord) (string, error) { if info.Source == "LOCAL" { - return info.FileDir + "/" + info.FileName, nil + localDir, err := loadLocalDir() + if err != nil { + return "", err + } + return path.Join(localDir, info.FileDir, info.FileName), nil } backup, _ := backupRepo.Get(commonRepo.WithByType(info.Source)) if backup.ID == 0 { @@ -381,9 +415,6 @@ func (u *BackupService) NewClient(backup *model.BackupAccount) (cloud_storage.Cl if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil { return nil, err } - if backup.Type == "LOCAL" { - return nil, errors.New("not support") - } varMap["bucket"] = backup.Bucket switch backup.Type { case constant.Sftp, constant.WebDAV: diff --git a/backend/app/service/backup_app.go b/backend/app/service/backup_app.go index 21458e897..5f2073643 100644 --- a/backend/app/service/backup_app.go +++ b/backend/app/service/backup_app.go @@ -35,8 +35,8 @@ func (u *BackupService) AppBackup(req dto.CommonBackup) error { return err } timeNow := time.Now().Format("20060102150405") - - backupDir := path.Join(localDir, fmt.Sprintf("app/%s/%s", req.Name, req.DetailName)) + itemDir := fmt.Sprintf("app/%s/%s", req.Name, req.DetailName) + backupDir := path.Join(localDir, itemDir) fileName := fmt.Sprintf("%s_%s.tar.gz", req.DetailName, timeNow) if err := handleAppBackup(&install, backupDir, fileName); err != nil { @@ -49,7 +49,7 @@ func (u *BackupService) AppBackup(req dto.CommonBackup) error { DetailName: req.DetailName, Source: "LOCAL", BackupType: "LOCAL", - FileDir: backupDir, + FileDir: itemDir, FileName: fileName, } diff --git a/backend/app/service/backup_mysql.go b/backend/app/service/backup_mysql.go index 4eb30915b..30285a0d7 100644 --- a/backend/app/service/backup_mysql.go +++ b/backend/app/service/backup_mysql.go @@ -2,13 +2,14 @@ package service import ( "fmt" - "github.com/1Panel-dev/1Panel/backend/buserr" "os" "path" "path/filepath" "strings" "time" + "github.com/1Panel-dev/1Panel/backend/buserr" + "github.com/1Panel-dev/1Panel/backend/app/dto" "github.com/1Panel-dev/1Panel/backend/app/model" "github.com/1Panel-dev/1Panel/backend/global" @@ -23,7 +24,8 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error { } timeNow := time.Now().Format("20060102150405") - targetDir := path.Join(localDir, fmt.Sprintf("database/%s/%s/%s", req.Type, req.Name, req.DetailName)) + itemDir := fmt.Sprintf("database/%s/%s/%s", req.Type, req.Name, req.DetailName) + targetDir := path.Join(localDir, itemDir) fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow) if err := handleMysqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil { @@ -36,7 +38,7 @@ func (u *BackupService) MysqlBackup(req dto.CommonBackup) error { DetailName: req.DetailName, Source: "LOCAL", BackupType: "LOCAL", - FileDir: targetDir, + FileDir: itemDir, FileName: fileName, } if err := backupRepo.CreateRecord(record); err != nil { diff --git a/backend/app/service/backup_postgresql.go b/backend/app/service/backup_postgresql.go index 07a50647d..836b8a35f 100644 --- a/backend/app/service/backup_postgresql.go +++ b/backend/app/service/backup_postgresql.go @@ -25,7 +25,8 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error { } timeNow := time.Now().Format("20060102150405") - targetDir := path.Join(localDir, fmt.Sprintf("database/%s/%s/%s", req.Type, req.Name, req.DetailName)) + itemDir := fmt.Sprintf("database/%s/%s/%s", req.Type, req.Name, req.DetailName) + targetDir := path.Join(localDir, itemDir) fileName := fmt.Sprintf("%s_%s.sql.gz", req.DetailName, timeNow) if err := handlePostgresqlBackup(req.Name, req.DetailName, targetDir, fileName); err != nil { @@ -38,7 +39,7 @@ func (u *BackupService) PostgresqlBackup(req dto.CommonBackup) error { DetailName: req.DetailName, Source: "LOCAL", BackupType: "LOCAL", - FileDir: targetDir, + FileDir: itemDir, FileName: fileName, } if err := backupRepo.CreateRecord(record); err != nil { diff --git a/backend/app/service/backup_redis.go b/backend/app/service/backup_redis.go index 45336bc02..9783d685b 100644 --- a/backend/app/service/backup_redis.go +++ b/backend/app/service/backup_redis.go @@ -43,7 +43,8 @@ func (u *BackupService) RedisBackup() error { fileName = fmt.Sprintf("%s.tar.gz", timeNow) } } - backupDir := path.Join(localDir, fmt.Sprintf("database/redis/%s", redisInfo.Name)) + itemDir := fmt.Sprintf("database/redis/%s", redisInfo.Name) + backupDir := path.Join(localDir, itemDir) if err := handleRedisBackup(redisInfo, backupDir, fileName); err != nil { return err } @@ -51,7 +52,7 @@ func (u *BackupService) RedisBackup() error { Type: "redis", Source: "LOCAL", BackupType: "LOCAL", - FileDir: backupDir, + FileDir: itemDir, FileName: fileName, } if err := backupRepo.CreateRecord(record); err != nil { diff --git a/backend/app/service/backup_website.go b/backend/app/service/backup_website.go index 8cbf5e5bf..7ec91e8a7 100644 --- a/backend/app/service/backup_website.go +++ b/backend/app/service/backup_website.go @@ -31,7 +31,8 @@ func (u *BackupService) WebsiteBackup(req dto.CommonBackup) error { } timeNow := time.Now().Format("20060102150405") - backupDir := path.Join(localDir, fmt.Sprintf("website/%s", req.Name)) + itemDir := fmt.Sprintf("website/%s", req.Name) + backupDir := path.Join(localDir, itemDir) fileName := fmt.Sprintf("%s_%s.tar.gz", website.PrimaryDomain, timeNow) if err := handleWebsiteBackup(&website, backupDir, fileName); err != nil { return err @@ -43,7 +44,7 @@ func (u *BackupService) WebsiteBackup(req dto.CommonBackup) error { DetailName: req.DetailName, Source: "LOCAL", BackupType: "LOCAL", - FileDir: backupDir, + FileDir: itemDir, FileName: fileName, } if err := backupRepo.CreateRecord(record); err != nil { diff --git a/backend/app/service/cornjob.go b/backend/app/service/cornjob.go index 12866cc36..82684873e 100644 --- a/backend/app/service/cornjob.go +++ b/backend/app/service/cornjob.go @@ -43,16 +43,29 @@ func NewICronjobService() ICronjobService { func (u *CronjobService) SearchWithPage(search dto.SearchWithPage) (int64, interface{}, error) { total, cronjobs, err := cronjobRepo.Page(search.Page, search.PageSize, commonRepo.WithLikeName(search.Info), commonRepo.WithOrderRuleBy(search.OrderBy, search.Order)) var dtoCronjobs []dto.CronjobInfo + accounts, _ := backupRepo.List() for _, cronjob := range cronjobs { var item dto.CronjobInfo if err := copier.Copy(&item, &cronjob); err != nil { return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) } if hasBackup(item.Type) { - backup, _ := backupRepo.Get(commonRepo.WithByID(uint(item.TargetDirID))) - if len(backup.Type) != 0 { - item.TargetDir = backup.Type + for _, account := range accounts { + if int(account.ID) == item.TargetDirID { + item.TargetDir = account.Type + } } + itemAccounts := strings.Split(item.TargetAccountIDs, ",") + var targetAccounts []string + for _, itemAccount := range itemAccounts { + for _, account := range accounts { + if itemAccount == fmt.Sprintf("%d", account.ID) { + targetAccounts = append(targetAccounts, account.Type) + break + } + } + } + item.TargetAccounts = strings.Join(targetAccounts, ",") } else { item.TargetDir = "-" } @@ -105,24 +118,22 @@ func (u *CronjobService) CleanRecord(req dto.CronjobClean) error { if err != nil { return err } - if req.CleanData && hasBackup(cronjob.Type) { - cronjob.RetainCopies = 0 - backup, err := backupRepo.Get(commonRepo.WithByID(uint(cronjob.TargetDirID))) - if err != nil { - return err - } - if backup.Type != "LOCAL" { - localDir, err := loadLocalDir() + if req.CleanData { + if hasBackup(cronjob.Type) { + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) if err != nil { return err } - client, err := NewIBackupService().NewClient(&backup) - if err != nil { - return err - } - u.HandleRmExpired(backup.Type, backup.BackupPath, localDir, &cronjob, client) + cronjob.RetainCopies = 0 + u.removeExpiredBackup(cronjob, accountMap, model.BackupRecord{}) } else { - u.HandleRmExpired(backup.Type, backup.BackupPath, "", &cronjob, nil) + u.removeExpiredLog(cronjob) + } + } else { + records, _ := backupRepo.ListRecord(backupRepo.WithByCronID(cronjob.ID)) + for _, records := range records { + records.CronjobID = 0 + _ = backupRepo.UpdateRecord(&records) } } delRecords, err := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(req.CronjobID))) @@ -283,8 +294,8 @@ func (u *CronjobService) Update(id uint, req dto.CronjobUpdate) error { upMap["db_name"] = req.DBName upMap["url"] = req.URL upMap["source_dir"] = req.SourceDir - upMap["keep_local"] = req.KeepLocal upMap["target_dir_id"] = req.TargetDirID + upMap["target_account_ids"] = req.TargetAccountIDs upMap["retain_copies"] = req.RetainCopies return cronjobRepo.Update(id, upMap) } diff --git a/backend/app/service/cronjob_backup.go b/backend/app/service/cronjob_backup.go new file mode 100644 index 000000000..d40f877cf --- /dev/null +++ b/backend/app/service/cronjob_backup.go @@ -0,0 +1,386 @@ +package service + +import ( + "fmt" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/1Panel-dev/1Panel/backend/app/dto" + "github.com/1Panel-dev/1Panel/backend/app/model" + "github.com/1Panel-dev/1Panel/backend/constant" + "github.com/1Panel-dev/1Panel/backend/global" +) + +func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) error { + var apps []model.AppInstall + if cronjob.AppID == "all" { + apps, _ = appInstallRepo.ListBy() + } else { + itemID, _ := (strconv.Atoi(cronjob.AppID)) + app, err := appInstallRepo.GetFirst(commonRepo.WithByID(uint(itemID))) + if err != nil { + return err + } + apps = append(apps, app) + } + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + for _, app := range apps { + var record model.BackupRecord + record.From = "cronjob" + record.Type = "app" + record.CronjobID = cronjob.ID + record.Name = app.App.Key + record.DetailName = app.Name + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name)) + record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format("20060102150405")) + if err := handleAppBackup(&app, backupDir, record.FileName); err != nil { + return err + } + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName)) + if err != nil { + return err + } + record.FileDir = path.Dir(downloadPath) + u.removeExpiredBackup(cronjob, accountMap, record) + } + return nil +} + +func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Time) error { + webs := loadWebsForJob(cronjob) + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + for _, web := range webs { + var record model.BackupRecord + record.From = "cronjob" + record.Type = "website" + record.CronjobID = cronjob.ID + record.Name = web.PrimaryDomain + record.DetailName = web.Alias + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("website/%s", web.PrimaryDomain)) + record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", web.PrimaryDomain, startTime.Format("20060102150405")) + if err := handleWebsiteBackup(&web, backupDir, record.FileName); err != nil { + return err + } + downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName)) + if err != nil { + return err + } + record.FileDir = path.Dir(downloadPath) + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + u.removeExpiredBackup(cronjob, accountMap, record) + } + return nil +} + +func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Time) error { + dbs := loadDbsForJob(cronjob) + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + for _, dbInfo := range dbs { + var record model.BackupRecord + record.From = "cronjob" + record.Type = dbInfo.DBType + record.CronjobID = cronjob.ID + record.Name = dbInfo.Database + record.DetailName = dbInfo.Name + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + + backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name)) + record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format("20060102150405")) + if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { + if err := handleMysqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { + return err + } + } else { + if err := handlePostgresqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { + return err + } + } + downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, record.FileName)) + if err != nil { + return err + } + record.FileDir = path.Dir(downloadPath) + + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + u.removeExpiredBackup(cronjob, accountMap, record) + } + return nil +} + +func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.Time) error { + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + fileName := fmt.Sprintf("directory%s_%s.tar.gz", strings.ReplaceAll(cronjob.SourceDir, "/", "_"), startTime.Format("20060102150405")) + backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("%s/%s", cronjob.Type, cronjob.Name)) + if err := handleTar(cronjob.SourceDir, backupDir, fileName, cronjob.ExclusionRules); err != nil { + return err + } + var record model.BackupRecord + record.From = "cronjob" + record.Type = "directory" + record.CronjobID = cronjob.ID + record.Name = cronjob.Name + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, fileName)) + if err != nil { + return err + } + record.FileDir = path.Dir(downloadPath) + record.FileName = fileName + + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + u.removeExpiredBackup(cronjob, accountMap, record) + return nil +} + +func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time) error { + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + fileName := fmt.Sprintf("system_log_%s.tar.gz", startTime.Format("20060102150405")) + backupDir := path.Join(global.CONF.System.TmpDir, "log", startTime.Format("20060102150405")) + if err := handleBackupLogs(backupDir, fileName); err != nil { + return err + } + var record model.BackupRecord + record.From = "cronjob" + record.Type = "log" + record.CronjobID = cronjob.ID + record.Name = cronjob.Name + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(path.Dir(backupDir), fileName)) + if err != nil { + return err + } + record.FileDir = path.Dir(downloadPath) + record.FileName = fileName + + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + u.removeExpiredBackup(cronjob, accountMap, record) + return nil +} + +func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Time, logPath string) error { + accountMap, err := u.loadClientMap(cronjob.TargetAccountIDs) + if err != nil { + return err + } + + var record model.BackupRecord + record.From = "cronjob" + record.Type = "directory" + record.CronjobID = cronjob.ID + record.Name = cronjob.Name + record.Source, record.BackupType = loadRecordPath(cronjob, accountMap) + record.FileDir = "system_snapshot" + + req := dto.SnapshotCreate{ + From: record.BackupType, + } + name, err := NewISnapshotService().HandleSnapshot(true, logPath, req, startTime.Format("20060102150405")) + if err != nil { + return err + } + record.FileName = name + ".tar.gz" + + if err := backupRepo.CreateRecord(&record); err != nil { + global.LOG.Errorf("save backup record failed, err: %v", err) + return err + } + u.removeExpiredBackup(cronjob, accountMap, record) + return nil +} + +type databaseHelper struct { + DBType string + Database string + Name string +} + +func loadDbsForJob(cronjob model.Cronjob) []databaseHelper { + var dbs []databaseHelper + if cronjob.DBName == "all" { + if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { + mysqlItems, _ := mysqlRepo.List() + for _, mysql := range mysqlItems { + dbs = append(dbs, databaseHelper{ + DBType: cronjob.DBType, + Database: mysql.MysqlName, + Name: mysql.Name, + }) + } + } else { + pgItems, _ := postgresqlRepo.List() + for _, pg := range pgItems { + dbs = append(dbs, databaseHelper{ + DBType: cronjob.DBType, + Database: pg.PostgresqlName, + Name: pg.Name, + }) + } + } + return dbs + } + itemID, _ := (strconv.Atoi(cronjob.DBName)) + if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { + mysqlItem, _ := mysqlRepo.Get(commonRepo.WithByID(uint(itemID))) + dbs = append(dbs, databaseHelper{ + DBType: cronjob.DBType, + Database: mysqlItem.MysqlName, + Name: mysqlItem.Name, + }) + } else { + pgItem, _ := postgresqlRepo.Get(commonRepo.WithByID(uint(itemID))) + dbs = append(dbs, databaseHelper{ + DBType: cronjob.DBType, + Database: pgItem.PostgresqlName, + Name: pgItem.Name, + }) + } + return dbs +} + +func loadWebsForJob(cronjob model.Cronjob) []model.Website { + var weblist []model.Website + if cronjob.Website == "all" { + weblist, _ = websiteRepo.List() + return weblist + } + itemID, _ := (strconv.Atoi(cronjob.Website)) + webItem, _ := websiteRepo.GetFirst(commonRepo.WithByID(uint(itemID))) + if webItem.ID != 0 { + weblist = append(weblist, webItem) + } + return weblist +} + +func loadRecordPath(cronjob model.Cronjob, accountMap map[string]cronjobUploadHelper) (string, string) { + source := accountMap[fmt.Sprintf("%v", cronjob.TargetDirID)].backType + targets := strings.Split(cronjob.TargetAccountIDs, ",") + var itemAccounts []string + for _, target := range targets { + if len(target) == 0 { + continue + } + if len(accountMap[target].backType) != 0 { + itemAccounts = append(itemAccounts, accountMap[target].backType) + } + } + backupType := strings.Join(itemAccounts, ",") + return source, backupType +} + +func handleBackupLogs(targetDir, fileName string) error { + websites, err := websiteRepo.List() + if err != nil { + return err + } + if len(websites) != 0 { + nginxInstall, err := getAppInstallByKey(constant.AppOpenresty) + if err != nil { + return err + } + webItem := path.Join(nginxInstall.GetPath(), "www/sites") + for _, website := range websites { + dirItem := path.Join(targetDir, "website", website.Alias) + if _, err := os.Stat(dirItem); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(dirItem, os.ModePerm); err != nil { + return err + } + } + itemDir := path.Join(webItem, website.Alias, "log") + logFiles, _ := os.ReadDir(itemDir) + if len(logFiles) != 0 { + for i := 0; i < len(logFiles); i++ { + if !logFiles[i].IsDir() { + _ = cpBinary([]string{path.Join(itemDir, logFiles[i].Name())}, dirItem) + } + } + } + itemDir2 := path.Join(global.CONF.System.Backup, "log/website", website.Alias) + logFiles2, _ := os.ReadDir(itemDir2) + if len(logFiles2) != 0 { + for i := 0; i < len(logFiles2); i++ { + if !logFiles2[i].IsDir() { + _ = cpBinary([]string{path.Join(itemDir2, logFiles2[i].Name())}, dirItem) + } + } + } + } + global.LOG.Debug("backup website log successful!") + } + + systemLogDir := path.Join(global.CONF.System.BaseDir, "1panel/log") + systemDir := path.Join(targetDir, "system") + if _, err := os.Stat(systemDir); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(systemDir, os.ModePerm); err != nil { + return err + } + } + systemLogFiles, _ := os.ReadDir(systemLogDir) + if len(systemLogFiles) != 0 { + for i := 0; i < len(systemLogFiles); i++ { + if !systemLogFiles[i].IsDir() { + _ = cpBinary([]string{path.Join(systemLogDir, systemLogFiles[i].Name())}, systemDir) + } + } + } + global.LOG.Debug("backup system log successful!") + + loginLogFiles, _ := os.ReadDir("/var/log") + loginDir := path.Join(targetDir, "login") + if _, err := os.Stat(loginDir); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(loginDir, os.ModePerm); err != nil { + return err + } + } + if len(loginLogFiles) != 0 { + for i := 0; i < len(loginLogFiles); i++ { + if !loginLogFiles[i].IsDir() && (strings.HasPrefix(loginLogFiles[i].Name(), "secure") || strings.HasPrefix(loginLogFiles[i].Name(), "auth.log")) { + _ = cpBinary([]string{path.Join("/var/log", loginLogFiles[i].Name())}, loginDir) + } + } + } + global.LOG.Debug("backup ssh log successful!") + + if err := handleTar(targetDir, path.Dir(targetDir), fileName, ""); err != nil { + return err + } + defer func() { + _ = os.RemoveAll(targetDir) + }() + return nil +} diff --git a/backend/app/service/cronjob_helper.go b/backend/app/service/cronjob_helper.go index 79c49525a..e031c4de6 100644 --- a/backend/app/service/cronjob_helper.go +++ b/backend/app/service/cronjob_helper.go @@ -5,15 +5,14 @@ import ( "fmt" "os" "path" - "strconv" "strings" "time" "github.com/1Panel-dev/1Panel/backend/buserr" "github.com/1Panel-dev/1Panel/backend/i18n" - "github.com/1Panel-dev/1Panel/backend/app/dto" "github.com/1Panel-dev/1Panel/backend/app/model" + "github.com/1Panel-dev/1Panel/backend/app/repo" "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/cloud_storage" @@ -35,32 +34,25 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) { if len(cronjob.Script) == 0 { return } + record.Records = u.generateLogsPath(*cronjob, record.StartTime) + _ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records}) + script := cronjob.Script if len(cronjob.ContainerName) != 0 { - message, err = u.handleShell(cronjob.Type, cronjob.Name, fmt.Sprintf("docker exec %s %s", cronjob.ContainerName, cronjob.Script)) - } else { - message, err = u.handleShell(cronjob.Type, cronjob.Name, cronjob.Script) + script = fmt.Sprintf("docker exec %s %s", cronjob.ContainerName, cronjob.Script) } - u.HandleRmExpired("LOCAL", "", "", cronjob, nil) - case "snapshot": - messageItem := "" - messageItem, record.File, err = u.handleSnapshot(cronjob, record.StartTime) - message = []byte(messageItem) + err = u.handleShell(cronjob.Type, cronjob.Name, script, record.Records) + u.removeExpiredLog(*cronjob) case "curl": if len(cronjob.URL) == 0 { return } - message, err = u.handleShell(cronjob.Type, cronjob.Name, fmt.Sprintf("curl '%s'", cronjob.URL)) - u.HandleRmExpired("LOCAL", "", "", cronjob, nil) + record.Records = u.generateLogsPath(*cronjob, record.StartTime) + _ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records}) + err = u.handleShell(cronjob.Type, cronjob.Name, fmt.Sprintf("curl '%s'", cronjob.URL), record.Records) + u.removeExpiredLog(*cronjob) case "ntp": err = u.handleNtpSync() - u.HandleRmExpired("LOCAL", "", "", cronjob, nil) - case "website", "database", "app": - record.File, err = u.handleBackup(cronjob, record.StartTime) - case "directory": - if len(cronjob.SourceDir) == 0 { - return - } - record.File, err = u.handleBackup(cronjob, record.StartTime) + u.removeExpiredLog(*cronjob) case "cutWebsiteLog": var messageItem []string messageItem, record.File, err = u.handleCutWebsiteLog(cronjob, record.StartTime) @@ -69,9 +61,24 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) { messageItem := "" messageItem, err = u.handleSystemClean() message = []byte(messageItem) - u.HandleRmExpired("LOCAL", "", "", cronjob, nil) + u.removeExpiredLog(*cronjob) + case "website": + err = u.handleWebsite(*cronjob, record.StartTime) + case "app": + err = u.handleApp(*cronjob, record.StartTime) + case "database": + err = u.handleDatabase(*cronjob, record.StartTime) + case "directory": + if len(cronjob.SourceDir) == 0 { + return + } + err = u.handleDirectory(*cronjob, record.StartTime) case "log": - record.File, err = u.handleSystemLog(*cronjob, record.StartTime) + err = u.handleSystemLog(*cronjob, record.StartTime) + case "snapshot": + record.Records = u.generateLogsPath(*cronjob, record.StartTime) + _ = cronjobRepo.UpdateRecords(record.ID, map[string]interface{}{"records": record.Records}) + err = u.handleSnapshot(*cronjob, record.StartTime, record.Records) } if err != nil { @@ -88,18 +95,17 @@ func (u *CronjobService) HandleJob(cronjob *model.Cronjob) { }() } -func (u *CronjobService) handleShell(cronType, cornName, script string) ([]byte, error) { +func (u *CronjobService) handleShell(cronType, cornName, script, logPath string) error { handleDir := fmt.Sprintf("%s/task/%s/%s", constant.DataDir, cronType, cornName) if _, err := os.Stat(handleDir); err != nil && os.IsNotExist(err) { if err = os.MkdirAll(handleDir, os.ModePerm); err != nil { - return nil, err + return err } } - stdout, err := cmd.ExecCronjobWithTimeOut(script, handleDir, 24*time.Hour) - if err != nil { - return []byte(stdout), err + if err := cmd.ExecCronjobWithTimeOut(script, handleDir, logPath, 24*time.Hour); err != nil { + return err } - return []byte(stdout), nil + return nil } func (u *CronjobService) handleNtpSync() error { @@ -117,98 +123,6 @@ func (u *CronjobService) handleNtpSync() error { return nil } -func (u *CronjobService) handleBackup(cronjob *model.Cronjob, startTime time.Time) (string, error) { - backup, err := backupRepo.Get(commonRepo.WithByID(uint(cronjob.TargetDirID))) - if err != nil { - return "", err - } - localDir, err := loadLocalDir() - if err != nil { - return "", err - } - global.LOG.Infof("start to backup %s %s to %s", cronjob.Type, cronjob.Name, backup.Type) - - switch cronjob.Type { - case "database": - paths, err := u.handleDatabase(*cronjob, backup, startTime) - return strings.Join(paths, ","), err - case "app": - paths, err := u.handleApp(*cronjob, backup, startTime) - return strings.Join(paths, ","), err - case "website": - paths, err := u.handleWebsite(*cronjob, backup, startTime) - return strings.Join(paths, ","), err - default: - fileName := fmt.Sprintf("directory%s_%s.tar.gz", strings.ReplaceAll(cronjob.SourceDir, "/", "_"), startTime.Format("20060102150405")) - backupDir := path.Join(localDir, fmt.Sprintf("%s/%s", cronjob.Type, cronjob.Name)) - itemFileDir := fmt.Sprintf("%s/%s", cronjob.Type, cronjob.Name) - global.LOG.Infof("handle tar %s to %s", backupDir, fileName) - if err := handleTar(cronjob.SourceDir, backupDir, fileName, cronjob.ExclusionRules); err != nil { - return "", err - } - var client cloud_storage.CloudStorageClient - if backup.Type != "LOCAL" { - if !cronjob.KeepLocal { - defer func() { - _ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, fileName)) - }() - } - client, err = NewIBackupService().NewClient(&backup) - if err != nil { - return "", err - } - if len(backup.BackupPath) != 0 { - itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) - } - if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil { - return "", err - } - } - u.HandleRmExpired(backup.Type, backup.BackupPath, localDir, cronjob, client) - if backup.Type == "LOCAL" || cronjob.KeepLocal { - return fmt.Sprintf("%s/%s", backupDir, fileName), nil - } else { - return fmt.Sprintf("%s/%s", itemFileDir, fileName), nil - } - } -} - -func (u *CronjobService) HandleRmExpired(backType, backupPath, localDir string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) { - global.LOG.Infof("start to handle remove expired, retain copies: %d", cronjob.RetainCopies) - records, _ := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(cronjob.ID)), commonRepo.WithOrderBy("created_at desc")) - if len(records) <= int(cronjob.RetainCopies) { - return - } - for i := int(cronjob.RetainCopies); i < len(records); i++ { - if len(records[i].File) != 0 { - files := strings.Split(records[i].File, ",") - for _, file := range files { - _ = os.Remove(file) - _ = backupRepo.DeleteRecord(context.TODO(), backupRepo.WithByFileName(path.Base(file))) - if backType == "LOCAL" { - continue - } - - fileItem := file - if cronjob.KeepLocal { - if len(backupPath) != 0 { - fileItem = path.Join(strings.TrimPrefix(backupPath, "/") + strings.TrimPrefix(file, localDir+"/")) - } else { - fileItem = strings.TrimPrefix(file, localDir+"/") - } - } - - if cronjob.Type == "snapshot" { - _ = snapshotRepo.Delete(commonRepo.WithByName(strings.TrimSuffix(path.Base(fileItem), ".tar.gz"))) - } - _, _ = backClient.Delete(fileItem) - } - } - _ = cronjobRepo.DeleteRecord(commonRepo.WithByID(uint(records[i].ID))) - _ = os.Remove(records[i].Records) - } -} - func handleTar(sourceDir, targetDir, name, exclusionRules string) error { if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { @@ -266,77 +180,6 @@ func handleUnTar(sourceFile, targetDir string) error { return nil } -func (u *CronjobService) handleDatabase(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) ([]string, error) { - var paths []string - localDir, err := loadLocalDir() - if err != nil { - return paths, err - } - - dbs := loadDbsForJob(cronjob) - - var client cloud_storage.CloudStorageClient - if backup.Type != "LOCAL" { - client, err = NewIBackupService().NewClient(&backup) - if err != nil { - return paths, err - } - } - - for _, dbInfo := range dbs { - var record model.BackupRecord - record.Type = dbInfo.DBType - record.Source = "LOCAL" - record.BackupType = backup.Type - - record.Name = dbInfo.Database - backupDir := path.Join(localDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name)) - record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format("20060102150405")) - if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { - if err = handleMysqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { - return paths, err - } - } else { - if err = handlePostgresqlBackup(dbInfo.Database, dbInfo.Name, backupDir, record.FileName); err != nil { - return paths, err - } - } - - record.DetailName = dbInfo.Name - record.FileDir = backupDir - itemFileDir := strings.TrimPrefix(backupDir, localDir+"/") - if !cronjob.KeepLocal && backup.Type != "LOCAL" { - record.Source = backup.Type - record.FileDir = itemFileDir - } - - if err := backupRepo.CreateRecord(&record); err != nil { - global.LOG.Errorf("save backup record failed, err: %v", err) - return paths, err - } - if backup.Type != "LOCAL" { - if !cronjob.KeepLocal { - defer func() { - _ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName)) - }() - } - if len(backup.BackupPath) != 0 { - itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) - } - if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { - return paths, err - } - } - if backup.Type == "LOCAL" || cronjob.KeepLocal { - paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName)) - } else { - paths = append(paths, fmt.Sprintf("%s/%s", itemFileDir, record.FileName)) - } - } - u.HandleRmExpired(backup.Type, backup.BackupPath, localDir, &cronjob, client) - return paths, nil -} - func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime time.Time) ([]string, string, error) { var ( err error @@ -377,7 +220,7 @@ func (u *CronjobService) handleCutWebsiteLog(cronjob *model.Cronjob, startTime t global.LOG.Infof(msg) msgs = append(msgs, msg) } - u.HandleRmExpired("LOCAL", "", "", cronjob, nil) + u.removeExpiredLog(*cronjob) return msgs, strings.Join(filePaths, ","), err } @@ -400,340 +243,124 @@ func backupLogFile(dstFilePath, websiteLogDir string, fileOp files.FileOp) error return nil } -func (u *CronjobService) handleApp(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) ([]string, error) { - var paths []string - localDir, err := loadLocalDir() - if err != nil { - return paths, err - } - - var applist []model.AppInstall - if cronjob.AppID == "all" { - applist, err = appInstallRepo.ListBy() - if err != nil { - return paths, err - } - } else { - itemID, _ := (strconv.Atoi(cronjob.AppID)) - app, err := appInstallRepo.GetFirst(commonRepo.WithByID(uint(itemID))) - if err != nil { - return paths, err - } - applist = append(applist, app) - } - - var client cloud_storage.CloudStorageClient - if backup.Type != "LOCAL" { - client, err = NewIBackupService().NewClient(&backup) - if err != nil { - return paths, err - } - } - - for _, app := range applist { - var record model.BackupRecord - record.Type = "app" - record.Name = app.App.Key - record.DetailName = app.Name - record.Source = "LOCAL" - record.BackupType = backup.Type - backupDir := path.Join(localDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name)) - record.FileDir = backupDir - itemFileDir := strings.TrimPrefix(backupDir, localDir+"/") - if !cronjob.KeepLocal && backup.Type != "LOCAL" { - record.Source = backup.Type - record.FileDir = strings.TrimPrefix(backupDir, localDir+"/") - } - record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format("20060102150405")) - if err := handleAppBackup(&app, backupDir, record.FileName); err != nil { - return paths, err - } - if err := backupRepo.CreateRecord(&record); err != nil { - global.LOG.Errorf("save backup record failed, err: %v", err) - return paths, err - } - if backup.Type != "LOCAL" { - if !cronjob.KeepLocal { - defer func() { - _ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName)) - }() - } - if len(backup.BackupPath) != 0 { - itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) - } - if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { - return paths, err - } - } - if backup.Type == "LOCAL" || cronjob.KeepLocal { - paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName)) - } else { - paths = append(paths, fmt.Sprintf("%s/%s", itemFileDir, record.FileName)) - } - } - u.HandleRmExpired(backup.Type, backup.BackupPath, localDir, &cronjob, client) - return paths, nil +func (u *CronjobService) handleSystemClean() (string, error) { + return NewIDeviceService().CleanForCronjob() } -func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) ([]string, error) { - var paths []string - localDir, err := loadLocalDir() +func (u *CronjobService) loadClientMap(targetAccountIDs string) (map[string]cronjobUploadHelper, error) { + clients := make(map[string]cronjobUploadHelper) + accounts, err := backupRepo.List() if err != nil { - return paths, err + return nil, err } - - weblist := loadWebsForJob(cronjob) - var client cloud_storage.CloudStorageClient - if backup.Type != "LOCAL" { - client, err = NewIBackupService().NewClient(&backup) - if err != nil { - return paths, err - } - } - - for _, websiteItem := range weblist { - var record model.BackupRecord - record.Type = "website" - record.Name = websiteItem.PrimaryDomain - record.DetailName = websiteItem.Alias - record.Source = "LOCAL" - record.BackupType = backup.Type - backupDir := path.Join(localDir, fmt.Sprintf("website/%s", websiteItem.PrimaryDomain)) - record.FileDir = backupDir - itemFileDir := strings.TrimPrefix(backupDir, localDir+"/") - if !cronjob.KeepLocal && backup.Type != "LOCAL" { - record.Source = backup.Type - record.FileDir = strings.TrimPrefix(backupDir, localDir+"/") - } - record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", websiteItem.PrimaryDomain, startTime.Format("20060102150405")) - if err := handleWebsiteBackup(&websiteItem, backupDir, record.FileName); err != nil { - return paths, err - } - if err := backupRepo.CreateRecord(&record); err != nil { - global.LOG.Errorf("save backup record failed, err: %v", err) - return paths, err + targets := strings.Split(targetAccountIDs, ",") + for _, target := range targets { + if len(target) == 0 { + continue } - if backup.Type != "LOCAL" { - if !cronjob.KeepLocal { - defer func() { - _ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName)) - }() - } - if len(backup.BackupPath) != 0 { - itemFileDir = path.Join(strings.TrimPrefix(backup.BackupPath, "/"), itemFileDir) - } - if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil { - return paths, err + for _, account := range accounts { + if target == fmt.Sprintf("%v", account.ID) { + client, err := NewIBackupService().NewClient(&account) + if err != nil { + return nil, err + } + pathItem := account.BackupPath + clients[target] = cronjobUploadHelper{ + client: client, + backupPath: pathItem, + backType: account.Type, + } } } - if backup.Type == "LOCAL" || cronjob.KeepLocal { - paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName)) - } else { - paths = append(paths, fmt.Sprintf("%s/%s", itemFileDir, record.FileName)) - } - } - u.HandleRmExpired(backup.Type, backup.BackupPath, localDir, &cronjob, client) - return paths, nil -} - -func (u *CronjobService) handleSnapshot(cronjob *model.Cronjob, startTime time.Time) (string, string, error) { - backup, err := backupRepo.Get(commonRepo.WithByID(uint(cronjob.TargetDirID))) - if err != nil { - return "", "", err - } - client, err := NewIBackupService().NewClient(&backup) - if err != nil { - return "", "", err - } - - req := dto.SnapshotCreate{ - From: backup.Type, } - message, name, err := NewISnapshotService().HandleSnapshot(true, req, startTime.Format("20060102150405")) - if err != nil { - return message, "", err - } - - path := path.Join(strings.TrimPrefix(backup.BackupPath, "/"), "system_snapshot", name+".tar.gz") - - u.HandleRmExpired(backup.Type, backup.BackupPath, "", cronjob, client) - return message, path, nil + return clients, nil } -func (u *CronjobService) handleSystemClean() (string, error) { - return NewIDeviceService().CleanForCronjob() +type cronjobUploadHelper struct { + backupPath string + backType string + client cloud_storage.CloudStorageClient } -func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.Time) (string, error) { - backup, err := backupRepo.Get(commonRepo.WithByID(uint(cronjob.TargetDirID))) - if err != nil { - return "", err +func (u *CronjobService) uploadCronjobBackFile(cronjob model.Cronjob, accountMap map[string]cronjobUploadHelper, file string) (string, error) { + defer func() { + _ = os.Remove(file) + }() + targets := strings.Split(cronjob.TargetAccountIDs, ",") + cloudSrc := strings.TrimPrefix(file, global.CONF.System.TmpDir+"/") + for _, target := range targets { + if len(target) != 0 { + if _, err := accountMap[target].client.Upload(file, path.Join(accountMap[target].backupPath, cloudSrc)); err != nil { + return "", err + } + } } + return cloudSrc, nil +} - pathItem := path.Join(global.CONF.System.BaseDir, "1panel/tmp/log", startTime.Format("20060102150405")) - websites, err := websiteRepo.List() - if err != nil { - return "", err +func (u *CronjobService) removeExpiredBackup(cronjob model.Cronjob, accountMap map[string]cronjobUploadHelper, record model.BackupRecord) { + global.LOG.Infof("start to handle remove expired, retain copies: %d", cronjob.RetainCopies) + var opts []repo.DBOption + opts = append(opts, commonRepo.WithByFrom("cronjob")) + opts = append(opts, backupRepo.WithByCronID(cronjob.ID)) + opts = append(opts, commonRepo.WithOrderBy("created_at desc")) + if record.ID != 0 { + opts = append(opts, backupRepo.WithByType(record.Type)) + opts = append(opts, commonRepo.WithByName(record.Name)) + opts = append(opts, backupRepo.WithByDetailName(record.DetailName)) + } + records, _ := backupRepo.ListRecord(opts...) + if len(records) <= int(cronjob.RetainCopies) { + return } - if len(websites) != 0 { - nginxInstall, err := getAppInstallByKey(constant.AppOpenresty) - if err != nil { - return "", err - } - webItem := path.Join(nginxInstall.GetPath(), "www/sites") - for _, website := range websites { - dirItem := path.Join(pathItem, "website", website.Alias) - if _, err := os.Stat(dirItem); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(dirItem, os.ModePerm); err != nil { - return "", err - } - } - itemDir := path.Join(webItem, website.Alias, "log") - logFiles, _ := os.ReadDir(itemDir) - if len(logFiles) != 0 { - for i := 0; i < len(logFiles); i++ { - if !logFiles[i].IsDir() { - _ = cpBinary([]string{path.Join(itemDir, logFiles[i].Name())}, dirItem) - } + for i := int(cronjob.RetainCopies); i < len(records); i++ { + targets := strings.Split(cronjob.TargetAccountIDs, ",") + if cronjob.Type == "snapshot" { + for _, target := range targets { + if len(target) != 0 { + _, _ = accountMap[target].client.Delete(path.Join(accountMap[target].backupPath, "system_snapshot", records[i].FileName)) } } - itemDir2 := path.Join(global.CONF.System.Backup, "log/website", website.Alias) - logFiles2, _ := os.ReadDir(itemDir2) - if len(logFiles2) != 0 { - for i := 0; i < len(logFiles2); i++ { - if !logFiles2[i].IsDir() { - _ = cpBinary([]string{path.Join(itemDir2, logFiles2[i].Name())}, dirItem) - } + _ = snapshotRepo.Delete(commonRepo.WithByName(strings.TrimSuffix(records[i].FileName, ".tar.gz"))) + } else { + for _, target := range targets { + if len(target) != 0 { + _, _ = accountMap[target].client.Delete(path.Join(accountMap[target].backupPath, records[i].FileDir, records[i].FileName)) } } } - global.LOG.Debug("backup website log successful!") + _ = backupRepo.DeleteRecord(context.Background(), commonRepo.WithByID(records[i].ID)) } +} - systemLogDir := path.Join(global.CONF.System.BaseDir, "1panel/log") - systemDir := path.Join(pathItem, "system") - if _, err := os.Stat(systemDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(systemDir, os.ModePerm); err != nil { - return "", err - } - } - systemLogFiles, _ := os.ReadDir(systemLogDir) - if len(systemLogFiles) != 0 { - for i := 0; i < len(systemLogFiles); i++ { - if !systemLogFiles[i].IsDir() { - _ = cpBinary([]string{path.Join(systemLogDir, systemLogFiles[i].Name())}, systemDir) - } - } - } - global.LOG.Debug("backup system log successful!") - - loginLogFiles, _ := os.ReadDir("/var/log") - loginDir := path.Join(pathItem, "login") - if _, err := os.Stat(loginDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(loginDir, os.ModePerm); err != nil { - return "", err - } +func (u *CronjobService) removeExpiredLog(cronjob model.Cronjob) { + global.LOG.Infof("start to handle remove expired, retain copies: %d", cronjob.RetainCopies) + records, _ := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(cronjob.ID)), commonRepo.WithOrderBy("created_at desc")) + if len(records) <= int(cronjob.RetainCopies) { + return } - if len(loginLogFiles) != 0 { - for i := 0; i < len(loginLogFiles); i++ { - if !loginLogFiles[i].IsDir() && (strings.HasPrefix(loginLogFiles[i].Name(), "secure") || strings.HasPrefix(loginLogFiles[i].Name(), "auth.log")) { - _ = cpBinary([]string{path.Join("/var/log", loginLogFiles[i].Name())}, loginDir) + for i := int(cronjob.RetainCopies); i < len(records); i++ { + if len(records[i].File) != 0 { + files := strings.Split(records[i].File, ",") + for _, file := range files { + _ = os.Remove(file) } } + _ = cronjobRepo.DeleteRecord(commonRepo.WithByID(uint(records[i].ID))) + _ = os.Remove(records[i].Records) } - global.LOG.Debug("backup ssh log successful!") - - fileName := fmt.Sprintf("system_log_%s.tar.gz", startTime.Format("20060102150405")) - targetDir := path.Dir(pathItem) - if err := handleTar(pathItem, targetDir, fileName, ""); err != nil { - return "", err - } - defer func() { - os.RemoveAll(pathItem) - os.RemoveAll(path.Join(targetDir, fileName)) - }() - - client, err := NewIBackupService().NewClient(&backup) - if err != nil { - return "", err - } - - targetPath := "log/" + fileName - if len(backup.BackupPath) != 0 { - targetPath = strings.TrimPrefix(backup.BackupPath, "/") + "/log/" + fileName - } +} - if _, err = client.Upload(path.Join(targetDir, fileName), targetPath); err != nil { - return "", err +func (u *CronjobService) generateLogsPath(cronjob model.Cronjob, startTime time.Time) string { + dir := fmt.Sprintf("%s/task/%s/%s", constant.DataDir, cronjob.Type, cronjob.Name) + if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(dir, os.ModePerm) } - u.HandleRmExpired(backup.Type, backup.BackupPath, "", &cronjob, client) - return targetPath, nil + path := fmt.Sprintf("%s/%s.log", dir, startTime.Format("20060102150405")) + return path } func hasBackup(cronjobType string) bool { return cronjobType == "app" || cronjobType == "database" || cronjobType == "website" || cronjobType == "directory" || cronjobType == "snapshot" || cronjobType == "log" } - -type databaseHelper struct { - DBType string - Database string - Name string -} - -func loadDbsForJob(cronjob model.Cronjob) []databaseHelper { - var dbs []databaseHelper - if cronjob.DBName == "all" { - if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { - mysqlItems, _ := mysqlRepo.List() - for _, mysql := range mysqlItems { - dbs = append(dbs, databaseHelper{ - DBType: cronjob.DBType, - Database: mysql.MysqlName, - Name: mysql.Name, - }) - } - } else { - pgItems, _ := postgresqlRepo.List() - for _, pg := range pgItems { - dbs = append(dbs, databaseHelper{ - DBType: cronjob.DBType, - Database: pg.PostgresqlName, - Name: pg.Name, - }) - } - } - return dbs - } - itemID, _ := (strconv.Atoi(cronjob.DBName)) - if cronjob.DBType == "mysql" || cronjob.DBType == "mariadb" { - mysqlItem, _ := mysqlRepo.Get(commonRepo.WithByID(uint(itemID))) - dbs = append(dbs, databaseHelper{ - DBType: cronjob.DBType, - Database: mysqlItem.MysqlName, - Name: mysqlItem.Name, - }) - } else { - pgItem, _ := postgresqlRepo.Get(commonRepo.WithByID(uint(itemID))) - dbs = append(dbs, databaseHelper{ - DBType: cronjob.DBType, - Database: pgItem.PostgresqlName, - Name: pgItem.Name, - }) - } - return dbs -} - -func loadWebsForJob(cronjob model.Cronjob) []model.Website { - var weblist []model.Website - if cronjob.Website == "all" { - weblist, _ = websiteRepo.List() - return weblist - } - itemID, _ := (strconv.Atoi(cronjob.Website)) - webItem, _ := websiteRepo.GetFirst(commonRepo.WithByID(uint(itemID))) - if webItem.ID != 0 { - weblist = append(weblist, webItem) - } - return weblist -} diff --git a/backend/app/service/snapshot.go b/backend/app/service/snapshot.go index 83a3d3028..227f1827e 100644 --- a/backend/app/service/snapshot.go +++ b/backend/app/service/snapshot.go @@ -41,7 +41,7 @@ type ISnapshotService interface { UpdateDescription(req dto.UpdateDescription) error readFromJson(path string) (SnapshotJson, error) - HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string) (string, string, error) + HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string) (string, error) } func NewISnapshotService() ISnapshotService { @@ -132,7 +132,7 @@ type SnapshotJson struct { } func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { - if _, _, err := u.HandleSnapshot(false, req, time.Now().Format("20060102150405")); err != nil { + if _, err := u.HandleSnapshot(false, "", req, time.Now().Format("20060102150405")); err != nil { return err } return nil @@ -469,10 +469,10 @@ func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) { return snap, nil } -func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string) (string, string, error) { +func (u *SnapshotService) HandleSnapshot(isCronjob bool, logPath string, req dto.SnapshotCreate, timeNow string) (string, error) { localDir, err := loadLocalDir() if err != nil { - return "", "", err + return "", err } var ( rootDir string @@ -501,7 +501,7 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, } else { snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) if err != nil { - return "", "", err + return "", err } snapStatus, _ = snapshotRepo.GetStatus(snap.ID) if snapStatus.ID == 0 { @@ -523,7 +523,7 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, BackupDataDir: localDir, PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), } - + loadLogByStatus(snapStatus, logPath) if snapStatus.PanelInfo != constant.StatusDone { wg.Add(1) go snapJson(itemHelper, jsonItem, rootDir) @@ -569,30 +569,35 @@ func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) }() - return "", "", nil + return "", nil } wg.Wait() if !checkIsAllDone(snap.ID) { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) + loadLogByStatus(snapStatus, logPath) + return snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name) } snapPanelData(itemHelper, localDir, backupPanelDir) if snapStatus.PanelData != constant.StatusDone { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name) + loadLogByStatus(snapStatus, logPath) + return snap.Name, fmt.Errorf("snapshot %s 1panel data failed", snap.Name) } snapCompress(itemHelper, rootDir) if snapStatus.Compress != constant.StatusDone { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name) + loadLogByStatus(snapStatus, logPath) + return snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name) } snapUpload(itemHelper, req.From, fmt.Sprintf("%s.tar.gz", rootDir)) if snapStatus.Upload != constant.StatusDone { _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) - return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name) + loadLogByStatus(snapStatus, logPath) + return snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name) } _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - return loadLogByStatus(snapStatus), snap.Name, nil + loadLogByStatus(snapStatus, logPath) + return snap.Name, nil } func (u *SnapshotService) handleDockerDatas(fileOp files.FileOp, operation string, source, target string) error { @@ -1060,7 +1065,7 @@ func checkIsAllDone(snapID uint) bool { return true } -func loadLogByStatus(status model.SnapshotStatus) string { +func loadLogByStatus(status model.SnapshotStatus, logPath string) { logs := "" logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo) logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel) @@ -1072,5 +1077,11 @@ func loadLogByStatus(status model.SnapshotStatus) string { logs += fmt.Sprintf("Snapshot size: %s \n", status.Size) logs += fmt.Sprintf("Upload snapshot file: %s \n", status.Upload) - return logs + file, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) + if err != nil { + global.LOG.Errorf("write snapshot logs failed, err: %v", err) + return + } + defer file.Close() + _, _ = file.Write([]byte(logs)) } diff --git a/backend/app/service/snapshot_create.go b/backend/app/service/snapshot_create.go index 04b1c86a5..a13317a67 100644 --- a/backend/app/service/snapshot_create.go +++ b/backend/app/service/snapshot_create.go @@ -175,7 +175,7 @@ func snapCompress(snap snapHelper, rootDir string) { _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size}) } -func snapUpload(snap snapHelper, account string, file string) { +func snapUpload(snap snapHelper, accounts string, file string) { source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) defer func() { global.LOG.Debugf("remove snapshot file %s", source) @@ -183,24 +183,20 @@ func snapUpload(snap snapHelper, account string, file string) { }() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading}) - backup, err := backupRepo.Get(commonRepo.WithByType(account)) + accountMap, err := loadClientMapForSnapshot(accounts) if err != nil { snap.Status.Upload = err.Error() _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) return } - client, err := NewIBackupService().NewClient(&backup) - if err != nil { - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return - } - target := path.Join(strings.TrimPrefix(backup.BackupPath, "/"), "system_snapshot", path.Base(file)) - global.LOG.Debugf("start upload snapshot to %s, dir: %s", backup.Type, target) - if _, err := client.Upload(source, target); err != nil { - snap.Status.Upload = err.Error() - _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) - return + targetAccounts := strings.Split(accounts, ",") + for _, item := range targetAccounts { + global.LOG.Debugf("start upload snapshot to %s, dir: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) + if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { + snap.Status.Upload = err.Error() + _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) + return + } } snap.Status.Upload = constant.StatusDone _ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) @@ -241,3 +237,32 @@ func checkPointOfWal() { global.LOG.Errorf("handle check point failed, err: %v", err) } } + +func loadClientMapForSnapshot(from string) (map[string]cronjobUploadHelper, error) { + clients := make(map[string]cronjobUploadHelper) + accounts, err := backupRepo.List() + if err != nil { + return nil, err + } + targets := strings.Split(from, ",") + for _, target := range targets { + if len(target) == 0 { + continue + } + for _, account := range accounts { + if target == fmt.Sprintf("%v", account.ID) { + client, err := NewIBackupService().NewClient(&account) + if err != nil { + return nil, err + } + pathItem := account.BackupPath + clients[target] = cronjobUploadHelper{ + client: client, + backupPath: pathItem, + backType: account.Type, + } + } + } + } + return clients, nil +} diff --git a/backend/constant/backup.go b/backend/constant/backup.go index d99a34822..37baa994e 100644 --- a/backend/constant/backup.go +++ b/backend/constant/backup.go @@ -12,6 +12,7 @@ const ( Cos = "COS" Kodo = "KODO" WebDAV = "WebDAV" + Local = "LOCAL" OneDriveRedirectURI = "http://localhost/login/authorized" ) diff --git a/backend/init/migration/migrate.go b/backend/init/migration/migrate.go index aba816dba..a34070037 100644 --- a/backend/init/migration/migrate.go +++ b/backend/init/migration/migrate.go @@ -68,6 +68,7 @@ func Init() { migrations.UpdateCronjobWithWebsite, migrations.UpdateOneDriveToken, migrations.UpdateCronjobSpec, + migrations.UpdateBackupRecordPath, }) if err := m.Migrate(); err != nil { global.LOG.Error(err) diff --git a/backend/init/migration/migrations/v_1_9.go b/backend/init/migration/migrations/v_1_9.go index df0c1a51a..36dd70811 100644 --- a/backend/init/migration/migrations/v_1_9.go +++ b/backend/init/migration/migrations/v_1_9.go @@ -3,6 +3,10 @@ package migrations import ( "encoding/base64" "encoding/json" + "errors" + "fmt" + "path" + "strings" "time" "github.com/1Panel-dev/1Panel/backend/app/dto/request" @@ -261,13 +265,161 @@ var UpdateCronjobSpec = &gormigrate.Migration{ if err := tx.AutoMigrate(&model.Cronjob{}); err != nil { return err } + if err := tx.AutoMigrate(&model.BackupRecord{}); err != nil { + return err + } + var ( + jobs []model.Cronjob + backupAccounts []model.BackupAccount + localAccountID uint + ) + mapAccount := make(map[uint]string) + mapAccountName := make(map[string]model.BackupAccount) + if err := tx.Find(&jobs).Error; err != nil { + return err + } + _ = tx.Find(&backupAccounts).Error + for _, item := range backupAccounts { + mapAccount[item.ID] = item.Type + mapAccountName[item.Type] = item + if item.Type == constant.Local { + localAccountID = item.ID + } + } + if localAccountID == 0 { + return errors.New("local backup account is unset!") + } + for _, job := range jobs { + if job.KeepLocal { + if err := tx.Model(&model.Cronjob{}). + Where("id = ?", job.ID). + Updates(map[string]interface{}{ + "target_account_ids": fmt.Sprintf("%v,%v", job.TargetDirID, localAccountID), + "target_dir_id": localAccountID, + }).Error; err != nil { + return err + } + } else { + if err := tx.Model(&model.Cronjob{}). + Where("id = ?", job.ID). + Updates(map[string]interface{}{ + "target_account_ids": job.TargetDirID, + }).Error; err != nil { + return err + } + } + if job.Type != "directory" && job.Type != "database" && job.Type != "website" && job.Type != "app" && job.Type != "snapshot" && job.Type != "log" { + continue + } + + var records []model.JobRecords + _ = tx.Where("cronjob_id = ?", job.ID).Find(&records).Error + for _, record := range records { + if job.Type == "snapshot" { + var snaps []model.Snapshot + _ = tx.Where("name like ?", "snapshot_"+"%").Find(&snaps).Error + for _, snap := range snaps { + item := model.BackupRecord{ + From: "cronjob", + CronjobID: job.ID, + Type: "snapshot", + Name: job.Name, + FileName: snap.Name + ".tar.gz", + Source: snap.From, + BackupType: snap.From, + } + _ = tx.Create(&item).Error + } + continue + } + if job.Type == "log" { + item := model.BackupRecord{ + From: "cronjob", + CronjobID: job.ID, + Type: "log", + Name: job.Name, + FileDir: path.Dir(record.File), + FileName: path.Base(record.File), + Source: mapAccount[uint(job.TargetDirID)], + BackupType: mapAccount[uint(job.TargetDirID)], + } + _ = tx.Create(&item).Error + continue + } + if job.Type == "directory" { + item := model.BackupRecord{ + From: "cronjob", + CronjobID: job.ID, + Type: "directory", + Name: job.Name, + FileDir: path.Dir(record.File), + FileName: path.Base(record.File), + BackupType: mapAccount[uint(job.TargetDirID)], + } + if record.FromLocal { + item.Source = constant.Local + } else { + item.Source = mapAccount[uint(job.TargetDirID)] + } + _ = tx.Create(&item).Error + continue + } + if strings.Contains(record.File, ",") { + files := strings.Split(record.File, ",") + for _, file := range files { + _ = tx.Model(&model.BackupRecord{}). + Where("file_dir = ? AND file_name = ?", path.Dir(file), path.Base(file)). + Updates(map[string]interface{}{"cronjob_id": job.ID, "from": "cronjob"}).Error + } + } else { + _ = tx.Model(&model.BackupRecord{}). + Where("file_dir = ? AND file_name = ?", path.Dir(record.File), path.Base(record.File)). + Updates(map[string]interface{}{"cronjob_id": job.ID, "from": "cronjob"}).Error + } + } + } + _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN spec_type;").Error _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN week;").Error _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN day;").Error _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN hour;").Error _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN minute;").Error _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN second;").Error + _ = tx.Exec("ALTER TABLE cronjobs DROP COLUMN entry_id;").Error + + return nil + }, +} + +var UpdateBackupRecordPath = &gormigrate.Migration{ + ID: "20240124-update-cronjob-spec", + Migrate: func(tx *gorm.DB) error { + var ( + backupRecords []model.BackupRecord + localAccount model.BackupAccount + ) + _ = tx.Where("type = ?", "LOCAL").First(&localAccount).Error + if localAccount.ID == 0 { + return nil + } + varMap := make(map[string]string) + if err := json.Unmarshal([]byte(localAccount.Vars), &varMap); err != nil { + return err + } + dir, ok := varMap["dir"] + if !ok { + return errors.New("load local backup dir failed") + } + if dir != "/" { + dir += "/" + } + _ = tx.Where("source = ?", "LOCAL").Find(&backupRecords).Error + for _, record := range backupRecords { + _ = tx.Model(&model.BackupRecord{}). + Where("id = ?", record.ID). + Updates(map[string]interface{}{"file_dir": strings.TrimPrefix(record.FileDir, dir)}).Error + } return nil }, } diff --git a/backend/router/ro_setting.go b/backend/router/ro_setting.go index 17a3dd41c..3ade938dc 100644 --- a/backend/router/ro_setting.go +++ b/backend/router/ro_setting.go @@ -54,6 +54,7 @@ func (s *SettingRouter) InitRouter(Router *gin.RouterGroup) { settingRouter.POST("/backup/del", baseApi.DeleteBackup) settingRouter.POST("/backup/update", baseApi.UpdateBackup) settingRouter.POST("/backup/record/search", baseApi.SearchBackupRecords) + settingRouter.POST("/backup/record/search/bycronjob", baseApi.SearchBackupRecordsByCronjob) settingRouter.POST("/backup/record/download", baseApi.DownloadRecord) settingRouter.POST("/backup/record/del", baseApi.DeleteBackupRecord) diff --git a/backend/utils/cloud_storage/client/local.go b/backend/utils/cloud_storage/client/local.go new file mode 100644 index 000000000..61e34ed9d --- /dev/null +++ b/backend/utils/cloud_storage/client/local.go @@ -0,0 +1,69 @@ +package client + +import ( + "fmt" + "os" + "path" + + "github.com/1Panel-dev/1Panel/backend/utils/cmd" +) + +type localClient struct { + dir string +} + +func NewLocalClient(vars map[string]interface{}) (*localClient, error) { + dir := loadParamFromVars("dir", true, vars) + return &localClient{dir: dir}, nil +} + +func (c localClient) ListBuckets() ([]interface{}, error) { + return nil, nil +} + +func (c localClient) Exist(file string) (bool, error) { + _, err := os.Stat(path.Join(c.dir, file)) + return err == nil, err +} + +func (c localClient) Size(file string) (int64, error) { + fileInfo, err := os.Stat(path.Join(c.dir, file)) + if err != nil { + return 0, err + } + return fileInfo.Size(), nil +} + +func (c localClient) Delete(file string) (bool, error) { + if err := os.RemoveAll(path.Join(c.dir, file)); err != nil { + return false, err + } + return true, nil +} + +func (c localClient) Upload(src, target string) (bool, error) { + targetFilePath := path.Join(c.dir, target) + if _, err := os.Stat(path.Dir(targetFilePath)); err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(path.Dir(targetFilePath), os.ModePerm); err != nil { + return false, err + } + } else { + return false, err + } + } + + stdout, err := cmd.Execf("\\cp -f %s %s", src, path.Join(c.dir, target)) + if err != nil { + return false, fmt.Errorf("cp file failed, stdout: %v, err: %v", stdout, err) + } + return true, nil +} + +func (c localClient) Download(src, target string) (bool, error) { + return true, nil +} + +func (c localClient) ListObjects(prefix string) ([]string, error) { + return nil, nil +} diff --git a/backend/utils/cloud_storage/client/sftp.go b/backend/utils/cloud_storage/client/sftp.go index e6e3acbb6..d41cec2a8 100644 --- a/backend/utils/cloud_storage/client/sftp.go +++ b/backend/utils/cloud_storage/client/sftp.go @@ -56,6 +56,17 @@ func (s sftpClient) Upload(src, target string) (bool, error) { } defer srcFile.Close() + targetFilePath := path.Join(s.bucket, target) + targetDir, _ := path.Split(targetFilePath) + if _, err = client.Stat(targetDir); err != nil { + if os.IsNotExist(err) { + if err = client.MkdirAll(targetDir); err != nil { + return false, err + } + } else { + return false, err + } + } dstFile, err := client.Create(path.Join(s.bucket, target)) if err != nil { return false, err diff --git a/backend/utils/cloud_storage/cloud_storage_client.go b/backend/utils/cloud_storage/cloud_storage_client.go index dbaea36e0..44412e7ba 100644 --- a/backend/utils/cloud_storage/cloud_storage_client.go +++ b/backend/utils/cloud_storage/cloud_storage_client.go @@ -18,6 +18,8 @@ type CloudStorageClient interface { func NewCloudStorageClient(backupType string, vars map[string]interface{}) (CloudStorageClient, error) { switch backupType { + case constant.Local: + return client.NewLocalClient(vars) case constant.S3: return client.NewS3Client(vars) case constant.OSS: diff --git a/backend/utils/cmd/cmd.go b/backend/utils/cmd/cmd.go index 1e56332f5..6ac65aada 100644 --- a/backend/utils/cmd/cmd.go +++ b/backend/utils/cmd/cmd.go @@ -4,7 +4,7 @@ import ( "bytes" "context" "fmt" - "io" + "os" "os/exec" "strings" "time" @@ -74,24 +74,27 @@ func ExecContainerScript(containerName, cmdStr string, timeout time.Duration) er return nil } -func ExecCronjobWithTimeOut(cmdStr string, workdir string, timeout time.Duration) (string, error) { +func ExecCronjobWithTimeOut(cmdStr, workdir, outPath string, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() + + file, err := os.OpenFile(outPath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + defer file.Close() + cmd := exec.Command("bash", "-c", cmdStr) cmd.Dir = workdir - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - output := new(bytes.Buffer) - cmd.Stdout = io.MultiWriter(output, cmd.Stdout) - cmd.Stderr = io.MultiWriter(output, cmd.Stderr) + cmd.Stdout = file + cmd.Stderr = file - err := cmd.Run() + err = cmd.Run() if ctx.Err() == context.DeadlineExceeded { - return "", buserr.New(constant.ErrCmdTimeout) + return buserr.New(constant.ErrCmdTimeout) } - return output.String(), err + return err } func Execf(cmdStr string, a ...interface{}) (string, error) { diff --git a/cmd/server/docs/docs.go b/cmd/server/docs/docs.go index 54f3f803a..fe2758d33 100644 --- a/cmd/server/docs/docs.go +++ b/cmd/server/docs/docs.go @@ -9523,6 +9523,39 @@ const docTemplate = `{ } } }, + "/settings/backup/record/search/bycronjob": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "通过计划任务获取备份记录列表分页", + "consumes": [ + "application/json" + ], + "tags": [ + "Backup Account" + ], + "summary": "Page backup records by cronjob", + "parameters": [ + { + "description": "request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/dto.RecordSearchByCronjob" + } + } + ], + "responses": { + "200": { + "description": "OK" + } + } + } + }, "/settings/backup/recover": { "post": { "security": [ @@ -14937,9 +14970,6 @@ const docTemplate = `{ "exclusionRules": { "type": "string" }, - "keepLocal": { - "type": "boolean" - }, "name": { "type": "string" }, @@ -14956,6 +14986,9 @@ const docTemplate = `{ "spec": { "type": "string" }, + "targetAccountIDs": { + "type": "string" + }, "targetDirID": { "type": "integer" }, @@ -15011,9 +15044,6 @@ const docTemplate = `{ "id": { "type": "integer" }, - "keepLocal": { - "type": "boolean" - }, "name": { "type": "string" }, @@ -15030,6 +15060,9 @@ const docTemplate = `{ "spec": { "type": "string" }, + "targetAccountIDs": { + "type": "string" + }, "targetDirID": { "type": "integer" }, @@ -17232,6 +17265,25 @@ const docTemplate = `{ } } }, + "dto.RecordSearchByCronjob": { + "type": "object", + "required": [ + "cronjobID", + "page", + "pageSize" + ], + "properties": { + "cronjobID": { + "type": "integer" + }, + "page": { + "type": "integer" + }, + "pageSize": { + "type": "integer" + } + } + }, "dto.RedisConf": { "type": "object", "properties": { @@ -17848,17 +17900,7 @@ const docTemplate = `{ "maxLength": 256 }, "from": { - "type": "string", - "enum": [ - "OSS", - "S3", - "SFTP", - "MINIO", - "COS", - "KODO", - "OneDrive", - "WebDAV" - ] + "type": "string" }, "id": { "type": "integer" diff --git a/cmd/server/docs/swagger.json b/cmd/server/docs/swagger.json index 8d5fce1ef..50d1c909d 100644 --- a/cmd/server/docs/swagger.json +++ b/cmd/server/docs/swagger.json @@ -9516,6 +9516,39 @@ } } }, + "/settings/backup/record/search/bycronjob": { + "post": { + "security": [ + { + "ApiKeyAuth": [] + } + ], + "description": "通过计划任务获取备份记录列表分页", + "consumes": [ + "application/json" + ], + "tags": [ + "Backup Account" + ], + "summary": "Page backup records by cronjob", + "parameters": [ + { + "description": "request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/dto.RecordSearchByCronjob" + } + } + ], + "responses": { + "200": { + "description": "OK" + } + } + } + }, "/settings/backup/recover": { "post": { "security": [ @@ -14930,9 +14963,6 @@ "exclusionRules": { "type": "string" }, - "keepLocal": { - "type": "boolean" - }, "name": { "type": "string" }, @@ -14949,6 +14979,9 @@ "spec": { "type": "string" }, + "targetAccountIDs": { + "type": "string" + }, "targetDirID": { "type": "integer" }, @@ -15004,9 +15037,6 @@ "id": { "type": "integer" }, - "keepLocal": { - "type": "boolean" - }, "name": { "type": "string" }, @@ -15023,6 +15053,9 @@ "spec": { "type": "string" }, + "targetAccountIDs": { + "type": "string" + }, "targetDirID": { "type": "integer" }, @@ -17225,6 +17258,25 @@ } } }, + "dto.RecordSearchByCronjob": { + "type": "object", + "required": [ + "cronjobID", + "page", + "pageSize" + ], + "properties": { + "cronjobID": { + "type": "integer" + }, + "page": { + "type": "integer" + }, + "pageSize": { + "type": "integer" + } + } + }, "dto.RedisConf": { "type": "object", "properties": { @@ -17841,17 +17893,7 @@ "maxLength": 256 }, "from": { - "type": "string", - "enum": [ - "OSS", - "S3", - "SFTP", - "MINIO", - "COS", - "KODO", - "OneDrive", - "WebDAV" - ] + "type": "string" }, "id": { "type": "integer" diff --git a/cmd/server/docs/swagger.yaml b/cmd/server/docs/swagger.yaml index 293882023..2ad3cd1f1 100644 --- a/cmd/server/docs/swagger.yaml +++ b/cmd/server/docs/swagger.yaml @@ -588,8 +588,6 @@ definitions: type: string exclusionRules: type: string - keepLocal: - type: boolean name: type: string retainCopies: @@ -601,6 +599,8 @@ definitions: type: string spec: type: string + targetAccountIDs: + type: string targetDirID: type: integer type: @@ -638,8 +638,6 @@ definitions: type: string id: type: integer - keepLocal: - type: boolean name: type: string retainCopies: @@ -651,6 +649,8 @@ definitions: type: string spec: type: string + targetAccountIDs: + type: string targetDirID: type: integer url: @@ -2147,6 +2147,19 @@ definitions: - pageSize - type type: object + dto.RecordSearchByCronjob: + properties: + cronjobID: + type: integer + page: + type: integer + pageSize: + type: integer + required: + - cronjobID + - page + - pageSize + type: object dto.RedisConf: properties: containerName: @@ -2555,15 +2568,6 @@ definitions: maxLength: 256 type: string from: - enum: - - OSS - - S3 - - SFTP - - MINIO - - COS - - KODO - - OneDrive - - WebDAV type: string id: type: integer @@ -11008,6 +11012,26 @@ paths: summary: Page backup records tags: - Backup Account + /settings/backup/record/search/bycronjob: + post: + consumes: + - application/json + description: 通过计划任务获取备份记录列表分页 + parameters: + - description: request + in: body + name: request + required: true + schema: + $ref: '#/definitions/dto.RecordSearchByCronjob' + responses: + "200": + description: OK + security: + - ApiKeyAuth: [] + summary: Page backup records by cronjob + tags: + - Backup Account /settings/backup/recover: post: consumes: diff --git a/frontend/src/api/interface/backup.ts b/frontend/src/api/interface/backup.ts index 3624cbac6..80e7b538c 100644 --- a/frontend/src/api/interface/backup.ts +++ b/frontend/src/api/interface/backup.ts @@ -50,6 +50,9 @@ export namespace Backup { name: string; detailName: string; } + export interface SearchBackupRecordByCronjob extends ReqPage { + cronjobID: number; + } export interface Backup { type: string; name: string; diff --git a/frontend/src/api/interface/cronjob.ts b/frontend/src/api/interface/cronjob.ts index 23460df25..0e84ec9f7 100644 --- a/frontend/src/api/interface/cronjob.ts +++ b/frontend/src/api/interface/cronjob.ts @@ -18,9 +18,9 @@ export namespace Cronjob { dbName: string; url: string; sourceDir: string; - keepLocal: boolean; targetDirID: number; - targetDir: string; + targetAccountIDs: string; + targetAccountIDList: Array; retainCopies: number; status: string; } @@ -37,8 +37,8 @@ export namespace Cronjob { dbName: string; url: string; sourceDir: string; - keepLocal: boolean; targetDirID: number; + targetAccountIDs: string; retainCopies: number; } export interface SpecObj { @@ -60,8 +60,8 @@ export namespace Cronjob { dbName: string; url: string; sourceDir: string; - keepLocal: boolean; targetDirID: number; + targetAccountIDs: string; retainCopies: number; } export interface CronjobDelete { diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index c3fa8f02e..efc1080f1 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -96,6 +96,9 @@ export const deleteBackupRecord = (params: { ids: number[] }) => { export const searchBackupRecords = (params: Backup.SearchBackupRecord) => { return http.post>(`/settings/backup/record/search`, params); }; +export const searchBackupRecordsByCronjob = (params: Backup.SearchBackupRecordByCronjob) => { + return http.post>(`/settings/backup/record/search/bycronjob`, params); +}; export const getBackupList = () => { return http.get>(`/settings/backup/search`); diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 322696eb1..3c99659f0 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -812,6 +812,7 @@ const message = { allOptionHelper: 'The current task plan is to back up all [{0}]. Direct download is not supported at the moment. You can check the backup list of [{0}] menu.', exclusionRules: 'Exclusive rule', + default_download_path: 'Default Download Link', saveLocal: 'Retain local backups (the same as the number of cloud storage copies)', url: 'URL Address', target: 'Target', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 656cf4891..7040521c2 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -773,6 +773,7 @@ const message = { snapshot: '系統快照', allOptionHelper: '當前計劃任務為備份所有【{0}】，暫不支持直接下載，可在【{0}】備份列表中查看', exclusionRules: '排除規則', + default_download_path: '默認下載地址', saveLocal: '同時保留本地備份（和雲存儲保留份數一致）', url: 'URL 地址', target: '備份到', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 6dce6281c..d4f65fb21 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -774,6 +774,7 @@ const message = { snapshot: '系统快照', allOptionHelper: '当前计划任务为备份所有【{0}】，暂不支持直接下载，可在【{0}】备份列表中查看', exclusionRules: '排除规则', + default_download_path: '默认下载地址', saveLocal: '同时保留本地备份（和云存储保留份数一致）', url: 'URL 地址', target: '备份到', diff --git a/frontend/src/views/cronjob/backup/index.vue b/frontend/src/views/cronjob/backup/index.vue new file mode 100644 index 000000000..3886d3a3e --- /dev/null +++ b/frontend/src/views/cronjob/backup/index.vue @@ -0,0 +1,127 @@ + + + diff --git a/frontend/src/views/cronjob/helper.ts b/frontend/src/views/cronjob/helper.ts index 888272740..b9661c6ed 100644 --- a/frontend/src/views/cronjob/helper.ts +++ b/frontend/src/views/cronjob/helper.ts @@ -2,6 +2,52 @@ import { Cronjob } from '@/api/interface/cronjob'; import i18n from '@/lang'; import { loadZero } from '@/utils/util'; +export const shortcuts = [ + { + text: i18n.global.t('monitor.today'), + value: () => { + const end = new Date(new Date().setHours(23, 59, 59, 999)); + const start = new Date(new Date().setHours(0, 0, 0, 0)); + return [start, end]; + }, + }, + { + text: i18n.global.t('monitor.yesterday'), + value: () => { + const itemDate = new Date(new Date().getTime() - 3600 * 1000 * 24 * 1); + const end = new Date(itemDate.setHours(23, 59, 59, 999)); + const start = new Date(itemDate.setHours(0, 0, 0, 0)); + return [start, end]; + }, + }, + { + text: i18n.global.t('monitor.lastNDay', [3]), + value: () => { + const itemDate = new Date(new Date().getTime() - 3600 * 1000 * 24 * 3); + const end = new Date(new Date().setHours(23, 59, 59, 999)); + const start = new Date(itemDate.setHours(0, 0, 0, 0)); + return [start, end]; + }, + }, + { + text: i18n.global.t('monitor.lastNDay', [7]), + value: () => { + const itemDate = new Date(new Date().getTime() - 3600 * 1000 * 24 * 7); + const end = new Date(new Date().setHours(23, 59, 59, 999)); + const start = new Date(itemDate.setHours(0, 0, 0, 0)); + return [start, end]; + }, + }, + { + text: i18n.global.t('monitor.lastNDay', [30]), + value: () => { + const itemDate = new Date(new Date().getTime() - 3600 * 1000 * 24 * 30); + const end = new Date(new Date().setHours(23, 59, 59, 999)); + const start = new Date(itemDate.setHours(0, 0, 0, 0)); + return [start, end]; + }, + }, +]; export const specOptions = [ { label: i18n.global.t('cronjob.perMonth'), value: 'perMonth' }, { label: i18n.global.t('cronjob.perWeek'), value: 'perWeek' }, diff --git a/frontend/src/views/cronjob/index.vue b/frontend/src/views/cronjob/index.vue index 048807ced..149e1578a 100644 --- a/frontend/src/views/cronjob/index.vue +++ b/frontend/src/views/cronjob/index.vue @@ -100,8 +100,14 @@ - - + + + @@ -146,6 +176,7 @@ import TableSetting from '@/components/table-setting/index.vue'; import Tooltip from '@/components/tooltip/index.vue'; import OperateDialog from '@/views/cronjob/operate/index.vue'; import Records from '@/views/cronjob/record/index.vue'; +import Backups from '@/views/cronjob/backup/index.vue'; import { onMounted, reactive, ref } from 'vue'; import { deleteCronjob, getCronjobPage, handleOnce, updateStatus } from '@/api/modules/cronjob'; import i18n from '@/lang'; @@ -190,9 +221,16 @@ const search = async (column?: any) => { loading.value = false; data.value = res.data.items || []; for (const item of data.value) { - if (item.targetDir !== '-' && item.targetDir !== '') { - item.targetDir = i18n.global.t('setting.' + item.targetDir); + item.targetAccounts = item.targetAccounts.split(',') || []; + let accounts = []; + for (const account of item.targetAccounts) { + if (account == item.targetDir) { + accounts.unshift(account); + } else { + accounts.push(account); + } } + item.targetAccounts = accounts.join(','); } paginationConfig.total = res.data.total; }) @@ -202,6 +240,7 @@ const search = async (column?: any) => { }; const dialogRecordRef = ref(); +const dialogBackupRef = ref(); const dialogRef = ref(); const onOpenDialog = async ( @@ -218,7 +257,6 @@ const onOpenDialog = async ( }, ], type: 'shell', - keepLocal: true, retainCopies: 7, }, ) => { @@ -294,6 +332,10 @@ const onBatchChangeStatus = async (status: string) => { }); }; +const loadBackups = async (row: any) => { + dialogBackupRef.value!.acceptParams({ cronjobID: row.id, cronjob: row.name }); +}; + const onHandle = async (row: Cronjob.CronjobInfo) => { loading.value = true; await handleOnce(row.id) diff --git a/frontend/src/views/cronjob/operate/index.vue b/frontend/src/views/cronjob/operate/index.vue index c2b243280..0ecd55701 100644 --- a/frontend/src/views/cronjob/operate/index.vue +++ b/frontend/src/views/cronjob/operate/index.vue @@ -114,10 +114,11 @@ type="primary" style="float: right; margin-top: 5px" @click="handleSpecDelete(index)" + v-if="dialogData.rowData.specObjs.length > 1" > {{ $t('commons.button.delete') }} - + {{ $t('commons.button.add') }} @@ -239,14 +240,15 @@
- - + +
- +
@@ -261,12 +263,12 @@ - - - {{ $t('cronjob.saveLocal') }} - + + +
+ +
+
@@ -356,6 +358,13 @@ const acceptParams = (params: DialogProps): void => { changeType(); dialogData.value.rowData.dbType = 'mysql'; } + if (dialogData.value.rowData.targetAccountIDs) { + dialogData.value.rowData.targetAccountIDList = []; + let ids = dialogData.value.rowData.targetAccountIDs.split(','); + for (const id of ids) { + dialogData.value.rowData.targetAccountIDList.push(Number(id)); + } + } title.value = i18n.global.t('cronjob.' + dialogData.value.title); if (dialogData.value?.rowData?.exclusionRules) { dialogData.value.rowData.exclusionRules = dialogData.value.rowData.exclusionRules.replaceAll(',', '\n'); @@ -389,6 +398,7 @@ const localDirID = ref(); const containerOptions = ref([]); const websiteOptions = ref([]); const backupOptions = ref([]); +const accountOptions = ref([]); const appOptions = ref([]); const dbInfo = reactive({ @@ -399,6 +409,9 @@ const dbInfo = reactive({ }); const verifySpec = (rule: any, value: any, callback: any) => { + if (dialogData.value.rowData!.specObjs.length === 0) { + callback(new Error(i18n.global.t('cronjob.cronSpecRule'))); + } for (const item of dialogData.value.rowData!.specObjs) { switch (item.specType) { case 'perMonth': @@ -451,6 +464,7 @@ const rules = reactive({ dbName: [Rules.requiredSelect], url: [Rules.requiredInput], sourceDir: [Rules.requiredInput], + targetAccountIDList: [Rules.requiredSelect], targetDirID: [Rules.requiredSelect, Rules.number], retainCopies: [Rules.number], }); @@ -475,17 +489,6 @@ const loadDatabases = async (dbType: string) => { }; const changeType = () => { - if (dialogData.value.rowData.type === 'snapshot') { - dialogData.value.rowData.keepLocal = false; - dialogData.value.rowData.targetDirID = null; - for (const item of backupOptions.value) { - if (item.label !== i18n.global.t('setting.LOCAL')) { - dialogData.value.rowData.targetDirID = item.value; - break; - } - } - } - dialogData.value.rowData!.specObjs = [loadDefaultSpec(dialogData.value.rowData.type)]; }; @@ -514,12 +517,29 @@ const loadBackups = async () => { } if (item.type === 'LOCAL') { localDirID.value = item.id; - if (!dialogData.value.rowData!.targetDirID) { - dialogData.value.rowData!.targetDirID = item.id; + if (!dialogData.value.rowData!.targetAccountIDList) { + dialogData.value.rowData!.targetAccountIDList = [item.id]; } } backupOptions.value.push({ label: i18n.global.t('setting.' + item.type), value: item.id }); } + changeAccount(); +}; + +const changeAccount = async () => { + accountOptions.value = []; + for (const item of backupOptions.value) { + let exit = false; + for (const ac of dialogData.value.rowData.targetAccountIDList) { + if (item.value == ac) { + exit = true; + break; + } + } + if (exit) { + accountOptions.value.push(item); + } + } }; const loadAppInstalls = async () => { @@ -566,6 +586,7 @@ const onSubmit = async (formEl: FormInstance | undefined) => { } specs.push(itemSpec); } + dialogData.value.rowData.targetAccountIDs = dialogData.value.rowData.targetAccountIDList.join(','); dialogData.value.rowData.spec = specs.join(','); if (!formEl) return; formEl.validate(async (valid) => { diff --git a/frontend/src/views/cronjob/record/index.vue b/frontend/src/views/cronjob/record/index.vue index 98f26dd29..a4e44ee8d 100644 --- a/frontend/src/views/cronjob/record/index.vue +++ b/frontend/src/views/cronjob/record/index.vue @@ -27,48 +27,6 @@ {{ $t('commons.status.stopped') }} - - - {{ $t('cronjob.' + dialogData.rowData?.specType) }}  - - {{ $t('cronjob.per') }} - - {{ dialogData.rowData?.day }}{{ $t('cronjob.day') }}  - {{ loadZero(dialogData.rowData?.hour) }} : - {{ loadZero(dialogData.rowData?.minute) }} - - - {{ loadZero(dialogData.rowData?.hour) }} : {{ loadZero(dialogData.rowData?.minute) }} - - - {{ loadWeek(dialogData.rowData?.week) }}  {{ loadZero(dialogData.rowData?.hour) }} : - {{ loadZero(dialogData.rowData?.minute) }} - - - {{ dialogData.rowData?.day }}{{ $t('commons.units.day') }},  - {{ loadZero(dialogData.rowData?.hour) }} : - {{ loadZero(dialogData.rowData?.minute) }} - - - {{ dialogData.rowData?.hour }}{{ $t('commons.units.hour') }},  - {{ loadZero(dialogData.rowData?.minute) }} - - -  {{ loadZero(dialogData.rowData?.minute) }} - - -  {{ dialogData.rowData?.minute }}{{ $t('commons.units.minute') }} - - -  {{ dialogData.rowData?.second }}{{ $t('commons.units.second') }} - -  {{ $t('cronjob.handle') }} - {{ $t('commons.button.handle') }} @@ -172,119 +130,6 @@ - - - - {{ dialogData.rowData!.targetDir }} - - {{ $t('file.download') }} - - - - - - {{ dialogData.rowData!.appID }} - - - {{ $t('commons.table.all') }} - - - - - - {{ dialogData.rowData!.website }} - - - {{ $t('commons.table.all') }} - - - - - - {{ $t('cronjob.logHelper') }} - - - - - - {{ dialogData.rowData!.dbName }} - - - {{ $t('commons.table.all') }} - - - - - - {{ dialogData.rowData!.sourceDir }} - -
- - - -
- - - - {{ dialogData.rowData!.retainCopies }} - - - - - - - {{ dialogData.rowData!.exclusionRules }} - -
- - - -
-