From f196d029cbaa00fa9bc2e63b767d723963a178dc Mon Sep 17 00:00:00 2001 From: ssongliu <73214554+ssongliu@users.noreply.github.com> Date: Wed, 23 Aug 2023 17:16:18 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E9=87=8D=E6=9E=84=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E5=8F=96=E6=B6=88=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=E8=BF=87=E7=A8=8B=20loading=20(#2039)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/v1/snapshot.go | 26 ++ backend/app/dto/setting.go | 15 + backend/app/model/snapshot.go | 16 + backend/app/repo/runtime.go | 13 + backend/app/repo/snapshot.go | 36 ++ backend/app/service/database_mysql.go | 2 +- backend/app/service/snapshot.go | 297 +++++++-------- backend/app/service/snapshot_create.go | 224 ++++++++++++ backend/constant/status.go | 1 + backend/init/hook/hook.go | 54 +++ backend/init/migration/migrations/init.go | 4 +- backend/router/ro_setting.go | 1 + frontend/src/api/interface/setting.ts | 14 + frontend/src/api/modules/setting.ts | 3 + frontend/src/lang/modules/en.ts | 13 + frontend/src/lang/modules/tw.ts | 13 + frontend/src/lang/modules/zh.ts | 13 + frontend/src/views/setting/snapshot/index.vue | 33 +- .../setting/snapshot/snap_status/index.vue | 338 ++++++++++++++++++ 19 files changed, 936 insertions(+), 180 deletions(-) create mode 100644 backend/app/service/snapshot_create.go create mode 100644 frontend/src/views/setting/snapshot/snap_status/index.vue diff --git a/backend/app/api/v1/snapshot.go b/backend/app/api/v1/snapshot.go index bc3b5edb2..b09eccdd7 100644 --- a/backend/app/api/v1/snapshot.go +++ b/backend/app/api/v1/snapshot.go @@ -60,6 +60,32 @@ func (b *BaseApi) ImportSnapshot(c *gin.Context) { helper.SuccessWithData(c, nil) } +// @Tags System Setting +// @Summary Load Snapshot status +// @Description 获取快照状态 +// @Accept json +// @Param request body dto.OperateByID true "request" +// @Success 200 +// @Security ApiKeyAuth +// @Router /settings/snapshot/status [post] +func (b *BaseApi) LoadSnapShotStatus(c *gin.Context) { + var req dto.OperateByID + if err := c.ShouldBindJSON(&req); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + if err := global.VALID.Struct(req); err != nil { + helper.ErrorWithDetail(c, constant.CodeErrBadRequest, constant.ErrTypeInvalidParams, err) + return + } + data, err := snapshotService.LoadSnapShotStatus(req.ID) + if err != nil { + helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err) + return + } + helper.SuccessWithData(c, data) +} + // @Tags System Setting // @Summary Update snapshot description // @Description 更新快照描述信息 diff --git a/backend/app/dto/setting.go b/backend/app/dto/setting.go index aa12157d9..7f7181900 100644 --- a/backend/app/dto/setting.go +++ b/backend/app/dto/setting.go @@ -75,7 +75,22 @@ type PortUpdate struct { ServerPort uint `json:"serverPort" validate:"required,number,max=65535,min=1"` } +type SnapshotStatus struct { + Panel string `json:"panel"` + PanelCtl string `json:"panelCtl"` + PanelService string `json:"panelService"` + PanelInfo string `json:"panelInfo"` + DaemonJson string `json:"daemonJson"` + AppData string `json:"appData"` + PanelData string `json:"panelData"` + BackupData string `json:"backupData"` + + Compress string `json:"compress"` + Upload string `json:"upload"` +} + type SnapshotCreate struct { + ID uint `json:"id"` From string `json:"from" validate:"required,oneof=OSS S3 SFTP MINIO COS KODO OneDrive"` Description string `json:"description" validate:"max=256"` } diff --git a/backend/app/model/snapshot.go b/backend/app/model/snapshot.go index ce2e1c39e..c56c9cfdb 100644 --- a/backend/app/model/snapshot.go +++ b/backend/app/model/snapshot.go @@ -17,3 +17,19 @@ type Snapshot struct { RollbackMessage string `json:"rollbackMessage" gorm:"type:varchar(256)"` LastRollbackedAt string `json:"lastRollbackedAt" gorm:"type:varchar(64)"` } + +type SnapshotStatus struct { + BaseModel + SnapID uint `gorm:"type:decimal" json:"snapID"` + Panel string `json:"panel" gorm:"type:varchar(64);default:Running"` + PanelCtl string `json:"panelCtl" gorm:"type:varchar(64);default:Running"` + PanelService string `json:"panelService" gorm:"type:varchar(64);default:Running"` + PanelInfo string `json:"panelInfo" gorm:"type:varchar(64);default:Running"` + DaemonJson string `json:"daemonJson" gorm:"type:varchar(64);default:Running"` + AppData string `json:"appData" gorm:"type:varchar(64);default:Running"` + PanelData string `json:"panelData" gorm:"type:varchar(64);default:Running"` + BackupData string `json:"backupData" gorm:"type:varchar(64);default:Running"` + + Compress string `json:"compress" gorm:"type:varchar(64);default:Waiting"` + Upload string `json:"upload" gorm:"type:varchar(64);default:Waiting"` +} diff --git a/backend/app/repo/runtime.go b/backend/app/repo/runtime.go index 00470a8a9..614f30a69 100644 --- a/backend/app/repo/runtime.go +++ b/backend/app/repo/runtime.go @@ -2,7 +2,9 @@ package repo import ( "context" + "github.com/1Panel-dev/1Panel/backend/app/model" + "github.com/1Panel-dev/1Panel/backend/global" "gorm.io/gorm" ) @@ -20,6 +22,7 @@ type IRuntimeRepo interface { Save(runtime *model.Runtime) error DeleteBy(opts ...DBOption) error GetFirst(opts ...DBOption) (*model.Runtime, error) + List(opts ...DBOption) ([]model.Runtime, error) } func NewIRunTimeRepo() IRuntimeRepo { @@ -65,6 +68,16 @@ func (r *RuntimeRepo) Page(page, size int, opts ...DBOption) (int64, []model.Run return count, runtimes, err } +func (r *RuntimeRepo) List(opts ...DBOption) ([]model.Runtime, error) { + var runtimes []model.Runtime + db := global.DB.Model(&model.Runtime{}) + for _, opt := range opts { + db = opt(db) + } + err := db.Find(&runtimes).Error + return runtimes, err +} + func (r *RuntimeRepo) Create(ctx context.Context, runtime *model.Runtime) error { db := getTx(ctx).Model(&model.Runtime{}) return db.Create(&runtime).Error diff --git a/backend/app/repo/snapshot.go b/backend/app/repo/snapshot.go index 931fd4e2d..0b6eec6f8 100644 --- a/backend/app/repo/snapshot.go +++ b/backend/app/repo/snapshot.go @@ -12,6 +12,12 @@ type ISnapshotRepo interface { Update(id uint, vars map[string]interface{}) error Page(limit, offset int, opts ...DBOption) (int64, []model.Snapshot, error) Delete(opts ...DBOption) error + + GetStatus(snapID uint) (model.SnapshotStatus, error) + GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) + CreateStatus(snap *model.SnapshotStatus) error + DeleteStatus(snapID uint) error + UpdateStatus(id uint, vars map[string]interface{}) error } func NewISnapshotRepo() ISnapshotRepo { @@ -67,3 +73,33 @@ func (u *SnapshotRepo) Delete(opts ...DBOption) error { } return db.Delete(&model.Snapshot{}).Error } + +func (u *SnapshotRepo) GetStatus(snapID uint) (model.SnapshotStatus, error) { + var data model.SnapshotStatus + if err := global.DB.Where("snap_id = ?", snapID).First(&data).Error; err != nil { + return data, err + } + return data, nil +} + +func (u *SnapshotRepo) GetStatusList(opts ...DBOption) ([]model.SnapshotStatus, error) { + var status []model.SnapshotStatus + db := global.DB.Model(&model.SnapshotStatus{}) + for _, opt := range opts { + db = opt(db) + } + err := db.Find(&status).Error + return status, err +} + +func (u *SnapshotRepo) CreateStatus(snap *model.SnapshotStatus) error { + return global.DB.Create(snap).Error +} + +func (u *SnapshotRepo) DeleteStatus(snapID uint) error { + return global.DB.Where("snap_id = ?", snapID).Delete(&model.SnapshotStatus{}).Error +} + +func (u *SnapshotRepo) UpdateStatus(id uint, vars map[string]interface{}) error { + return global.DB.Model(&model.SnapshotStatus{}).Where("id = ?", id).Updates(vars).Error +} diff --git a/backend/app/service/database_mysql.go b/backend/app/service/database_mysql.go index 76150698f..ea3f89dea 100644 --- a/backend/app/service/database_mysql.go +++ b/backend/app/service/database_mysql.go @@ -234,7 +234,7 @@ func (u *MysqlService) Delete(ctx context.Context, req dto.MysqlDBDelete) error Username: db.Username, Permission: db.Permission, Timeout: 300, - }); err != nil { + }); err != nil && !req.ForceDelete { return err } diff --git a/backend/app/service/snapshot.go b/backend/app/service/snapshot.go index 85f1ca433..82ad97bf7 100644 --- a/backend/app/service/snapshot.go +++ b/backend/app/service/snapshot.go @@ -9,6 +9,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/1Panel-dev/1Panel/backend/app/dto" @@ -34,6 +35,8 @@ type ISnapshotService interface { SnapshotImport(req dto.SnapshotImport) error Delete(req dto.BatchDeleteReq) error + LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) + UpdateDescription(req dto.UpdateDescription) error readFromJson(path string) (SnapshotJson, error) } @@ -99,6 +102,18 @@ func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error { return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description}) } +func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) { + var data dto.SnapshotStatus + status, err := snapshotRepo.GetStatus(id) + if err != nil { + return nil, err + } + if err := copier.Copy(&data, &status); err != nil { + return nil, errors.WithMessage(constant.ErrStructTransform, err.Error()) + } + return &data, nil +} + type SnapshotJson struct { OldBaseDir string `json:"oldBaseDir"` OldDockerDataDir string `json:"oldDockerDataDir"` @@ -113,112 +128,103 @@ type SnapshotJson struct { } func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error { - global.LOG.Info("start to create snapshot now") localDir, err := loadLocalDir() if err != nil { return err } - backup, err := backupRepo.Get(commonRepo.WithByType(req.From)) - if err != nil { - return err - } - backupAccount, err := NewIBackupService().NewClient(&backup) - if err != nil { - return err - } - timeNow := time.Now().Format("20060102150405") - versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) - rootDir := path.Join(localDir, fmt.Sprintf("system/1panel_%s_%s", versionItem.Value, timeNow)) - backupPanelDir := fmt.Sprintf("%s/1panel", rootDir) - _ = os.MkdirAll(backupPanelDir, os.ModePerm) - backupDockerDir := fmt.Sprintf("%s/docker", rootDir) - _ = os.MkdirAll(backupDockerDir, os.ModePerm) + var ( + snap model.Snapshot + snapStatus model.SnapshotStatus + rootDir string + ) - _ = settingRepo.Update("SystemStatus", "Snapshoting") - snap := model.Snapshot{ - Name: fmt.Sprintf("1panel_%s_%s", versionItem.Value, timeNow), - Description: req.Description, - From: req.From, - Version: versionItem.Value, - Status: constant.StatusSuccess, - } - _ = snapshotRepo.Create(&snap) - go func() { - _ = global.Cron.Stop() - defer func() { - global.Cron.Start() - _ = os.RemoveAll(rootDir) - }() - fileOp := files.NewFileOp() - - snapJson := SnapshotJson{ - BaseDir: global.CONF.System.BaseDir, - BackupDataDir: localDir, - } + if req.ID == 0 { + timeNow := time.Now().Format("20060102150405") + versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion")) + rootDir = path.Join(localDir, fmt.Sprintf("system/1panel_%s_%s", versionItem.Value, timeNow)) - if err := u.handleDockerDatasWithSave(fileOp, "snapshot", "", backupDockerDir); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if err := u.handleDaemonJson(fileOp, "snapshot", "", backupDockerDir); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - - if err := u.handlePanelBinary(fileOp, "snapshot", "", backupPanelDir+"/1panel"); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - if err := u.handlePanelctlBinary(fileOp, "snapshot", "", backupPanelDir+"/1pctl"); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return + snap = model.Snapshot{ + Name: fmt.Sprintf("1panel_%s_%s", versionItem.Value, timeNow), + Description: req.Description, + From: req.From, + Version: versionItem.Value, + Status: constant.StatusWaiting, } - if err := u.handlePanelService(fileOp, "snapshot", "", backupPanelDir+"/1panel.service"); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return + _ = snapshotRepo.Create(&snap) + snapStatus.SnapID = snap.ID + _ = snapshotRepo.CreateStatus(&snapStatus) + } else { + snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID)) + if err != nil { + return err } - - if err := u.handleBackupDatas(fileOp, "snapshot", localDir, backupPanelDir); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return + snapStatus, _ = snapshotRepo.GetStatus(snap.ID) + if snapStatus.ID == 0 { + snapStatus.SnapID = snap.ID + _ = snapshotRepo.CreateStatus(&snapStatus) } + rootDir = path.Join(localDir, fmt.Sprintf("system/%s", snap.Name)) + } - dataDir := path.Join(global.CONF.System.BaseDir, "1panel") - if err := u.handlePanelDatas(snap.ID, fileOp, "snapshot", dataDir, backupPanelDir, localDir, snapJson.DockerDataDir); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } - _, _ = cmd.Exec("systemctl restart docker") - snapJson.PanelDataDir = dataDir + var wg sync.WaitGroup + itemHelper := snapHelper{SnapID: snap.ID, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()} + backupPanelDir := path.Join(rootDir, "1panel") + _ = os.MkdirAll(backupPanelDir, os.ModePerm) + backupDockerDir := path.Join(rootDir, "docker") + _ = os.MkdirAll(backupDockerDir, os.ModePerm) - if err := u.saveJson(snapJson, rootDir); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, fmt.Sprintf("save snapshot json failed, err: %v", err)) - return - } + jsonItem := SnapshotJson{ + BaseDir: global.CONF.System.BaseDir, + BackupDataDir: localDir, + PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"), + } - if err := handleTar(rootDir, path.Join(localDir, "system"), fmt.Sprintf("1panel_%s_%s.tar.gz", versionItem.Value, timeNow), ""); err != nil { - updateSnapshotStatus(snap.ID, constant.StatusFailed, err.Error()) - return - } + if snapStatus.PanelInfo != constant.StatusDone { + wg.Add(1) + go snapJson(itemHelper, snapStatus.ID, jsonItem, rootDir) + } + if snapStatus.Panel != constant.StatusDone { + wg.Add(1) + go snapPanel(itemHelper, snapStatus.ID, backupPanelDir) + } + if snapStatus.PanelCtl != constant.StatusDone { + wg.Add(1) + go snapPanelCtl(itemHelper, snapStatus.ID, backupPanelDir) + } + if snapStatus.PanelService != constant.StatusDone { + wg.Add(1) + go snapPanelService(itemHelper, snapStatus.ID, backupPanelDir) + } + if snapStatus.DaemonJson != constant.StatusDone { + wg.Add(1) + go snapDaemonJson(itemHelper, snapStatus.ID, backupDockerDir) + } + if snapStatus.AppData != constant.StatusDone { + wg.Add(1) + go snapAppData(itemHelper, snapStatus.ID, backupDockerDir) + } + if snapStatus.BackupData != constant.StatusDone { + wg.Add(1) + go snapBackup(itemHelper, snapStatus.ID, localDir, backupPanelDir) + } + if snapStatus.PanelData != constant.StatusDone { + wg.Add(1) + go snapPanelData(itemHelper, snapStatus.ID, localDir, backupPanelDir) + } - _ = settingRepo.Update("SystemStatus", "Free") + go func() { + wg.Wait() + if checkIsAllDone(snap.ID) { + snapCompress(itemHelper, snapStatus.ID, rootDir) - global.LOG.Infof("start to upload snapshot to %s, please wait", backup.Type) - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusUploading}) - localPath := path.Join(localDir, fmt.Sprintf("system/1panel_%s_%s.tar.gz", versionItem.Value, timeNow)) - itemBackupPath := strings.TrimPrefix(backup.BackupPath, "/") - itemBackupPath = strings.TrimSuffix(itemBackupPath, "/") - if ok, err := backupAccount.Upload(localPath, fmt.Sprintf("%s/system_snapshot/1panel_%s_%s.tar.gz", itemBackupPath, versionItem.Value, timeNow)); err != nil || !ok { - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed, "message": err.Error()}) - global.LOG.Errorf("upload snapshot to %s failed, err: %v", backup.Type, err) - return + snapUpload(req.From, snapStatus.ID, fmt.Sprintf("%s.tar.gz", rootDir)) + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) + } else { + _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed}) } - _ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) - _ = os.RemoveAll(path.Join(localDir, fmt.Sprintf("system/1panel_%s_%s.tar.gz", versionItem.Value, timeNow))) - - global.LOG.Infof("upload snapshot to %s success", backup.Type) }() + return nil } @@ -556,11 +562,11 @@ func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) { func (u *SnapshotService) handleDockerDatas(fileOp files.FileOp, operation string, source, target string) error { switch operation { case "snapshot": - if err := u.handleTar(source, target, "docker_data.tar.gz", ""); err != nil { + if err := handleSnapTar(source, target, "docker_data.tar.gz", ""); err != nil { return fmt.Errorf("backup docker data failed, err: %v", err) } case "recover": - if err := u.handleTar(target, u.OriginalPath, "docker_data.tar.gz", ""); err != nil { + if err := handleSnapTar(target, u.OriginalPath, "docker_data.tar.gz", ""); err != nil { return fmt.Errorf("backup docker data failed, err: %v", err) } if err := u.handleUnTar(source+"/docker/docker_data.tar.gz", target); err != nil { @@ -581,31 +587,6 @@ func (u *SnapshotService) handleDockerDatas(fileOp files.FileOp, operation strin func (u *SnapshotService) handleDockerDatasWithSave(fileOp files.FileOp, operation, source, target string) error { switch operation { - case "snapshot": - appInstalls, err := appInstallRepo.ListBy() - if err != nil { - return err - } - imageRegex := regexp.MustCompile(`image:\s*(.*)`) - var imageSaveList []string - existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") - existImages := strings.Split(existStr, "\n") - duplicateMap := make(map[string]bool) - for _, app := range appInstalls { - matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) - for _, match := range matches { - for _, existImage := range existImages { - if match[1] == existImage && !duplicateMap[match[1]] { - imageSaveList = append(imageSaveList, match[1]) - duplicateMap[match[1]] = true - } - } - } - } - std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(target, "docker_image.tar")) - if err != nil { - return errors.New(std) - } case "recover": if err := u.handleDockerDatasWithSave(fileOp, "snapshot", "", u.OriginalPath); err != nil { return fmt.Errorf("backup docker data failed, err: %v", err) @@ -731,12 +712,8 @@ func (u *SnapshotService) handlePanelService(fileOp files.FileOp, operation stri func (u *SnapshotService) handleBackupDatas(fileOp files.FileOp, operation string, source, target string) error { switch operation { - case "snapshot": - if err := u.handleTar(source, target, "1panel_backup.tar.gz", "./system;"); err != nil { - return fmt.Errorf("backup panel local backup dir data failed, err: %v", err) - } case "recover": - if err := u.handleTar(target, u.OriginalPath, "1panel_backup.tar.gz", "./system;"); err != nil { + if err := handleSnapTar(target, u.OriginalPath, "1panel_backup.tar.gz", "./system;"); err != nil { return fmt.Errorf("restore original local backup dir data failed, err: %v", err) } if err := u.handleUnTar(source+"/1panel/1panel_backup.tar.gz", target); err != nil { @@ -766,7 +743,7 @@ func (u *SnapshotService) handlePanelDatas(snapID uint, fileOp files.FileOp, ope exclusionRules += ("." + strings.ReplaceAll(dockerDir, source, "") + ";") } - if err := u.handleTar(source, target, "1panel_data.tar.gz", exclusionRules); err != nil { + if err := handleSnapTar(source, target, "1panel_data.tar.gz", exclusionRules); err != nil { return fmt.Errorf("backup panel data failed, err: %v", err) } case "recover": @@ -779,7 +756,7 @@ func (u *SnapshotService) handlePanelDatas(snapID uint, fileOp files.FileOp, ope } _ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": ""}) - if err := u.handleTar(target, u.OriginalPath, "1panel_data.tar.gz", exclusionRules); err != nil { + if err := handleSnapTar(target, u.OriginalPath, "1panel_data.tar.gz", exclusionRules); err != nil { return fmt.Errorf("restore original panel data failed, err: %v", err) } _ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": constant.StatusWaiting}) @@ -825,6 +802,7 @@ func (u *SnapshotService) Delete(req dto.BatchDeleteReq) error { if _, err := os.Stat(itemFile); err == nil { _ = os.Remove(itemFile) } + _ = snapshotRepo.DeleteStatus(snap.ID) } if err := snapshotRepo.Delete(commonRepo.WithIdsIn(req.Ids)); err != nil { return err @@ -833,18 +811,6 @@ func (u *SnapshotService) Delete(req dto.BatchDeleteReq) error { return nil } -func updateSnapshotStatus(id uint, status string, message string) { - if status != constant.StatusSuccess { - global.LOG.Errorf("snapshot failed, err: %s", message) - } - if err := snapshotRepo.Update(id, map[string]interface{}{ - "status": status, - "message": message, - }); err != nil { - global.LOG.Errorf("update snap snapshot status failed, err: %v", err) - } - _ = settingRepo.Update("SystemStatus", "Free") -} func updateRecoverStatus(id uint, interruptStep, status string, message string) { if status != constant.StatusSuccess { global.LOG.Errorf("recover failed, err: %s", message) @@ -939,35 +905,6 @@ func (u *SnapshotService) updateLiveRestore(enabled bool) error { return nil } -func (u *SnapshotService) handleTar(sourceDir, targetDir, name, exclusionRules string) error { - if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { - if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { - return err - } - } - - exStr := "" - excludes := strings.Split(exclusionRules, ";") - for _, exclude := range excludes { - if len(exclude) == 0 { - continue - } - exStr += " --exclude " - exStr += exclude - } - - commands := fmt.Sprintf("tar --warning=no-file-changed -zcf %s %s -C %s .", targetDir+"/"+name, exStr, sourceDir) - global.LOG.Debug(commands) - stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) - if err != nil { - if len(stdout) != 0 { - global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) - } - } - return nil -} - func (u *SnapshotService) handleUnTar(sourceDir, targetDir string) error { if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { @@ -1035,3 +972,35 @@ func min(a, b int) int { } return b } + +func checkIsAllDone(snapID uint) bool { + status, err := snapshotRepo.GetStatus(snapID) + if err != nil { + return false + } + if status.Panel != constant.StatusDone { + return false + } + if status.PanelCtl != constant.StatusDone { + return false + } + if status.PanelService != constant.StatusDone { + return false + } + if status.PanelInfo != constant.StatusDone { + return false + } + if status.DaemonJson != constant.StatusDone { + return false + } + if status.AppData != constant.StatusDone { + return false + } + if status.PanelData != constant.StatusDone { + return false + } + if status.BackupData != constant.StatusDone { + return false + } + return true +} diff --git a/backend/app/service/snapshot_create.go b/backend/app/service/snapshot_create.go new file mode 100644 index 000000000..a11c0cb94 --- /dev/null +++ b/backend/app/service/snapshot_create.go @@ -0,0 +1,224 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path" + "regexp" + "strings" + "sync" + "time" + + "github.com/1Panel-dev/1Panel/backend/constant" + "github.com/1Panel-dev/1Panel/backend/global" + "github.com/1Panel-dev/1Panel/backend/utils/cmd" + "github.com/1Panel-dev/1Panel/backend/utils/files" +) + +type snapHelper struct { + SnapID uint + Ctx context.Context + FileOp files.FileOp + Wg *sync.WaitGroup +} + +func snapJson(snap snapHelper, statusID uint, snapJson SnapshotJson, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_info": constant.Running}) + status := constant.StatusDone + remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") + if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_info": status}) +} + +func snapPanel(snap snapHelper, statusID uint, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel": constant.Running}) + status := constant.StatusDone + if err := cpBinary("/usr/local/bin/1panel", path.Join(targetDir, "1panel")); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel": status}) +} + +func snapPanelCtl(snap snapHelper, statusID uint, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_ctl": constant.Running}) + status := constant.StatusDone + if err := cpBinary("/usr/local/bin/1pctl", path.Join(targetDir, "1pctl")); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_ctl": status}) +} + +func snapPanelService(snap snapHelper, statusID uint, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_service": constant.Running}) + status := constant.StatusDone + if err := cpBinary("/etc/systemd/system/1panel.service", path.Join(targetDir, "1panel.service")); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_service": status}) +} + +func snapDaemonJson(snap snapHelper, statusID uint, targetDir string) { + defer snap.Wg.Done() + if !snap.FileOp.Stat("/etc/docker/daemon.json") { + return + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"daemon_json": constant.Running}) + status := constant.StatusDone + if err := cpBinary("/etc/docker/daemon.json", path.Join(targetDir, "daemon.json")); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"daemon_json": status}) +} + +func snapAppData(snap snapHelper, statusID uint, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"app_data": constant.Running}) + appInstalls, err := appInstallRepo.ListBy() + if err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"app_data": err.Error()}) + return + } + runtimes, err := runtimeRepo.List() + if err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"app_data": err.Error()}) + return + } + imageRegex := regexp.MustCompile(`image:\s*(.*)`) + var imageSaveList []string + existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") + existImages := strings.Split(existStr, "\n") + duplicateMap := make(map[string]bool) + for _, app := range appInstalls { + matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) + for _, match := range matches { + for _, existImage := range existImages { + if match[1] == existImage && !duplicateMap[match[1]] { + imageSaveList = append(imageSaveList, match[1]) + duplicateMap[match[1]] = true + } + } + } + } + for _, rumtime := range runtimes { + for _, existImage := range existImages { + if rumtime.Image == existImage && !duplicateMap[rumtime.Image] { + imageSaveList = append(imageSaveList, rumtime.Image) + duplicateMap[rumtime.Image] = true + } + } + } + + global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) + std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) + if err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"app_data": std}) + return + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"app_data": constant.StatusDone}) +} + +func snapBackup(snap snapHelper, statusID uint, localDir, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"backup_data": constant.Running}) + status := constant.StatusDone + if err := handleSnapTar(localDir, targetDir, "1panel_backup.tar.gz", "./system;"); err != nil { + status = err.Error() + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"backup_data": status}) +} + +func snapPanelData(snap snapHelper, statusID uint, localDir, targetDir string) { + defer snap.Wg.Done() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_data": constant.Running}) + status := constant.StatusDone + dataDir := path.Join(global.CONF.System.BaseDir, "1panel") + exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" + if strings.Contains(localDir, dataDir) { + exclusionRules += ("." + strings.ReplaceAll(localDir, dataDir, "") + ";") + } + + _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": "OnSaveData"}) + if err := handleSnapTar(dataDir, targetDir, "1panel_data.tar.gz", exclusionRules); err != nil { + status = err.Error() + } + _ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": constant.StatusWaiting}) + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"panel_data": status}) +} + +func snapCompress(snap snapHelper, statusID uint, rootDir string) { + defer func() { + global.LOG.Debugf("remove snapshot file %s", rootDir) + _ = os.RemoveAll(rootDir) + }() + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"compress": constant.StatusRunning}) + tmpDir := path.Join(global.CONF.System.TmpDir, "system") + fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) + if err := snap.FileOp.Compress([]string{rootDir}, tmpDir, fileName, files.TarGz); err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"compress": err.Error()}) + return + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"compress": constant.StatusDone}) +} + +func snapUpload(account string, statusID uint, file string) { + source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) + defer func() { + global.LOG.Debugf("remove snapshot file %s", source) + _ = os.Remove(source) + }() + + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"upload": constant.StatusUploading}) + backup, err := backupRepo.Get(commonRepo.WithByType(account)) + if err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"upload": err.Error()}) + return + } + client, err := NewIBackupService().NewClient(&backup) + if err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"upload": err.Error()}) + return + } + target := path.Join(backup.BackupPath, "system_snapshot", path.Base(file)) + if _, err := client.Upload(source, target); err != nil { + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"upload": err.Error()}) + return + } + _ = snapshotRepo.UpdateStatus(statusID, map[string]interface{}{"upload": constant.StatusDone}) +} + +func handleSnapTar(sourceDir, targetDir, name, exclusionRules string) error { + if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { + if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { + return err + } + } + + exStr := "" + excludes := strings.Split(exclusionRules, ";") + for _, exclude := range excludes { + if len(exclude) == 0 { + continue + } + exStr += " --exclude " + exStr += exclude + } + + commands := fmt.Sprintf("tar --warning=no-file-changed -zcf %s %s -C %s .", targetDir+"/"+name, exStr, sourceDir) + global.LOG.Debug(commands) + stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) + if err != nil { + if len(stdout) != 0 { + global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) + return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) + } + } + return nil +} diff --git a/backend/constant/status.go b/backend/constant/status.go index 2e9ec0383..7c248af27 100644 --- a/backend/constant/status.go +++ b/backend/constant/status.go @@ -2,6 +2,7 @@ package constant const ( StatusRunning = "Running" + StatusDone = "Done" StatusStoped = "Stoped" StatusWaiting = "Waiting" StatusSuccess = "Success" diff --git a/backend/init/hook/hook.go b/backend/init/hook/hook.go index 131cdbfdc..0f2c3f990 100644 --- a/backend/init/hook/hook.go +++ b/backend/init/hook/hook.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "github.com/1Panel-dev/1Panel/backend/app/repo" + "github.com/1Panel-dev/1Panel/backend/constant" "github.com/1Panel-dev/1Panel/backend/global" "github.com/1Panel-dev/1Panel/backend/utils/cmd" "github.com/1Panel-dev/1Panel/backend/utils/common" @@ -63,4 +64,57 @@ func Init() { sudo := cmd.SudoHandleCmd() _, _ = cmd.Execf("%s sed -i '/CHANGE_USER_INFO=true/d' /usr/local/bin/1pctl", sudo) } + + handleSnapStatus() +} + +func handleSnapStatus() { + snapRepo := repo.NewISnapshotRepo() + snaps, _ := snapRepo.GetList() + for _, snap := range snaps { + if snap.Status == "OnSaveData" { + _ = snapRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess}) + } + if snap.Status == constant.StatusWaiting { + _ = snapRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed, "message": "the task was interrupted due to the restart of the 1panel service"}) + } + } + + status, _ := snapRepo.GetStatusList() + for _, statu := range status { + updatas := make(map[string]interface{}) + if statu.Panel == constant.StatusRunning { + updatas["panel"] = constant.StatusFailed + } + if statu.PanelCtl == constant.StatusRunning { + updatas["panel_ctl"] = constant.StatusFailed + } + if statu.PanelService == constant.StatusRunning { + updatas["panel_service"] = constant.StatusFailed + } + if statu.PanelInfo == constant.StatusRunning { + updatas["panel_info"] = constant.StatusFailed + } + if statu.DaemonJson == constant.StatusRunning { + updatas["daemon_json"] = constant.StatusFailed + } + if statu.AppData == constant.StatusRunning { + updatas["app_data"] = constant.StatusFailed + } + if statu.PanelData == constant.StatusRunning { + updatas["panel_data"] = constant.StatusFailed + } + if statu.BackupData == constant.StatusRunning { + updatas["backup_data"] = constant.StatusFailed + } + if statu.Compress == constant.StatusRunning { + updatas["compress"] = constant.StatusFailed + } + if statu.Upload == constant.StatusUploading { + updatas["upload"] = constant.StatusFailed + } + if len(updatas) != 0 { + _ = snapRepo.UpdateStatus(statu.ID, updatas) + } + } } diff --git a/backend/init/migration/migrations/init.go b/backend/init/migration/migrations/init.go index 0b1352421..d90316252 100644 --- a/backend/init/migration/migrations/init.go +++ b/backend/init/migration/migrations/init.go @@ -572,9 +572,9 @@ var UpdateCronjobWithDb = &gormigrate.Migration{ } var AddTableFirewall = &gormigrate.Migration{ - ID: "20230814-add-table-firewall", + ID: "20230821-add-table-firewall", Migrate: func(tx *gorm.DB) error { - if err := tx.AutoMigrate(&model.Firewall{}); err != nil { + if err := tx.AutoMigrate(&model.Firewall{}, model.SnapshotStatus{}); err != nil { return err } return nil diff --git a/backend/router/ro_setting.go b/backend/router/ro_setting.go index 3736c09f4..31a045cc7 100644 --- a/backend/router/ro_setting.go +++ b/backend/router/ro_setting.go @@ -34,6 +34,7 @@ func (s *SettingRouter) InitSettingRouter(Router *gin.RouterGroup) { settingRouter.POST("/mfa/bind", baseApi.MFABind) settingRouter.POST("/snapshot", baseApi.CreateSnapshot) + settingRouter.POST("/snapshot/status", baseApi.LoadSnapShotStatus) settingRouter.POST("/snapshot/search", baseApi.SearchSnapshot) settingRouter.POST("/snapshot/import", baseApi.ImportSnapshot) settingRouter.POST("/snapshot/del", baseApi.DeleteSnapshot) diff --git a/frontend/src/api/interface/setting.ts b/frontend/src/api/interface/setting.ts index 2b9537fb2..3dc388b18 100644 --- a/frontend/src/api/interface/setting.ts +++ b/frontend/src/api/interface/setting.ts @@ -76,6 +76,7 @@ export namespace Setting { interval: string; } export interface SnapshotCreate { + id: number; from: string; description: string; } @@ -106,6 +107,19 @@ export namespace Setting { rollbackMessage: string; lastRollbackedAt: string; } + export interface SnapshotStatus { + panel: string; + panelCtl: string; + panelService: string; + panelInfo: string; + daemonJson: string; + appData: string; + panelData: string; + backupData: string; + + compress: string; + upload: string; + } export interface UpgradeInfo { newVersion: string; latestVersion: string; diff --git a/frontend/src/api/modules/setting.ts b/frontend/src/api/modules/setting.ts index 2a6e1dd6b..103cac40f 100644 --- a/frontend/src/api/modules/setting.ts +++ b/frontend/src/api/modules/setting.ts @@ -132,6 +132,9 @@ export const listBucket = (params: Backup.ForBucket) => { export const snapshotCreate = (param: Setting.SnapshotCreate) => { return http.post(`/settings/snapshot`, param); }; +export const loadSnapStatus = (id: number) => { + return http.post(`/settings/snapshot/status`, { id: id }); +}; export const snapshotImport = (param: Setting.SnapshotImport) => { return http.post(`/settings/snapshot/import`, param); }; diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 6a4d31ffd..8fa19cd0f 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -198,8 +198,10 @@ const message = { }, status: { running: 'Running', + done: 'Done', success: 'Success', waiting: 'Waiting', + waiting1: 'Waiting', failed: 'Failed', stopped: 'Stopped', error: 'Error', @@ -1106,6 +1108,17 @@ const message = { certificate: 'Certificate', snapshot: 'Snapshot', + status: 'Snapshot status', + panelBin: 'Backup 1Panel binary', + panelCtl: 'Backup 1Panel script', + panelService: 'Backup 1Panel service', + panelInfo: 'Backup 1Panel basic information', + daemonJson: 'Backup Docker daemon.json', + appData: 'Backup 1Panel application', + panelData: 'Backup 1Panel data directory', + backupData: 'Backup 1Panel local backup directory', + compress: 'Compress snapshot file', + upload: 'Upload snapshot file', thirdPartySupport: 'Only third-party accounts are supported', recoverDetail: 'Recover detail', createSnapshot: 'Create snapshot', diff --git a/frontend/src/lang/modules/tw.ts b/frontend/src/lang/modules/tw.ts index 82ba995c6..2f67af6f6 100644 --- a/frontend/src/lang/modules/tw.ts +++ b/frontend/src/lang/modules/tw.ts @@ -196,8 +196,10 @@ const message = { }, status: { running: '已啟動', + done: '已完成', success: '成功', waiting: '執行中', + waiting1: '等待中', failed: '失敗', stopped: '已停止', error: '失敗', @@ -997,6 +999,17 @@ const message = { path: '路徑', snapshot: '快照', + status: '快照狀態', + panelBin: '備份 1Panel 二進製', + panelCtl: '備份 1Panel 腳本', + panelService: '備份 1Panel 服務', + panelInfo: '備份 1Panel 基礎信息', + daemonJson: '備份 Docker 配置', + appData: '備份 1Panel 應用', + panelData: '備份 1Panel 數據目錄', + backupData: '備份 1Panel 本地備份目錄', + compress: '壓縮快照文件', + upload: '上傳快照文件', thirdPartySupport: '僅支持第三方賬號', recoverDetail: '恢復詳情', createSnapshot: '創建快照', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index 75eca3143..a91b4d619 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -196,8 +196,10 @@ const message = { }, status: { running: '已启动', + done: '已完成', success: '成功', waiting: '执行中', + waiting1: '等待中', failed: '失败', stopped: '已停止', error: '失败', @@ -997,6 +999,17 @@ const message = { path: '路径', snapshot: '快照', + status: '快照状态', + panelBin: '备份 1Panel 二进制', + panelCtl: '备份 1Panel 脚本', + panelService: '备份 1Panel 服务', + panelInfo: '备份 1Panel 基础信息', + daemonJson: '备份 Docker 配置', + appData: '备份 1Panel 应用', + panelData: '备份 1Panel 数据目录', + backupData: '备份 1Panel 本地备份目录', + compress: '压缩快照文件', + upload: '上传快照文件', thirdPartySupport: '仅支持第三方账号', recoverDetail: '恢复详情', createSnapshot: '创建快照', diff --git a/frontend/src/views/setting/snapshot/index.vue b/frontend/src/views/setting/snapshot/index.vue index bc70337e5..67888f003 100644 --- a/frontend/src/views/setting/snapshot/index.vue +++ b/frontend/src/views/setting/snapshot/index.vue @@ -56,21 +56,20 @@ @@ -142,6 +141,7 @@ + @@ -156,6 +156,7 @@ import { ElForm } from 'element-plus'; import { Rules } from '@/global/form-rules'; import i18n from '@/lang'; import { Setting } from '@/api/interface/setting'; +import SnapStatus from '@/views/setting/snapshot/snap_status/index.vue'; import RecoverStatus from '@/views/setting/snapshot/status/index.vue'; import SnapshotImport from '@/views/setting/snapshot/import/index.vue'; import { getBackupList } from '@/api/modules/setting'; @@ -171,6 +172,7 @@ const paginationConfig = reactive({ }); const searchName = ref(); +const snapStatusRef = ref(); const recoverStatusRef = ref(); const importRef = ref(); const isRecordShow = ref(); @@ -182,6 +184,7 @@ const rules = reactive({ }); let snapInfo = reactive({ + id: 0, from: '', description: '', }); @@ -226,6 +229,10 @@ const submitAddSnapshot = (formEl: FormInstance | undefined) => { }); }; +const onLoadStatus = (row: Setting.SnapshotInfo) => { + snapStatusRef.value.acceptParams({ id: row.id, from: row.from, description: row.description }); +}; + const loadBackups = async () => { const res = await getBackupList(); backupOptions.value = []; diff --git a/frontend/src/views/setting/snapshot/snap_status/index.vue b/frontend/src/views/setting/snapshot/snap_status/index.vue new file mode 100644 index 000000000..39113299e --- /dev/null +++ b/frontend/src/views/setting/snapshot/snap_status/index.vue @@ -0,0 +1,338 @@ + + + +