2023-01-06 10:53:25 +00:00
|
|
|
package service
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
2023-01-09 14:55:10 +00:00
|
|
|
"encoding/json"
|
2023-01-06 10:53:25 +00:00
|
|
|
"fmt"
|
|
|
|
"os"
|
2023-07-27 07:52:25 +00:00
|
|
|
"path"
|
2023-08-01 10:40:54 +00:00
|
|
|
"regexp"
|
|
|
|
"strconv"
|
2023-01-09 14:55:10 +00:00
|
|
|
"strings"
|
2023-08-23 09:16:18 +00:00
|
|
|
"sync"
|
2023-01-06 10:53:25 +00:00
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/app/dto"
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/app/model"
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/constant"
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/global"
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/utils/cmd"
|
2023-09-14 06:06:13 +00:00
|
|
|
"github.com/1Panel-dev/1Panel/backend/utils/compose"
|
2023-01-06 10:53:25 +00:00
|
|
|
"github.com/1Panel-dev/1Panel/backend/utils/docker"
|
|
|
|
"github.com/1Panel-dev/1Panel/backend/utils/files"
|
|
|
|
"github.com/jinzhu/copier"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
)
|
|
|
|
|
2023-02-09 08:36:28 +00:00
|
|
|
type SnapshotService struct {
|
|
|
|
OriginalPath string
|
|
|
|
}
|
2023-01-06 10:53:25 +00:00
|
|
|
|
|
|
|
type ISnapshotService interface {
|
2023-02-10 07:55:56 +00:00
|
|
|
SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error)
|
2023-01-09 14:55:10 +00:00
|
|
|
SnapshotCreate(req dto.SnapshotCreate) error
|
|
|
|
SnapshotRecover(req dto.SnapshotRecover) error
|
|
|
|
SnapshotRollback(req dto.SnapshotRecover) error
|
2023-02-13 07:48:18 +00:00
|
|
|
SnapshotImport(req dto.SnapshotImport) error
|
2023-01-09 14:55:10 +00:00
|
|
|
Delete(req dto.BatchDeleteReq) error
|
|
|
|
|
2023-08-23 09:16:18 +00:00
|
|
|
LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error)
|
|
|
|
|
2023-02-13 07:48:18 +00:00
|
|
|
UpdateDescription(req dto.UpdateDescription) error
|
2023-01-09 14:55:10 +00:00
|
|
|
readFromJson(path string) (SnapshotJson, error)
|
2023-08-24 14:26:16 +00:00
|
|
|
|
|
|
|
HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string) (string, string, error)
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func NewISnapshotService() ISnapshotService {
|
|
|
|
return &SnapshotService{}
|
|
|
|
}
|
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
func (u *SnapshotService) SearchWithPage(req dto.SearchWithPage) (int64, interface{}, error) {
|
|
|
|
total, systemBackups, err := snapshotRepo.Page(req.Page, req.PageSize, commonRepo.WithLikeName(req.Info))
|
2023-01-06 10:53:25 +00:00
|
|
|
var dtoSnap []dto.SnapshotInfo
|
2023-01-09 14:55:10 +00:00
|
|
|
for _, systemBackup := range systemBackups {
|
2023-01-06 10:53:25 +00:00
|
|
|
var item dto.SnapshotInfo
|
2023-01-09 14:55:10 +00:00
|
|
|
if err := copier.Copy(&item, &systemBackup); err != nil {
|
2023-01-06 10:53:25 +00:00
|
|
|
return 0, nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
|
|
|
|
}
|
|
|
|
dtoSnap = append(dtoSnap, item)
|
|
|
|
}
|
|
|
|
return total, dtoSnap, err
|
|
|
|
}
|
|
|
|
|
2023-02-13 07:48:18 +00:00
|
|
|
func (u *SnapshotService) SnapshotImport(req dto.SnapshotImport) error {
|
|
|
|
if len(req.Names) == 0 {
|
|
|
|
return fmt.Errorf("incorrect snapshot request body: %v", req.Names)
|
|
|
|
}
|
2023-06-29 02:30:10 +00:00
|
|
|
for _, snapName := range req.Names {
|
2023-07-12 08:39:16 +00:00
|
|
|
snap, _ := snapshotRepo.Get(commonRepo.WithByName(strings.ReplaceAll(snapName, ".tar.gz", "")))
|
2023-06-29 02:30:10 +00:00
|
|
|
if snap.ID != 0 {
|
|
|
|
return constant.ErrRecordExist
|
|
|
|
}
|
|
|
|
}
|
2023-02-13 07:48:18 +00:00
|
|
|
for _, snap := range req.Names {
|
2023-09-14 03:38:12 +00:00
|
|
|
shortName := strings.TrimPrefix(snap, "snapshot_")
|
|
|
|
nameItems := strings.Split(shortName, "_")
|
|
|
|
if !strings.HasPrefix(shortName, "1panel_v") || !strings.HasSuffix(shortName, ".tar.gz") || len(nameItems) != 3 {
|
|
|
|
return fmt.Errorf("incorrect snapshot name format of %s", shortName)
|
2023-02-13 07:48:18 +00:00
|
|
|
}
|
|
|
|
formatTime, err := time.Parse("20060102150405", strings.ReplaceAll(nameItems[2], ".tar.gz", ""))
|
|
|
|
if err != nil {
|
2023-09-14 03:38:12 +00:00
|
|
|
return fmt.Errorf("incorrect snapshot name format of %s", shortName)
|
2023-02-13 07:48:18 +00:00
|
|
|
}
|
2023-03-21 10:46:31 +00:00
|
|
|
if strings.HasSuffix(snap, ".tar.gz") {
|
|
|
|
snap = strings.ReplaceAll(snap, ".tar.gz", "")
|
|
|
|
}
|
2023-02-13 07:48:18 +00:00
|
|
|
itemSnap := model.Snapshot{
|
|
|
|
Name: snap,
|
|
|
|
From: req.From,
|
|
|
|
Version: nameItems[1],
|
|
|
|
Description: req.Description,
|
|
|
|
Status: constant.StatusSuccess,
|
|
|
|
BaseModel: model.BaseModel{
|
|
|
|
CreatedAt: formatTime,
|
|
|
|
UpdatedAt: formatTime,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
if err := snapshotRepo.Create(&itemSnap); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) UpdateDescription(req dto.UpdateDescription) error {
|
|
|
|
return snapshotRepo.Update(req.ID, map[string]interface{}{"description": req.Description})
|
|
|
|
}
|
|
|
|
|
2023-08-23 09:16:18 +00:00
|
|
|
func (u *SnapshotService) LoadSnapShotStatus(id uint) (*dto.SnapshotStatus, error) {
|
|
|
|
var data dto.SnapshotStatus
|
|
|
|
status, err := snapshotRepo.GetStatus(id)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err := copier.Copy(&data, &status); err != nil {
|
|
|
|
return nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
|
|
|
|
}
|
|
|
|
return &data, nil
|
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
type SnapshotJson struct {
|
2023-02-13 07:48:18 +00:00
|
|
|
OldBaseDir string `json:"oldBaseDir"`
|
2023-01-09 14:55:10 +00:00
|
|
|
OldDockerDataDir string `json:"oldDockerDataDir"`
|
2023-05-30 07:30:57 +00:00
|
|
|
OldBackupDataDir string `json:"oldBackupDataDir"`
|
2023-01-09 14:55:10 +00:00
|
|
|
OldPanelDataDir string `json:"oldPanelDataDir"`
|
|
|
|
|
2023-02-13 07:48:18 +00:00
|
|
|
BaseDir string `json:"baseDir"`
|
2023-01-09 14:55:10 +00:00
|
|
|
DockerDataDir string `json:"dockerDataDir"`
|
|
|
|
BackupDataDir string `json:"backupDataDir"`
|
|
|
|
PanelDataDir string `json:"panelDataDir"`
|
|
|
|
LiveRestoreEnabled bool `json:"liveRestoreEnabled"`
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) SnapshotCreate(req dto.SnapshotCreate) error {
|
2023-08-24 14:26:16 +00:00
|
|
|
if _, _, err := u.HandleSnapshot(false, req, time.Now().Format("20060102150405")); err != nil {
|
2023-01-06 10:53:25 +00:00
|
|
|
return err
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-01-06 10:53:25 +00:00
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
|
2023-10-07 07:46:44 +00:00
|
|
|
global.LOG.Info("start to recover panel by snapshot now")
|
2023-01-09 14:55:10 +00:00
|
|
|
snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
|
|
|
|
if err != nil {
|
2023-01-06 10:53:25 +00:00
|
|
|
return err
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
if !req.IsNew && len(snap.InterruptStep) != 0 && len(snap.RollbackStatus) != 0 {
|
|
|
|
return fmt.Errorf("the snapshot has been rolled back and cannot be restored again")
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
isNewSnapshot := isNewSnapVersion(snap.Version)
|
2023-01-09 14:55:10 +00:00
|
|
|
isReTry := false
|
|
|
|
if len(snap.InterruptStep) != 0 && !req.IsNew {
|
|
|
|
isReTry = true
|
|
|
|
}
|
|
|
|
backup, err := backupRepo.Get(commonRepo.WithByType(snap.From))
|
2023-01-06 10:53:25 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
client, err := NewIBackupService().NewClient(&backup)
|
2023-01-06 10:53:25 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
localDir, err := loadLocalDir()
|
2023-01-06 10:53:25 +00:00
|
|
|
if err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return err
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
baseDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("system/%s", snap.Name))
|
2023-01-09 14:55:10 +00:00
|
|
|
if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) {
|
|
|
|
_ = os.MkdirAll(baseDir, os.ModePerm)
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting})
|
2023-02-09 08:36:28 +00:00
|
|
|
_ = settingRepo.Update("SystemStatus", "Recovering")
|
2023-01-09 14:55:10 +00:00
|
|
|
go func() {
|
2023-08-09 06:24:12 +00:00
|
|
|
_ = global.Cron.Stop()
|
|
|
|
defer func() {
|
|
|
|
global.Cron.Start()
|
|
|
|
}()
|
2023-01-09 14:55:10 +00:00
|
|
|
operation := "recover"
|
|
|
|
if isReTry {
|
|
|
|
operation = "re-recover"
|
|
|
|
}
|
|
|
|
if !isReTry || snap.InterruptStep == "Download" || (isReTry && req.ReDownload) {
|
2023-08-02 08:47:30 +00:00
|
|
|
itemBackupPath := strings.TrimPrefix(backup.BackupPath, "/")
|
|
|
|
itemBackupPath = strings.TrimSuffix(itemBackupPath, "/")
|
2023-06-29 03:24:11 +00:00
|
|
|
ok, err := client.Download(fmt.Sprintf("%s/system_snapshot/%s.tar.gz", itemBackupPath, snap.Name), fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name))
|
2023-01-09 14:55:10 +00:00
|
|
|
if err != nil || !ok {
|
|
|
|
if req.ReDownload {
|
|
|
|
updateRecoverStatus(snap.ID, snap.InterruptStep, constant.StatusFailed, fmt.Sprintf("download file %s from %s failed, err: %v", snap.Name, backup.Type, err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
updateRecoverStatus(snap.ID, "Download", constant.StatusFailed, fmt.Sprintf("download file %s from %s failed, err: %v", snap.Name, backup.Type, err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
fileOp := files.NewFileOp()
|
|
|
|
if !isReTry || snap.InterruptStep == "Decompress" || (isReTry && req.ReDownload) {
|
2023-03-07 07:51:48 +00:00
|
|
|
if err := handleUnTar(fmt.Sprintf("%s/%s.tar.gz", baseDir, snap.Name), baseDir); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
if req.ReDownload {
|
|
|
|
updateRecoverStatus(snap.ID, snap.InterruptStep, constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
updateRecoverStatus(snap.ID, "Decompress", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
rootDir := fmt.Sprintf("%s/%s", baseDir, snap.Name)
|
|
|
|
|
|
|
|
snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", rootDir))
|
|
|
|
if err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "Readjson", constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "Readjson" {
|
|
|
|
isReTry = false
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
u.OriginalPath = fmt.Sprintf("%s/1panel_original/original_%s", snapJson.BaseDir, snap.Name)
|
2023-02-13 07:48:18 +00:00
|
|
|
_ = os.MkdirAll(u.OriginalPath, os.ModePerm)
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-03-14 13:55:28 +00:00
|
|
|
snapJson.OldBaseDir = global.CONF.System.BaseDir
|
2023-07-27 07:52:25 +00:00
|
|
|
snapJson.OldPanelDataDir = path.Join(global.CONF.System.BaseDir, "1panel")
|
2023-01-09 14:55:10 +00:00
|
|
|
snapJson.OldBackupDataDir = localDir
|
|
|
|
recoverPanelDir := fmt.Sprintf("%s/%s/1panel", baseDir, snap.Name)
|
|
|
|
liveRestore := false
|
2023-08-01 10:40:54 +00:00
|
|
|
|
2023-08-07 07:10:26 +00:00
|
|
|
if !isNewSnapshot {
|
|
|
|
if !isReTry || snap.InterruptStep == "LoadDockerJson" {
|
2023-08-01 10:40:54 +00:00
|
|
|
snapJson.OldDockerDataDir, liveRestore, err = u.loadDockerDataDir()
|
|
|
|
if err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "LoadDockerJson", constant.StatusFailed, fmt.Sprintf("load docker data dir failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
if liveRestore {
|
|
|
|
if err := u.updateLiveRestore(false); err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "UpdateLiveRestore", constant.StatusFailed, fmt.Sprintf("update docker daemon.json live-restore conf failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
_ = u.saveJson(snapJson, rootDir)
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-08-01 10:40:54 +00:00
|
|
|
_, _ = cmd.Exec("systemctl stop docker")
|
2023-08-07 07:10:26 +00:00
|
|
|
if !isReTry || snap.InterruptStep == "DockerDir" {
|
2023-08-01 10:40:54 +00:00
|
|
|
if err := u.handleDockerDatas(fileOp, operation, rootDir, snapJson.DockerDataDir); err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "DockerDir", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if !isReTry || snap.InterruptStep == "DockerDir" {
|
|
|
|
if err := u.handleDockerDatasWithSave(fileOp, operation, rootDir, ""); err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "DockerDir", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
if !isReTry || snap.InterruptStep == "DaemonJson" {
|
2023-02-09 08:36:28 +00:00
|
|
|
if err := u.handleDaemonJson(fileOp, operation, rootDir+"/docker/daemon.json", u.OriginalPath); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRecoverStatus(snap.ID, "DaemonJson", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
_, _ = cmd.Exec("systemctl restart docker")
|
|
|
|
|
|
|
|
if !isReTry || snap.InterruptStep == "1PanelBinary" {
|
2023-02-09 08:36:28 +00:00
|
|
|
if err := u.handlePanelBinary(fileOp, operation, recoverPanelDir+"/1panel", u.OriginalPath+"/1panel"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRecoverStatus(snap.ID, "1PanelBinary", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
if !isReTry || snap.InterruptStep == "1PctlBinary" {
|
2023-02-09 08:36:28 +00:00
|
|
|
if err := u.handlePanelctlBinary(fileOp, operation, recoverPanelDir+"/1pctl", u.OriginalPath+"/1pctl"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRecoverStatus(snap.ID, "1PctlBinary", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
if !isReTry || snap.InterruptStep == "1PanelService" {
|
2023-02-09 08:36:28 +00:00
|
|
|
if err := u.handlePanelService(fileOp, operation, recoverPanelDir+"/1panel.service", u.OriginalPath+"/1panel.service"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRecoverStatus(snap.ID, "1PanelService", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
|
|
|
|
if !isReTry || snap.InterruptStep == "1PanelBackups" {
|
|
|
|
if err := u.handleBackupDatas(fileOp, operation, rootDir, snapJson.BackupDataDir); err != nil {
|
|
|
|
updateRecoverStatus(snap.ID, "1PanelBackups", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
|
|
|
|
|
|
|
if !isReTry || snap.InterruptStep == "1PanelData" {
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handlePanelDatas(snap.ID, fileOp, operation, rootDir, snapJson.PanelDataDir, localDir, snapJson.OldDockerDataDir); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRecoverStatus(snap.ID, "1PanelData", constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
isReTry = false
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
|
|
|
|
if isNewSnapshot {
|
|
|
|
_ = rebuildAllAppInstall()
|
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
_ = os.RemoveAll(rootDir)
|
|
|
|
global.LOG.Info("recover successful")
|
2023-02-06 03:42:51 +00:00
|
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
2023-01-09 14:55:10 +00:00
|
|
|
}()
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) SnapshotRollback(req dto.SnapshotRecover) error {
|
|
|
|
global.LOG.Info("start to rollback now")
|
|
|
|
snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "Download" || snap.InterruptStep == "Decompress" || snap.InterruptStep == "Readjson" {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
localDir, err := loadLocalDir()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
fileOp := files.NewFileOp()
|
2023-08-01 10:40:54 +00:00
|
|
|
isNewSnapshot := isNewSnapVersion(snap.Version)
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-07-27 07:52:25 +00:00
|
|
|
rootDir := path.Join(localDir, fmt.Sprintf("system/%s/%s", snap.Name, snap.Name))
|
2023-02-10 07:55:56 +00:00
|
|
|
|
|
|
|
_ = settingRepo.Update("SystemStatus", "Rollbacking")
|
2023-01-09 14:55:10 +00:00
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"rollback_status": constant.StatusWaiting})
|
2023-02-10 07:55:56 +00:00
|
|
|
go func() {
|
2023-08-09 06:24:12 +00:00
|
|
|
_ = global.Cron.Stop()
|
|
|
|
defer func() {
|
|
|
|
global.Cron.Start()
|
|
|
|
}()
|
2023-02-10 07:55:56 +00:00
|
|
|
snapJson, err := u.readFromJson(fmt.Sprintf("%s/snapshot.json", rootDir))
|
|
|
|
if err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, fmt.Sprintf("decompress file failed, err: %v", err))
|
|
|
|
return
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
u.OriginalPath = fmt.Sprintf("%s/1panel_original/original_%s", snapJson.OldBaseDir, snap.Name)
|
2023-02-13 07:48:18 +00:00
|
|
|
if _, err := os.Stat(u.OriginalPath); err != nil && os.IsNotExist(err) {
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-08-01 10:40:54 +00:00
|
|
|
if !isNewSnapshot {
|
|
|
|
_, _ = cmd.Exec("systemctl stop docker")
|
|
|
|
if err := u.handleDockerDatas(fileOp, "rollback", u.OriginalPath, snapJson.OldDockerDataDir); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer func() {
|
|
|
|
_, _ = cmd.Exec("systemctl restart docker")
|
|
|
|
}()
|
|
|
|
if snap.InterruptStep == "DockerDir" {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snapJson.LiveRestoreEnabled {
|
|
|
|
if err := u.updateLiveRestore(true); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "UpdateLiveRestore" {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if err := u.handleDockerDatasWithSave(fileOp, "rollback", u.OriginalPath, ""); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
defer func() {
|
|
|
|
_ = rebuildAllAppInstall()
|
|
|
|
}()
|
|
|
|
if snap.InterruptStep == "DockerDir" {
|
|
|
|
return
|
|
|
|
}
|
2023-02-10 07:55:56 +00:00
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handleDaemonJson(fileOp, "rollback", u.OriginalPath+"/daemon.json", ""); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
2023-02-10 07:55:56 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "DaemonJson" {
|
|
|
|
return
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handlePanelBinary(fileOp, "rollback", u.OriginalPath+"/1panel", ""); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "1PanelBinary" {
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handlePanelctlBinary(fileOp, "rollback", u.OriginalPath+"/1pctl", ""); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "1PctlBinary" {
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handlePanelService(fileOp, "rollback", u.OriginalPath+"/1panel.service", ""); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "1PanelService" {
|
|
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handleBackupDatas(fileOp, "rollback", u.OriginalPath, snapJson.OldBackupDataDir); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "1PanelBackups" {
|
|
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
if err := u.handlePanelDatas(snap.ID, fileOp, "rollback", u.OriginalPath, snapJson.OldPanelDataDir, "", ""); err != nil {
|
|
|
|
updateRollbackStatus(snap.ID, constant.StatusFailed, err.Error())
|
|
|
|
return
|
|
|
|
}
|
|
|
|
if snap.InterruptStep == "1PanelData" {
|
|
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
|
|
|
return
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
_ = os.RemoveAll(rootDir)
|
|
|
|
global.LOG.Info("rollback successful")
|
|
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
|
|
|
}()
|
2023-01-09 14:55:10 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) saveJson(snapJson SnapshotJson, path string) error {
|
|
|
|
remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t")
|
2023-04-07 03:30:10 +00:00
|
|
|
if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", path), remarkInfo, 0640); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) readFromJson(path string) (SnapshotJson, error) {
|
|
|
|
var snap SnapshotJson
|
|
|
|
if _, err := os.Stat(path); err != nil {
|
|
|
|
return snap, fmt.Errorf("find snapshot json file in recover package failed, err: %v", err)
|
|
|
|
}
|
|
|
|
fileByte, err := os.ReadFile(path)
|
|
|
|
if err != nil {
|
|
|
|
return snap, fmt.Errorf("read file from path %s failed, err: %v", path, err)
|
|
|
|
}
|
|
|
|
if err := json.Unmarshal(fileByte, &snap); err != nil {
|
|
|
|
return snap, fmt.Errorf("unmarshal snapjson failed, err: %v", err)
|
|
|
|
}
|
|
|
|
return snap, nil
|
|
|
|
}
|
|
|
|
|
2023-08-24 14:26:16 +00:00
|
|
|
func (u *SnapshotService) HandleSnapshot(isCronjob bool, req dto.SnapshotCreate, timeNow string) (string, string, error) {
|
|
|
|
localDir, err := loadLocalDir()
|
|
|
|
if err != nil {
|
|
|
|
return "", "", err
|
|
|
|
}
|
|
|
|
var (
|
|
|
|
rootDir string
|
|
|
|
snap model.Snapshot
|
|
|
|
snapStatus model.SnapshotStatus
|
|
|
|
)
|
|
|
|
|
|
|
|
if req.ID == 0 {
|
|
|
|
versionItem, _ := settingRepo.Get(settingRepo.WithByKey("SystemVersion"))
|
|
|
|
name := fmt.Sprintf("1panel_%s_%s", versionItem.Value, timeNow)
|
|
|
|
if isCronjob {
|
|
|
|
name = fmt.Sprintf("snapshot_1panel_%s_%s", versionItem.Value, timeNow)
|
|
|
|
}
|
|
|
|
rootDir = path.Join(localDir, "system", name)
|
|
|
|
|
|
|
|
snap = model.Snapshot{
|
|
|
|
Name: name,
|
|
|
|
Description: req.Description,
|
|
|
|
From: req.From,
|
|
|
|
Version: versionItem.Value,
|
|
|
|
Status: constant.StatusWaiting,
|
|
|
|
}
|
|
|
|
_ = snapshotRepo.Create(&snap)
|
|
|
|
snapStatus.SnapID = snap.ID
|
|
|
|
_ = snapshotRepo.CreateStatus(&snapStatus)
|
|
|
|
} else {
|
|
|
|
snap, err = snapshotRepo.Get(commonRepo.WithByID(req.ID))
|
|
|
|
if err != nil {
|
|
|
|
return "", "", err
|
|
|
|
}
|
|
|
|
snapStatus, _ = snapshotRepo.GetStatus(snap.ID)
|
|
|
|
if snapStatus.ID == 0 {
|
|
|
|
snapStatus.SnapID = snap.ID
|
|
|
|
_ = snapshotRepo.CreateStatus(&snapStatus)
|
|
|
|
}
|
|
|
|
rootDir = path.Join(localDir, fmt.Sprintf("system/%s", snap.Name))
|
|
|
|
}
|
|
|
|
|
|
|
|
var wg sync.WaitGroup
|
|
|
|
itemHelper := snapHelper{SnapID: snap.ID, Status: &snapStatus, Wg: &wg, FileOp: files.NewFileOp(), Ctx: context.Background()}
|
|
|
|
backupPanelDir := path.Join(rootDir, "1panel")
|
|
|
|
_ = os.MkdirAll(backupPanelDir, os.ModePerm)
|
|
|
|
backupDockerDir := path.Join(rootDir, "docker")
|
|
|
|
_ = os.MkdirAll(backupDockerDir, os.ModePerm)
|
|
|
|
|
|
|
|
jsonItem := SnapshotJson{
|
|
|
|
BaseDir: global.CONF.System.BaseDir,
|
|
|
|
BackupDataDir: localDir,
|
|
|
|
PanelDataDir: path.Join(global.CONF.System.BaseDir, "1panel"),
|
|
|
|
}
|
|
|
|
|
|
|
|
if snapStatus.PanelInfo != constant.StatusDone {
|
|
|
|
wg.Add(1)
|
|
|
|
go snapJson(itemHelper, jsonItem, rootDir)
|
|
|
|
}
|
|
|
|
if snapStatus.Panel != constant.StatusDone {
|
|
|
|
wg.Add(1)
|
|
|
|
go snapPanel(itemHelper, backupPanelDir)
|
|
|
|
}
|
|
|
|
if snapStatus.DaemonJson != constant.StatusDone {
|
|
|
|
wg.Add(1)
|
|
|
|
go snapDaemonJson(itemHelper, backupDockerDir)
|
|
|
|
}
|
|
|
|
if snapStatus.AppData != constant.StatusDone {
|
|
|
|
wg.Add(1)
|
|
|
|
go snapAppData(itemHelper, backupDockerDir)
|
|
|
|
}
|
|
|
|
if snapStatus.BackupData != constant.StatusDone {
|
|
|
|
wg.Add(1)
|
|
|
|
go snapBackup(itemHelper, localDir, backupPanelDir)
|
|
|
|
}
|
|
|
|
|
|
|
|
if !isCronjob {
|
|
|
|
go func() {
|
|
|
|
wg.Wait()
|
|
|
|
if !checkIsAllDone(snap.ID) {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return
|
|
|
|
}
|
2024-01-02 09:10:13 +00:00
|
|
|
snapPanelData(itemHelper, localDir, backupPanelDir)
|
|
|
|
if snapStatus.PanelData != constant.StatusDone {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return
|
|
|
|
}
|
2023-08-24 14:26:16 +00:00
|
|
|
snapCompress(itemHelper, rootDir)
|
|
|
|
if snapStatus.Compress != constant.StatusDone {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
snapUpload(itemHelper, req.From, fmt.Sprintf("%s.tar.gz", rootDir))
|
|
|
|
if snapStatus.Upload != constant.StatusDone {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess})
|
|
|
|
}()
|
|
|
|
return "", "", nil
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
if !checkIsAllDone(snap.ID) {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s backup failed", snap.Name)
|
|
|
|
}
|
|
|
|
snapCompress(itemHelper, rootDir)
|
|
|
|
if snapStatus.Compress != constant.StatusDone {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s compress failed", snap.Name)
|
|
|
|
}
|
|
|
|
snapUpload(itemHelper, req.From, fmt.Sprintf("%s.tar.gz", rootDir))
|
|
|
|
if snapStatus.Upload != constant.StatusDone {
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusFailed})
|
|
|
|
return loadLogByStatus(snapStatus), snap.Name, fmt.Errorf("snapshot %s upload failed", snap.Name)
|
|
|
|
}
|
|
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"status": constant.StatusSuccess})
|
|
|
|
return loadLogByStatus(snapStatus), snap.Name, nil
|
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
func (u *SnapshotService) handleDockerDatas(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
switch operation {
|
|
|
|
case "snapshot":
|
2023-08-23 09:16:18 +00:00
|
|
|
if err := handleSnapTar(source, target, "docker_data.tar.gz", ""); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup docker data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "recover":
|
2023-08-23 09:16:18 +00:00
|
|
|
if err := handleSnapTar(target, u.OriginalPath, "docker_data.tar.gz", ""); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup docker data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
if err := u.handleUnTar(source+"/docker/docker_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("recover docker data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "re-recover":
|
|
|
|
if err := u.handleUnTar(source+"/docker/docker_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("re-recover docker data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "rollback":
|
|
|
|
if err := u.handleUnTar(source+"/docker_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("rollback docker data failed, err: %v", err)
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
global.LOG.Info("handle docker data dir successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-08-01 10:40:54 +00:00
|
|
|
func (u *SnapshotService) handleDockerDatasWithSave(fileOp files.FileOp, operation, source, target string) error {
|
|
|
|
switch operation {
|
|
|
|
case "recover":
|
|
|
|
if err := u.handleDockerDatasWithSave(fileOp, "snapshot", "", u.OriginalPath); err != nil {
|
|
|
|
return fmt.Errorf("backup docker data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
std, err := cmd.Execf("docker load < %s", path.Join(source, "docker/docker_image.tar"))
|
|
|
|
if err != nil {
|
|
|
|
return errors.New(std)
|
|
|
|
}
|
|
|
|
case "re-recover":
|
|
|
|
std, err := cmd.Execf("docker load < %s", path.Join(source, "docker/docker_image.tar"))
|
|
|
|
if err != nil {
|
|
|
|
return errors.New(std)
|
|
|
|
}
|
|
|
|
case "rollback":
|
|
|
|
std, err := cmd.Execf("docker load < %s", path.Join(source, "docker_image.tar"))
|
|
|
|
if err != nil {
|
|
|
|
return errors.New(std)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
global.LOG.Info("handle docker data successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
func (u *SnapshotService) handleDaemonJson(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
daemonJsonPath := "/etc/docker/daemon.json"
|
|
|
|
if operation == "snapshot" || operation == "recover" {
|
|
|
|
_, err := os.Stat(daemonJsonPath)
|
|
|
|
if os.IsNotExist(err) {
|
2023-06-16 09:54:11 +00:00
|
|
|
global.LOG.Info("no daemon.json in snapshot and system now, nothing happened")
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
|
|
|
if err == nil {
|
|
|
|
if err := fileOp.CopyFile(daemonJsonPath, target); err != nil {
|
|
|
|
return fmt.Errorf("backup docker daemon.json failed, err: %v", err)
|
|
|
|
}
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
if operation == "recover" || operation == "rollback" || operation == "re-recover" {
|
|
|
|
_, sourceErr := os.Stat(source)
|
|
|
|
if os.IsNotExist(sourceErr) {
|
|
|
|
_ = os.Remove(daemonJsonPath)
|
|
|
|
}
|
|
|
|
if sourceErr == nil {
|
|
|
|
if err := fileOp.CopyFile(source, "/etc/docker"); err != nil {
|
|
|
|
return fmt.Errorf("recover docker daemon.json failed, err: %v", err)
|
|
|
|
}
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
global.LOG.Info("handle docker daemon.json successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) handlePanelBinary(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
panelPath := "/usr/local/bin/1panel"
|
|
|
|
if operation == "snapshot" || operation == "recover" {
|
|
|
|
if _, err := os.Stat(panelPath); err != nil {
|
|
|
|
return fmt.Errorf("1panel binary is not found in %s, err: %v", panelPath, err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{panelPath}, target); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup 1panel binary failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if operation == "recover" || operation == "rollback" || operation == "re-recover" {
|
|
|
|
if _, err := os.Stat(source); err != nil {
|
|
|
|
return fmt.Errorf("1panel binary is not found in snapshot, err: %v", err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{source}, "/usr/local/bin/1panel"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("recover 1panel binary failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
global.LOG.Info("handle binary panel successful!")
|
|
|
|
return nil
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
func (u *SnapshotService) handlePanelctlBinary(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
panelctlPath := "/usr/local/bin/1pctl"
|
|
|
|
if operation == "snapshot" || operation == "recover" {
|
|
|
|
if _, err := os.Stat(panelctlPath); err != nil {
|
|
|
|
return fmt.Errorf("1pctl binary is not found in %s, err: %v", panelctlPath, err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{panelctlPath}, target); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup 1pctl binary failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if operation == "recover" || operation == "rollback" || operation == "re-recover" {
|
|
|
|
if _, err := os.Stat(source); err != nil {
|
|
|
|
return fmt.Errorf("1pctl binary is not found in snapshot, err: %v", err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{source}, "/usr/local/bin/1pctl"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("recover 1pctl binary failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
global.LOG.Info("handle binary 1pactl successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) handlePanelService(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
panelServicePath := "/etc/systemd/system/1panel.service"
|
|
|
|
if operation == "snapshot" || operation == "recover" {
|
|
|
|
if _, err := os.Stat(panelServicePath); err != nil {
|
|
|
|
return fmt.Errorf("1panel service is not found in %s, err: %v", panelServicePath, err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{panelServicePath}, target); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup 1panel service failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if operation == "recover" || operation == "rollback" || operation == "re-recover" {
|
|
|
|
if _, err := os.Stat(source); err != nil {
|
|
|
|
return fmt.Errorf("1panel service is not found in snapshot, err: %v", err)
|
|
|
|
} else {
|
2023-08-24 14:26:16 +00:00
|
|
|
if err := cpBinary([]string{source}, "/etc/systemd/system/1panel.service"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("recover 1panel service failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
global.LOG.Info("handle panel service successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) handleBackupDatas(fileOp files.FileOp, operation string, source, target string) error {
|
|
|
|
switch operation {
|
|
|
|
case "recover":
|
2023-08-23 09:16:18 +00:00
|
|
|
if err := handleSnapTar(target, u.OriginalPath, "1panel_backup.tar.gz", "./system;"); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("restore original local backup dir data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
if err := u.handleUnTar(source+"/1panel/1panel_backup.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("recover local backup dir data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "re-recover":
|
|
|
|
if err := u.handleUnTar(source+"/1panel/1panel_backup.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("retry recover local backup dir data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "rollback":
|
|
|
|
if err := u.handleUnTar(source+"/1panel_backup.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("rollback local backup dir data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
global.LOG.Info("handle backup data successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-02-10 07:55:56 +00:00
|
|
|
func (u *SnapshotService) handlePanelDatas(snapID uint, fileOp files.FileOp, operation string, source, target, backupDir, dockerDir string) error {
|
2023-01-09 14:55:10 +00:00
|
|
|
switch operation {
|
|
|
|
case "snapshot":
|
2023-09-08 15:12:13 +00:00
|
|
|
exclusionRules := "./tmp;./log;./cache;"
|
2023-01-09 14:55:10 +00:00
|
|
|
if strings.Contains(backupDir, source) {
|
|
|
|
exclusionRules += ("." + strings.ReplaceAll(backupDir, source, "") + ";")
|
|
|
|
}
|
|
|
|
if strings.Contains(dockerDir, source) {
|
|
|
|
exclusionRules += ("." + strings.ReplaceAll(dockerDir, source, "") + ";")
|
|
|
|
}
|
2023-02-10 07:55:56 +00:00
|
|
|
|
2023-08-23 09:16:18 +00:00
|
|
|
if err := handleSnapTar(source, target, "1panel_data.tar.gz", exclusionRules); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("backup panel data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "recover":
|
2024-01-02 09:10:13 +00:00
|
|
|
exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;"
|
2023-01-09 14:55:10 +00:00
|
|
|
if strings.Contains(backupDir, target) {
|
2023-02-09 08:36:28 +00:00
|
|
|
exclusionRules += ("." + strings.ReplaceAll(backupDir, target, "") + ";")
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
|
|
|
if strings.Contains(dockerDir, target) {
|
2023-02-09 08:36:28 +00:00
|
|
|
exclusionRules += ("." + strings.ReplaceAll(dockerDir, target, "") + ";")
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
2023-02-10 07:55:56 +00:00
|
|
|
|
|
|
|
_ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": ""})
|
2024-01-02 09:10:13 +00:00
|
|
|
checkPointOfWal()
|
2023-08-23 09:16:18 +00:00
|
|
|
if err := handleSnapTar(target, u.OriginalPath, "1panel_data.tar.gz", exclusionRules); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return fmt.Errorf("restore original panel data failed, err: %v", err)
|
|
|
|
}
|
2023-02-10 07:55:56 +00:00
|
|
|
_ = snapshotRepo.Update(snapID, map[string]interface{}{"recover_status": constant.StatusWaiting})
|
2023-01-09 14:55:10 +00:00
|
|
|
|
2023-08-09 08:38:13 +00:00
|
|
|
_ = fileOp.Fs.RemoveAll(path.Join(target, "apps"))
|
2024-01-02 09:10:13 +00:00
|
|
|
checkPointOfWal()
|
2023-01-09 14:55:10 +00:00
|
|
|
if err := u.handleUnTar(source+"/1panel/1panel_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("recover panel data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "re-recover":
|
2023-08-09 08:38:13 +00:00
|
|
|
_ = fileOp.Fs.RemoveAll(path.Join(target, "apps"))
|
2024-01-02 09:10:13 +00:00
|
|
|
checkPointOfWal()
|
2023-01-09 14:55:10 +00:00
|
|
|
if err := u.handleUnTar(source+"/1panel/1panel_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("retry recover panel data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
case "rollback":
|
2023-08-09 08:38:13 +00:00
|
|
|
_ = fileOp.Fs.RemoveAll(path.Join(target, "apps"))
|
2024-01-02 09:10:13 +00:00
|
|
|
checkPointOfWal()
|
2023-01-09 14:55:10 +00:00
|
|
|
if err := u.handleUnTar(source+"/1panel_data.tar.gz", target); err != nil {
|
|
|
|
return fmt.Errorf("rollback panel data failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) loadDockerDataDir() (string, bool, error) {
|
|
|
|
client, err := docker.NewDockerClient()
|
|
|
|
if err != nil {
|
|
|
|
return "", false, fmt.Errorf("new docker client failed, err: %v", err)
|
|
|
|
}
|
|
|
|
info, err := client.Info(context.Background())
|
|
|
|
if err != nil {
|
|
|
|
return "", false, fmt.Errorf("load docker info failed, err: %v", err)
|
|
|
|
}
|
|
|
|
return info.DockerRootDir, info.LiveRestoreEnabled, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) Delete(req dto.BatchDeleteReq) error {
|
|
|
|
backups, _ := snapshotRepo.GetList(commonRepo.WithIdsIn(req.Ids))
|
|
|
|
localDir, err := loadLocalDir()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
for _, snap := range backups {
|
2023-08-24 14:26:16 +00:00
|
|
|
itemFile := path.Join(localDir, "system", snap.Name)
|
|
|
|
_ = os.RemoveAll(itemFile)
|
|
|
|
|
|
|
|
itemTarFile := path.Join(global.CONF.System.TmpDir, "system", snap.Name+".tar.gz")
|
|
|
|
_ = os.Remove(itemTarFile)
|
|
|
|
|
2023-08-23 09:16:18 +00:00
|
|
|
_ = snapshotRepo.DeleteStatus(snap.ID)
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
if err := snapshotRepo.Delete(commonRepo.WithIdsIn(req.Ids)); err != nil {
|
2023-01-06 10:53:25 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-01-09 14:55:10 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func updateRecoverStatus(id uint, interruptStep, status string, message string) {
|
|
|
|
if status != constant.StatusSuccess {
|
|
|
|
global.LOG.Errorf("recover failed, err: %s", message)
|
|
|
|
}
|
|
|
|
if err := snapshotRepo.Update(id, map[string]interface{}{
|
|
|
|
"interrupt_step": interruptStep,
|
|
|
|
"recover_status": status,
|
|
|
|
"recover_message": message,
|
|
|
|
"last_recovered_at": time.Now().Format("2006-01-02 15:04:05"),
|
|
|
|
}); err != nil {
|
|
|
|
global.LOG.Errorf("update snap recover status failed, err: %v", err)
|
|
|
|
}
|
2023-02-09 08:36:28 +00:00
|
|
|
_ = settingRepo.Update("SystemStatus", "Free")
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
|
|
|
func updateRollbackStatus(id uint, status string, message string) {
|
2023-02-10 07:55:56 +00:00
|
|
|
_ = settingRepo.Update("SystemStatus", "Free")
|
2023-01-09 14:55:10 +00:00
|
|
|
if status == constant.StatusSuccess {
|
|
|
|
if err := snapshotRepo.Update(id, map[string]interface{}{
|
|
|
|
"recover_status": "",
|
|
|
|
"recover_message": "",
|
|
|
|
"interrupt_step": "",
|
|
|
|
"rollback_status": "",
|
|
|
|
"rollback_message": "",
|
|
|
|
"last_rollbacked_at": time.Now().Format("2006-01-02 15:04:05"),
|
|
|
|
}); err != nil {
|
|
|
|
global.LOG.Errorf("update snap recover status failed, err: %v", err)
|
2023-01-06 10:53:25 +00:00
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
global.LOG.Errorf("rollback failed, err: %s", message)
|
|
|
|
if err := snapshotRepo.Update(id, map[string]interface{}{
|
|
|
|
"rollback_status": status,
|
|
|
|
"rollback_message": message,
|
|
|
|
"last_rollbacked_at": time.Now().Format("2006-01-02 15:04:05"),
|
|
|
|
}); err != nil {
|
|
|
|
global.LOG.Errorf("update snap recover status failed, err: %v", err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-08-24 14:26:16 +00:00
|
|
|
func cpBinary(src []string, dst string) error {
|
2023-09-19 09:20:22 +00:00
|
|
|
global.LOG.Debugf(fmt.Sprintf("\\cp -f %s %s", strings.Join(src, " "), dst))
|
|
|
|
stdout, err := cmd.Exec(fmt.Sprintf("\\cp -f %s %s", strings.Join(src, " "), dst))
|
2023-01-09 14:55:10 +00:00
|
|
|
if err != nil {
|
2023-09-19 09:20:22 +00:00
|
|
|
return fmt.Errorf("cp file failed, stdout: %v, err: %v", stdout, err)
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) updateLiveRestore(enabled bool) error {
|
|
|
|
if _, err := os.Stat(constant.DaemonJsonPath); err != nil {
|
|
|
|
return fmt.Errorf("load docker daemon.json conf failed, err: %v", err)
|
|
|
|
}
|
2023-04-07 03:30:10 +00:00
|
|
|
file, err := os.ReadFile(constant.DaemonJsonPath)
|
2023-01-09 14:55:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-05-30 07:30:57 +00:00
|
|
|
daemonMap := make(map[string]interface{})
|
|
|
|
_ = json.Unmarshal(file, &daemonMap)
|
2023-01-09 14:55:10 +00:00
|
|
|
|
|
|
|
if !enabled {
|
2023-05-30 07:30:57 +00:00
|
|
|
delete(daemonMap, "live-restore")
|
2023-01-09 14:55:10 +00:00
|
|
|
} else {
|
2023-05-30 07:30:57 +00:00
|
|
|
daemonMap["live-restore"] = enabled
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
2023-05-30 07:30:57 +00:00
|
|
|
newJson, err := json.MarshalIndent(daemonMap, "", "\t")
|
2023-01-09 14:55:10 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-04-07 03:30:10 +00:00
|
|
|
if err := os.WriteFile(constant.DaemonJsonPath, newJson, 0640); err != nil {
|
2023-01-09 14:55:10 +00:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
stdout, err := cmd.Exec("systemctl restart docker")
|
|
|
|
if err != nil {
|
|
|
|
return errors.New(stdout)
|
|
|
|
}
|
2023-02-06 03:42:51 +00:00
|
|
|
|
|
|
|
ticker := time.NewTicker(3 * time.Second)
|
2023-05-30 07:30:57 +00:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
|
|
|
|
defer cancel()
|
2023-02-06 03:42:51 +00:00
|
|
|
for range ticker.C {
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
return errors.New("the docker service cannot be restarted")
|
|
|
|
default:
|
|
|
|
stdout, err := cmd.Exec("systemctl is-active docker")
|
|
|
|
if string(stdout) == "active\n" && err == nil {
|
|
|
|
global.LOG.Info("docker restart with new live-restore successful!")
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (u *SnapshotService) handleUnTar(sourceDir, targetDir string) error {
|
|
|
|
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
|
|
|
|
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
2023-03-09 06:53:29 +00:00
|
|
|
|
|
|
|
commands := fmt.Sprintf("tar -zxf %s -C %s .", sourceDir, targetDir)
|
|
|
|
global.LOG.Debug(commands)
|
2023-04-18 09:32:59 +00:00
|
|
|
stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute)
|
2023-01-09 14:55:10 +00:00
|
|
|
if err != nil {
|
2023-08-02 08:47:30 +00:00
|
|
|
if len(stdout) != 0 {
|
|
|
|
global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
|
|
|
|
return fmt.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
|
|
|
|
}
|
2023-01-09 14:55:10 +00:00
|
|
|
}
|
2023-01-06 10:53:25 +00:00
|
|
|
return nil
|
|
|
|
}
|
2023-08-01 10:40:54 +00:00
|
|
|
|
|
|
|
func rebuildAllAppInstall() error {
|
2023-09-14 06:06:13 +00:00
|
|
|
global.LOG.Debug("start to rebuild all app")
|
2023-08-01 10:40:54 +00:00
|
|
|
appInstalls, err := appInstallRepo.ListBy()
|
|
|
|
if err != nil {
|
|
|
|
global.LOG.Errorf("get all app installed for rebuild failed, err: %v", err)
|
|
|
|
return err
|
|
|
|
}
|
2023-09-14 06:06:13 +00:00
|
|
|
var wg sync.WaitGroup
|
|
|
|
for i := 0; i < len(appInstalls); i++ {
|
|
|
|
wg.Add(1)
|
|
|
|
appInstalls[i].Status = constant.Rebuilding
|
|
|
|
_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
|
|
|
|
go func(app model.AppInstall) {
|
|
|
|
defer wg.Done()
|
|
|
|
dockerComposePath := app.GetComposePath()
|
|
|
|
out, err := compose.Down(dockerComposePath)
|
|
|
|
if err != nil {
|
|
|
|
_ = handleErr(app, err, out)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
out, err = compose.Up(dockerComposePath)
|
|
|
|
if err != nil {
|
|
|
|
_ = handleErr(app, err, out)
|
|
|
|
return
|
|
|
|
}
|
|
|
|
app.Status = constant.Running
|
|
|
|
_ = appInstallRepo.Save(context.Background(), &app)
|
|
|
|
}(appInstalls[i])
|
2023-08-01 10:40:54 +00:00
|
|
|
}
|
2023-09-14 06:06:13 +00:00
|
|
|
wg.Wait()
|
2023-08-01 10:40:54 +00:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func isNewSnapVersion(version string) bool {
|
|
|
|
versionItem := "v1.5.0"
|
|
|
|
if version == versionItem {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
version1s := strings.Split(version, ".")
|
|
|
|
version2s := strings.Split(versionItem, ".")
|
|
|
|
|
|
|
|
n := min(len(version1s), len(version2s))
|
|
|
|
re := regexp.MustCompile("[0-9]+")
|
|
|
|
for i := 0; i < n; i++ {
|
|
|
|
sVersion1s := re.FindAllString(version1s[i], -1)
|
|
|
|
sVersion2s := re.FindAllString(version2s[i], -1)
|
|
|
|
if len(sVersion1s) == 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if len(sVersion2s) == 0 {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
v1num, _ := strconv.Atoi(sVersion1s[0])
|
|
|
|
v2num, _ := strconv.Atoi(sVersion2s[0])
|
|
|
|
if v1num == v2num {
|
|
|
|
continue
|
|
|
|
} else {
|
|
|
|
return v1num > v2num
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
|
|
|
|
func min(a, b int) int {
|
|
|
|
if a < b {
|
|
|
|
return a
|
|
|
|
}
|
|
|
|
return b
|
|
|
|
}
|
2023-08-23 09:16:18 +00:00
|
|
|
|
|
|
|
func checkIsAllDone(snapID uint) bool {
|
|
|
|
status, err := snapshotRepo.GetStatus(snapID)
|
|
|
|
if err != nil {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.Panel != constant.StatusDone {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.PanelInfo != constant.StatusDone {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.DaemonJson != constant.StatusDone {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.AppData != constant.StatusDone {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
if status.BackupData != constant.StatusDone {
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
return true
|
|
|
|
}
|
2023-08-24 14:26:16 +00:00
|
|
|
|
|
|
|
func loadLogByStatus(status model.SnapshotStatus) string {
|
|
|
|
logs := ""
|
|
|
|
logs += fmt.Sprintf("Write 1Panel basic information: %s \n", status.PanelInfo)
|
|
|
|
logs += fmt.Sprintf("Backup 1Panel system files: %s \n", status.Panel)
|
|
|
|
logs += fmt.Sprintf("Backup Docker configuration file: %s \n", status.DaemonJson)
|
|
|
|
logs += fmt.Sprintf("Backup installed apps from 1Panel: %s \n", status.AppData)
|
|
|
|
logs += fmt.Sprintf("Backup 1Panel data directory: %s \n", status.PanelData)
|
|
|
|
logs += fmt.Sprintf("Backup local backup directory for 1Panel: %s \n", status.BackupData)
|
2023-09-08 15:12:13 +00:00
|
|
|
logs += fmt.Sprintf("Create snapshot file: %s \n", status.Compress)
|
|
|
|
logs += fmt.Sprintf("Snapshot size: %s \n", status.Size)
|
|
|
|
logs += fmt.Sprintf("Upload snapshot file: %s \n", status.Upload)
|
2023-08-24 14:26:16 +00:00
|
|
|
|
|
|
|
return logs
|
|
|
|
}
|