mirror of https://github.com/1Panel-dev/1Panel
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
411 lines
14 KiB
411 lines
14 KiB
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/1Panel-dev/1Panel/agent/app/dto"
|
|
"github.com/1Panel-dev/1Panel/agent/app/model"
|
|
"github.com/1Panel-dev/1Panel/agent/app/task"
|
|
"github.com/1Panel-dev/1Panel/agent/constant"
|
|
"github.com/1Panel-dev/1Panel/agent/global"
|
|
"github.com/1Panel-dev/1Panel/agent/i18n"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/compose"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/files"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type snapRecoverHelper struct {
|
|
FileOp files.FileOp
|
|
Task *task.Task
|
|
}
|
|
|
|
func (u *SnapshotService) SnapshotRecover(req dto.SnapshotRecover) error {
|
|
global.LOG.Info("start to recover panel by snapshot now")
|
|
snap, err := snapshotRepo.Get(commonRepo.WithByID(req.ID))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if hasOs(snap.Name) && !strings.Contains(snap.Name, loadOs()) {
|
|
errInfo := fmt.Sprintf("restoring snapshots(%s) between different server architectures(%s) is not supported", snap.Name, loadOs())
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": errInfo})
|
|
return errors.New(errInfo)
|
|
}
|
|
if len(snap.RollbackStatus) != 0 && snap.RollbackStatus != constant.StatusSuccess {
|
|
req.IsNew = true
|
|
}
|
|
if !req.IsNew && (snap.InterruptStep == "RecoverDownload" || snap.InterruptStep == "RecoverDecompress" || snap.InterruptStep == "BackupBeforeRecover") {
|
|
req.IsNew = true
|
|
}
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"recover_status": constant.StatusWaiting})
|
|
_ = settingRepo.Update("SystemStatus", "Recovering")
|
|
|
|
if len(snap.InterruptStep) == 0 {
|
|
req.IsNew = true
|
|
}
|
|
if len(snap.TaskRecoverID) != 0 {
|
|
req.TaskID = snap.TaskRecoverID
|
|
} else {
|
|
_ = snapshotRepo.Update(snap.ID, map[string]interface{}{"task_recover_id": req.TaskID})
|
|
}
|
|
taskItem, err := task.NewTaskWithOps(snap.Name, task.TaskRecover, task.TaskScopeSnapshot, req.TaskID, snap.ID)
|
|
if err != nil {
|
|
global.LOG.Errorf("new task for create snapshot failed, err: %v", err)
|
|
return err
|
|
}
|
|
rootDir := path.Join(global.CONF.System.TmpDir, "system", snap.Name)
|
|
if _, err := os.Stat(rootDir); err != nil && os.IsNotExist(err) {
|
|
_ = os.MkdirAll(rootDir, os.ModePerm)
|
|
}
|
|
itemHelper := snapRecoverHelper{Task: taskItem, FileOp: files.NewFileOp()}
|
|
|
|
go func() {
|
|
_ = global.Cron.Stop()
|
|
defer func() {
|
|
global.Cron.Start()
|
|
}()
|
|
|
|
if req.IsNew || snap.InterruptStep == "RecoverDownload" || req.ReDownload {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverDownload",
|
|
func(t *task.Task) error { return handleDownloadSnapshot(&itemHelper, snap, rootDir) },
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "RecoverDecompress" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverDecompress",
|
|
func(t *task.Task) error {
|
|
itemHelper.Task.Log("######################## 2 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetWithName("RecoverDecompress", snap.Name))
|
|
err := itemHelper.FileOp.TarGzExtractPro(fmt.Sprintf("%s/%s.tar.gz", rootDir, snap.Name), rootDir, req.Secret)
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
|
|
return err
|
|
},
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "BackupBeforeRecover" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"BackupBeforeRecover",
|
|
func(t *task.Task) error { return backupBeforeRecover(snap.Name, &itemHelper) },
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
|
|
var snapJson SnapshotJson
|
|
taskItem.AddSubTaskWithAlias(
|
|
"Readjson",
|
|
func(t *task.Task) error {
|
|
snapJson, err = readFromJson(path.Join(rootDir, snap.Name), &itemHelper)
|
|
return err
|
|
},
|
|
nil,
|
|
)
|
|
if req.IsNew || snap.InterruptStep == "RecoverApp" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverApp",
|
|
func(t *task.Task) error { return recoverAppData(path.Join(rootDir, snap.Name), &itemHelper) },
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "RecoverBaseData" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverBaseData",
|
|
func(t *task.Task) error { return recoverBaseData(path.Join(rootDir, snap.Name, "base"), &itemHelper) },
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "RecoverDBData" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverDBData",
|
|
func(t *task.Task) error { return recoverDBData(path.Join(rootDir, snap.Name, "db"), &itemHelper) },
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "RecoverBackups" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverBackups",
|
|
func(t *task.Task) error {
|
|
itemHelper.Task.Log("######################## 8 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetWithName("RecoverBackups", snap.Name))
|
|
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_backup.tar.gz"), snapJson.BackupDataDir, "")
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
|
|
return err
|
|
},
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
if req.IsNew || snap.InterruptStep == "RecoverPanelData" {
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverPanelData",
|
|
func(t *task.Task) error {
|
|
itemHelper.Task.Log("######################## 9 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetWithName("RecoverPanelData", snap.Name))
|
|
err := itemHelper.FileOp.TarGzExtractPro(path.Join(rootDir, snap.Name, "/1panel_data.tar.gz"), path.Join(snapJson.BaseDir, "1panel"), "")
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Decompress"), err)
|
|
return err
|
|
},
|
|
nil,
|
|
)
|
|
req.IsNew = true
|
|
}
|
|
taskItem.AddSubTaskWithAlias(
|
|
"RecoverDBData",
|
|
func(t *task.Task) error {
|
|
return restartCompose(path.Join(snapJson.BaseDir, "1panel/docker/compose"), &itemHelper)
|
|
},
|
|
nil,
|
|
)
|
|
|
|
if err := taskItem.Execute(); err != nil {
|
|
_ = settingRepo.Update("SystemStatus", "Free")
|
|
_ = snapshotRepo.Update(req.ID, map[string]interface{}{"recover_status": constant.StatusFailed, "recover_message": err.Error(), "interrupt_step": taskItem.Task.CurrentStep})
|
|
return
|
|
}
|
|
_ = os.RemoveAll(rootDir)
|
|
_, _ = cmd.Exec("systemctl daemon-reload && systemctl restart 1panel.service")
|
|
}()
|
|
return nil
|
|
}
|
|
|
|
func handleDownloadSnapshot(itemHelper *snapRecoverHelper, snap model.Snapshot, targetDir string) error {
|
|
itemHelper.Task.Log("######################## 1 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDownload"))
|
|
|
|
account, client, err := NewBackupClientWithID(snap.DownloadAccountID)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("RecoverDownloadAccount", fmt.Sprintf("%s - %s", account.Type, account.Name)), err)
|
|
pathItem := account.BackupPath
|
|
if account.BackupPath != "/" {
|
|
pathItem = strings.TrimPrefix(account.BackupPath, "/")
|
|
}
|
|
filePath := fmt.Sprintf("%s/%s.tar.gz", targetDir, snap.Name)
|
|
_ = os.RemoveAll(filePath)
|
|
_, err = client.Download(path.Join(pathItem, fmt.Sprintf("system_snapshot/%s.tar.gz", snap.Name)), filePath)
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("Download"), err)
|
|
return err
|
|
}
|
|
|
|
func backupBeforeRecover(name string, itemHelper *snapRecoverHelper) error {
|
|
itemHelper.Task.Log("######################## 3 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("BackupBeforeRecover"))
|
|
|
|
rootDir := fmt.Sprintf("%s/1panel_original/original_%s", global.CONF.System.BaseDir, name)
|
|
baseDir := path.Join(rootDir, "base")
|
|
if _, err := os.Stat(baseDir); err != nil {
|
|
_ = os.MkdirAll(baseDir, os.ModePerm)
|
|
}
|
|
|
|
err := itemHelper.FileOp.CopyDirWithExclude(path.Join(global.CONF.System.BaseDir, "1panel"), rootDir, []string{"cache", "tmp"})
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", path.Join(global.CONF.System.BaseDir, "1panel")), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyDirWithExclude(global.CONF.System.Backup, rootDir, []string{"system_snapshot"})
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", global.CONF.System.Backup), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1pctl", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/usr/local/bin/1panel_agent", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel.service", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/etc/systemd/system/1panel_agent.service", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile("/etc/docker/daemon.json", baseDir)
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/docker/daemon.json"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func readFromJson(rootDir string, itemHelper *snapRecoverHelper) (SnapshotJson, error) {
|
|
itemHelper.Task.Log("######################## 4 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("Readjson"))
|
|
|
|
snapJsonPath := path.Join(rootDir, "base/snapshot.json")
|
|
var snap SnapshotJson
|
|
_, err := os.Stat(snapJsonPath)
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonPath"), err)
|
|
if err != nil {
|
|
return snap, err
|
|
}
|
|
fileByte, err := os.ReadFile(snapJsonPath)
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonContent"), err)
|
|
if err != nil {
|
|
return snap, err
|
|
}
|
|
err = json.Unmarshal(fileByte, &snap)
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("ReadjsonMarshal"), err)
|
|
if err != nil {
|
|
return snap, err
|
|
}
|
|
return snap, nil
|
|
}
|
|
|
|
func recoverAppData(src string, itemHelper *snapRecoverHelper) error {
|
|
itemHelper.Task.Log("######################## 5 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverApp"))
|
|
|
|
if _, err := os.Stat(path.Join(src, "images.tar.gz")); err != nil {
|
|
itemHelper.Task.Log(i18n.GetMsgByKey("RecoverAppEmpty"))
|
|
return nil
|
|
} else {
|
|
std, err := cmd.Execf("docker load < %s", path.Join(src, "images.tar.gz"))
|
|
if err != nil {
|
|
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverAppImage"), errors.New(std))
|
|
return fmt.Errorf("docker load images failed, err: %v", err)
|
|
}
|
|
itemHelper.Task.LogSuccess(i18n.GetMsgByKey("RecoverAppImage"))
|
|
}
|
|
|
|
appInstalls, err := appInstallRepo.ListBy()
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverAppList"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < len(appInstalls); i++ {
|
|
wg.Add(1)
|
|
appInstalls[i].Status = constant.Rebuilding
|
|
_ = appInstallRepo.Save(context.Background(), &appInstalls[i])
|
|
go func(app model.AppInstall) {
|
|
defer wg.Done()
|
|
dockerComposePath := app.GetComposePath()
|
|
out, err := compose.Down(dockerComposePath)
|
|
if err != nil {
|
|
_ = handleErr(app, err, out)
|
|
return
|
|
}
|
|
out, err = compose.Up(dockerComposePath)
|
|
if err != nil {
|
|
_ = handleErr(app, err, out)
|
|
return
|
|
}
|
|
app.Status = constant.Running
|
|
_ = appInstallRepo.Save(context.Background(), &app)
|
|
}(appInstalls[i])
|
|
}
|
|
wg.Wait()
|
|
return nil
|
|
}
|
|
|
|
func recoverBaseData(src string, itemHelper *snapRecoverHelper) error {
|
|
itemHelper.Task.Log("######################## 6 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("SnapBaseInfo"))
|
|
|
|
err := itemHelper.FileOp.CopyFile(path.Join(src, "1pctl"), "/usr/local/bin")
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1pctl"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel"), "/usr/local/bin")
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent"), "/usr/local/bin")
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/usr/local/bin/1panel_agent"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel.service"), "/etc/systemd/system")
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel.service"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = itemHelper.FileOp.CopyFile(path.Join(src, "1panel_agent.service"), "/etc/systemd/system")
|
|
itemHelper.Task.LogWithStatus(i18n.GetWithName("SnapCopy", "/etc/systemd/system/1panel_agent.service"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
daemonJsonPath := "/etc/docker/daemon.json"
|
|
_, errSrc := os.Stat(path.Join(src, "docker/daemon.json"))
|
|
_, errPath := os.Stat(daemonJsonPath)
|
|
if os.IsNotExist(errSrc) && os.IsNotExist(errPath) {
|
|
itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJsonEmpty"))
|
|
return nil
|
|
}
|
|
if errSrc == nil {
|
|
err = itemHelper.FileOp.CopyFile(path.Join(src, "docker/daemon.json"), "/etc/docker")
|
|
itemHelper.Task.Log(i18n.GetMsgByKey("RecoverDaemonJson"))
|
|
if err != nil {
|
|
return fmt.Errorf("recover docker daemon.json failed, err: %v", err)
|
|
}
|
|
}
|
|
|
|
_, _ = cmd.Exec("systemctl restart docker")
|
|
return nil
|
|
}
|
|
|
|
func recoverDBData(src string, itemHelper *snapRecoverHelper) error {
|
|
itemHelper.Task.Log("######################## 7 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverDBData"))
|
|
err := itemHelper.FileOp.CopyDirWithExclude(src, path.Join(global.CONF.System.BaseDir, "1panel"), nil)
|
|
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverDBData"), err)
|
|
return err
|
|
}
|
|
|
|
func restartCompose(composePath string, itemHelper *snapRecoverHelper) error {
|
|
itemHelper.Task.Log("######################## 10 / 10 ########################")
|
|
itemHelper.Task.LogStart(i18n.GetMsgByKey("RecoverCompose"))
|
|
|
|
composes, err := composeRepo.ListRecord()
|
|
itemHelper.Task.LogWithStatus(i18n.GetMsgByKey("RecoverComposeList"), err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, compose := range composes {
|
|
pathItem := path.Join(composePath, compose.Name, "docker-compose.yml")
|
|
if _, err := os.Stat(pathItem); err != nil {
|
|
continue
|
|
}
|
|
upCmd := fmt.Sprintf("docker compose -f %s up -d", pathItem)
|
|
stdout, err := cmd.Exec(upCmd)
|
|
if err != nil {
|
|
itemHelper.Task.LogFailedWithErr(i18n.GetMsgByKey("RecoverCompose"), errors.New(stdout))
|
|
continue
|
|
}
|
|
itemHelper.Task.LogSuccess(i18n.GetWithName("RecoverComposeItem", pathItem))
|
|
}
|
|
return nil
|
|
}
|