mirror of https://github.com/1Panel-dev/1Panel
appstorecrontabdatabasedockerdocker-composedocker-containerdocker-imagedocker-uifilemanagerlamplnmppanel
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
279 lines
10 KiB
279 lines
10 KiB
package service |
|
|
|
import ( |
|
"context" |
|
"encoding/json" |
|
"fmt" |
|
"os" |
|
"path" |
|
"regexp" |
|
"strings" |
|
"sync" |
|
"time" |
|
|
|
"github.com/1Panel-dev/1Panel/backend/app/model" |
|
"github.com/1Panel-dev/1Panel/backend/constant" |
|
"github.com/1Panel-dev/1Panel/backend/global" |
|
"github.com/1Panel-dev/1Panel/backend/utils/cmd" |
|
"github.com/1Panel-dev/1Panel/backend/utils/common" |
|
"github.com/1Panel-dev/1Panel/backend/utils/files" |
|
) |
|
|
|
type snapHelper struct { |
|
SnapID uint |
|
Status *model.SnapshotStatus |
|
Ctx context.Context |
|
FileOp files.FileOp |
|
Wg *sync.WaitGroup |
|
} |
|
|
|
func snapJson(snap snapHelper, snapJson SnapshotJson, targetDir string) { |
|
defer snap.Wg.Done() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": constant.Running}) |
|
status := constant.StatusDone |
|
remarkInfo, _ := json.MarshalIndent(snapJson, "", "\t") |
|
if err := os.WriteFile(fmt.Sprintf("%s/snapshot.json", targetDir), remarkInfo, 0640); err != nil { |
|
status = err.Error() |
|
} |
|
snap.Status.PanelInfo = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_info": status}) |
|
} |
|
|
|
func snapPanel(snap snapHelper, targetDir string) { |
|
defer snap.Wg.Done() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": constant.Running}) |
|
status := constant.StatusDone |
|
if err := common.CopyFile("/usr/local/bin/1panel", path.Join(targetDir, "1panel")); err != nil { |
|
status = err.Error() |
|
} |
|
|
|
if err := common.CopyFile("/usr/local/bin/1pctl", targetDir); err != nil { |
|
status = err.Error() |
|
} |
|
|
|
if err := common.CopyFile("/etc/systemd/system/1panel.service", targetDir); err != nil { |
|
status = err.Error() |
|
} |
|
snap.Status.Panel = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel": status}) |
|
} |
|
|
|
func snapDaemonJson(snap snapHelper, targetDir string) { |
|
defer snap.Wg.Done() |
|
status := constant.StatusDone |
|
if !snap.FileOp.Stat("/etc/docker/daemon.json") { |
|
snap.Status.DaemonJson = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) |
|
return |
|
} |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": constant.Running}) |
|
if err := common.CopyFile("/etc/docker/daemon.json", targetDir); err != nil { |
|
status = err.Error() |
|
} |
|
snap.Status.DaemonJson = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"daemon_json": status}) |
|
} |
|
|
|
func snapAppData(snap snapHelper, targetDir string) { |
|
defer snap.Wg.Done() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.Running}) |
|
appInstalls, err := appInstallRepo.ListBy() |
|
if err != nil { |
|
snap.Status.AppData = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) |
|
return |
|
} |
|
runtimes, err := runtimeRepo.List() |
|
if err != nil { |
|
snap.Status.AppData = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": err.Error()}) |
|
return |
|
} |
|
imageRegex := regexp.MustCompile(`image:\s*(.*)`) |
|
var imageSaveList []string |
|
existStr, _ := cmd.Exec("docker images | awk '{print $1\":\"$2}' | grep -v REPOSITORY:TAG") |
|
existImages := strings.Split(existStr, "\n") |
|
duplicateMap := make(map[string]bool) |
|
for _, app := range appInstalls { |
|
matches := imageRegex.FindAllStringSubmatch(app.DockerCompose, -1) |
|
for _, match := range matches { |
|
for _, existImage := range existImages { |
|
if match[1] == existImage && !duplicateMap[match[1]] { |
|
imageSaveList = append(imageSaveList, match[1]) |
|
duplicateMap[match[1]] = true |
|
} |
|
} |
|
} |
|
} |
|
for _, runtime := range runtimes { |
|
for _, existImage := range existImages { |
|
if runtime.Image == existImage && !duplicateMap[runtime.Image] { |
|
imageSaveList = append(imageSaveList, runtime.Image) |
|
duplicateMap[runtime.Image] = true |
|
} |
|
} |
|
} |
|
|
|
if len(imageSaveList) != 0 { |
|
global.LOG.Debugf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) |
|
std, err := cmd.Execf("docker save %s | gzip -c > %s", strings.Join(imageSaveList, " "), path.Join(targetDir, "docker_image.tar")) |
|
if err != nil { |
|
snap.Status.AppData = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": std}) |
|
return |
|
} |
|
} |
|
snap.Status.AppData = constant.StatusDone |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"app_data": constant.StatusDone}) |
|
} |
|
|
|
func snapBackup(snap snapHelper, localDir, targetDir string) { |
|
defer snap.Wg.Done() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": constant.Running}) |
|
status := constant.StatusDone |
|
if err := handleSnapTar(localDir, targetDir, "1panel_backup.tar.gz", "./system;./system_snapshot;", ""); err != nil { |
|
status = err.Error() |
|
} |
|
snap.Status.BackupData = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"backup_data": status}) |
|
} |
|
|
|
func snapPanelData(snap snapHelper, localDir, targetDir string) { |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": constant.Running}) |
|
status := constant.StatusDone |
|
dataDir := path.Join(global.CONF.System.BaseDir, "1panel") |
|
exclusionRules := "./tmp;./log;./cache;./db/1Panel.db-*;" |
|
if strings.Contains(localDir, dataDir) { |
|
exclusionRules += ("." + strings.ReplaceAll(localDir, dataDir, "") + ";") |
|
} |
|
ignoreVal, _ := settingRepo.Get(settingRepo.WithByKey("SnapshotIgnore")) |
|
rules := strings.Split(ignoreVal.Value, ",") |
|
for _, ignore := range rules { |
|
if len(ignore) == 0 || cmd.CheckIllegal(ignore) { |
|
continue |
|
} |
|
exclusionRules += ("." + strings.ReplaceAll(ignore, dataDir, "") + ";") |
|
} |
|
_ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": "OnSaveData"}) |
|
sysIP, _ := settingRepo.Get(settingRepo.WithByKey("SystemIP")) |
|
_ = settingRepo.Update("SystemIP", "") |
|
checkPointOfWal() |
|
if err := handleSnapTar(dataDir, targetDir, "1panel_data.tar.gz", exclusionRules, ""); err != nil { |
|
status = err.Error() |
|
} |
|
_ = snapshotRepo.Update(snap.SnapID, map[string]interface{}{"status": constant.StatusWaiting}) |
|
|
|
snap.Status.PanelData = status |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"panel_data": status}) |
|
_ = settingRepo.Update("SystemIP", sysIP.Value) |
|
} |
|
|
|
func snapCompress(snap snapHelper, rootDir string, secret string) { |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusRunning}) |
|
tmpDir := path.Join(global.CONF.System.TmpDir, "system") |
|
fileName := fmt.Sprintf("%s.tar.gz", path.Base(rootDir)) |
|
if err := handleSnapTar(rootDir, tmpDir, fileName, "", secret); err != nil { |
|
snap.Status.Compress = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) |
|
return |
|
} |
|
|
|
stat, err := os.Stat(path.Join(tmpDir, fileName)) |
|
if err != nil { |
|
snap.Status.Compress = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": err.Error()}) |
|
return |
|
} |
|
size := common.LoadSizeUnit2F(float64(stat.Size())) |
|
global.LOG.Debugf("compress successful! size of file: %s", size) |
|
snap.Status.Compress = constant.StatusDone |
|
snap.Status.Size = size |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"compress": constant.StatusDone, "size": size}) |
|
|
|
global.LOG.Debugf("remove snapshot file %s", rootDir) |
|
_ = os.RemoveAll(rootDir) |
|
} |
|
|
|
func snapUpload(snap snapHelper, accounts string, file string) { |
|
source := path.Join(global.CONF.System.TmpDir, "system", path.Base(file)) |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusUploading}) |
|
accountMap, err := loadClientMap(accounts) |
|
if err != nil { |
|
snap.Status.Upload = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) |
|
return |
|
} |
|
targetAccounts := strings.Split(accounts, ",") |
|
for _, item := range targetAccounts { |
|
global.LOG.Debugf("start upload snapshot to %s, path: %s", item, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))) |
|
if _, err := accountMap[item].client.Upload(source, path.Join(accountMap[item].backupPath, "system_snapshot", path.Base(file))); err != nil { |
|
global.LOG.Debugf("upload to %s failed, err: %v", item, err) |
|
snap.Status.Upload = err.Error() |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": err.Error()}) |
|
return |
|
} |
|
global.LOG.Debugf("upload to %s successful", item) |
|
} |
|
snap.Status.Upload = constant.StatusDone |
|
_ = snapshotRepo.UpdateStatus(snap.Status.ID, map[string]interface{}{"upload": constant.StatusDone}) |
|
|
|
global.LOG.Debugf("remove snapshot file %s", source) |
|
_ = os.Remove(source) |
|
} |
|
|
|
func handleSnapTar(sourceDir, targetDir, name, exclusionRules string, secret string) error { |
|
if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) { |
|
if err = os.MkdirAll(targetDir, os.ModePerm); err != nil { |
|
return err |
|
} |
|
} |
|
|
|
exMap := make(map[string]struct{}) |
|
exStr := "" |
|
excludes := strings.Split(exclusionRules, ";") |
|
for _, exclude := range excludes { |
|
if len(exclude) == 0 { |
|
continue |
|
} |
|
if _, ok := exMap[exclude]; ok { |
|
continue |
|
} |
|
exStr += " --exclude " |
|
exStr += exclude |
|
exMap[exclude] = struct{}{} |
|
} |
|
path := "" |
|
if strings.Contains(sourceDir, "/") { |
|
itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "") |
|
aheadDir := sourceDir[:strings.LastIndex(sourceDir, "/")] |
|
if len(aheadDir) == 0 { |
|
aheadDir = "/" |
|
} |
|
path += fmt.Sprintf("-C %s %s", aheadDir, itemDir) |
|
} else { |
|
path = sourceDir |
|
} |
|
commands := "" |
|
if len(secret) != 0 { |
|
extraCmd := "| openssl enc -aes-256-cbc -salt -k '" + secret + "' -out" |
|
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -print) -zcf %s %s %s %s", sourceDir, " -"+exStr, path, extraCmd, targetDir+"/"+name) |
|
global.LOG.Debug(strings.ReplaceAll(commands, fmt.Sprintf(" %s ", secret), "******")) |
|
} else { |
|
commands = fmt.Sprintf("tar --warning=no-file-changed --ignore-failed-read --exclude-from=<(find %s -type s -printf '%s' | sed 's|^|./|') -zcf %s %s -C %s .", sourceDir, "%P\n", targetDir+"/"+name, exStr, sourceDir) |
|
global.LOG.Debug(commands) |
|
} |
|
stdout, err := cmd.ExecWithTimeOut(commands, 30*time.Minute) |
|
if err != nil { |
|
if len(stdout) != 0 { |
|
global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) |
|
return fmt.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err) |
|
} |
|
} |
|
return nil |
|
} |
|
|
|
func checkPointOfWal() { |
|
if err := global.DB.Exec("PRAGMA wal_checkpoint(TRUNCATE);").Error; err != nil { |
|
global.LOG.Errorf("handle check point failed, err: %v", err) |
|
} |
|
}
|
|
|