1Panel/backend/app/service/cronjob_helper.go

package service

import (
	"context"
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/1Panel-dev/1Panel/backend/app/model"
	"github.com/1Panel-dev/1Panel/backend/app/repo"
	"github.com/1Panel-dev/1Panel/backend/constant"
	"github.com/1Panel-dev/1Panel/backend/global"
	"github.com/1Panel-dev/1Panel/backend/utils/cloud_storage"
	"github.com/1Panel-dev/1Panel/backend/utils/cmd"
	"github.com/pkg/errors"
)

func (u *CronjobService) HandleJob(cronjob *model.Cronjob) {
	var (
		message []byte
		err     error
	)
	record := cronjobRepo.StartRecords(cronjob.ID, cronjob.KeepLocal, "")
	go func() {
		switch cronjob.Type {
		case "shell":
			if len(cronjob.Script) == 0 {
				return
			}
			stdout, errExec := cmd.ExecCronjobWithTimeOut(cronjob.Script, 5*time.Minute)
			if errExec != nil {
				err = errExec
			}
			message = []byte(stdout)
			u.HandleRmExpired("LOCAL", "", cronjob, nil)
		case "website":
			record.File, err = u.HandleBackup(cronjob, record.StartTime)
		case "database":
			record.File, err = u.HandleBackup(cronjob, record.StartTime)
		case "directory":
			if len(cronjob.SourceDir) == 0 {
				return
			}
			record.File, err = u.HandleBackup(cronjob, record.StartTime)
		case "curl":
			if len(cronjob.URL) == 0 {
				return
			}
			stdout, errCurl := cmd.ExecWithTimeOut("curl "+cronjob.URL, 5*time.Minute)
			if err != nil {
				err = errCurl
			}
			message = []byte(stdout)
			u.HandleRmExpired("LOCAL", "", cronjob, nil)
		}
		if err != nil {
			cronjobRepo.EndRecords(record, constant.StatusFailed, err.Error(), string(message))
			return
		}
		if len(message) != 0 {
			record.Records, err = mkdirAndWriteFile(cronjob, record.StartTime, message)
			if err != nil {
				global.LOG.Errorf("save file %s failed, err: %v", record.Records, err)
			}
		}
		cronjobRepo.EndRecords(record, constant.StatusSuccess, "", record.Records)
	}()
}

func (u *CronjobService) HandleBackup(cronjob *model.Cronjob, startTime time.Time) (string, error) {
	backup, err := backupRepo.Get(commonRepo.WithByID(uint(cronjob.TargetDirID)))
	if err != nil {
		return "", err
	}
	localDir, err := loadLocalDir()
	if err != nil {
		return "", err
	}
	global.LOG.Infof("start to backup %s %s to %s", cronjob.Type, cronjob.Name, backup.Type)

	switch cronjob.Type {
	case "database":
		app, err := appInstallRepo.LoadBaseInfo("mysql", "")
		if err != nil {
			return "", err
		}
		paths, err := u.handleDatabase(*cronjob, app, backup, startTime)
		return strings.Join(paths, ","), err
	case "website":
		paths, err := u.handleWebsite(*cronjob, backup, startTime)
		return strings.Join(paths, ","), err
	default:
		fileName := fmt.Sprintf("directory%s_%s.tar.gz", strings.ReplaceAll(cronjob.SourceDir, "/", "_"), startTime.Format("20060102150405"))
		backupDir := fmt.Sprintf("%s/%s/%s", localDir, cronjob.Type, cronjob.Name)
		itemFileDir := fmt.Sprintf("%s/%s", cronjob.Type, cronjob.Name)
		global.LOG.Infof("handle tar %s to %s", backupDir, fileName)
		if err := handleTar(cronjob.SourceDir, backupDir, fileName, cronjob.ExclusionRules); err != nil {
			return "", err
		}
		var client cloud_storage.CloudStorageClient
		if backup.Type != "LOCAL" {
			if !cronjob.KeepLocal {
				defer func() {
					_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, fileName))
				}()
			}
			client, err = NewIBackupService().NewClient(&backup)
			if err != nil {
				return "", err
			}
			if _, err = client.Upload(backupDir+"/"+fileName, itemFileDir+"/"+fileName); err != nil {
				return "", err
			}
		}
		u.HandleRmExpired(backup.Type, localDir, cronjob, client)
		if backup.Type == "LOCAL" || cronjob.KeepLocal {
			return fmt.Sprintf("%s/%s/%s/%s", localDir, cronjob.Type, cronjob.Name, fileName), nil
		}
		return fmt.Sprintf("%s/%s/%s", cronjob.Type, cronjob.Name, fileName), nil
	}
}

func (u *CronjobService) HandleRmExpired(backType, localDir string, cronjob *model.Cronjob, backClient cloud_storage.CloudStorageClient) {
	global.LOG.Infof("start to handle remove expired, retain copies: %d", cronjob.RetainCopies)
	records, _ := cronjobRepo.ListRecord(cronjobRepo.WithByJobID(int(cronjob.ID)), commonRepo.WithOrderBy("created_at desc"))
	if len(records) > int(cronjob.RetainCopies) {
		for i := int(cronjob.RetainCopies); i < len(records); i++ {
			files := strings.Split(records[i].File, ",")
			for _, file := range files {
				if backType != "LOCAL" {
					_, _ = backClient.Delete(strings.ReplaceAll(file, localDir+"/", ""))
					_ = os.Remove(file)
				} else {
					_ = os.Remove(file)
				}
				_ = backupRepo.DeleteRecord(context.TODO(), backupRepo.WithByFileName(path.Base(file)))
			}

			_ = cronjobRepo.DeleteRecord(commonRepo.WithByID(uint(records[i].ID)))
			_ = os.Remove(records[i].Records)
		}
	}
}

func handleTar(sourceDir, targetDir, name, exclusionRules string) error {
	if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
		if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
			return err
		}
	}

	excludes := strings.Split(exclusionRules, ";")
	excludeRules := ""
	for _, exclude := range excludes {
		if len(exclude) == 0 {
			continue
		}
		excludeRules += (" --exclude " + exclude)
	}
	path := ""
	if strings.Contains(sourceDir, "/") {
		itemDir := strings.ReplaceAll(sourceDir[strings.LastIndex(sourceDir, "/"):], "/", "")
		aheadDir := strings.ReplaceAll(sourceDir, itemDir, "")
		path += fmt.Sprintf("-C %s %s", aheadDir, itemDir)
	} else {
		path = sourceDir
	}

	commands := fmt.Sprintf("tar zcvf %s %s %s", targetDir+"/"+name, excludeRules, path)
	global.LOG.Debug(commands)
	stdout, err := cmd.ExecWithTimeOut(commands, 5*time.Minute)
	if err != nil {
		global.LOG.Errorf("do handle tar failed, stdout: %s, err: %v", stdout, err)
		return errors.New(stdout)
	}
	return nil
}

func handleUnTar(sourceFile, targetDir string) error {
	if _, err := os.Stat(targetDir); err != nil && os.IsNotExist(err) {
		if err = os.MkdirAll(targetDir, os.ModePerm); err != nil {
			return err
		}
	}

	commands := fmt.Sprintf("tar zxvfC %s %s", sourceFile, targetDir)
	global.LOG.Debug(commands)
	stdout, err := cmd.ExecWithTimeOut(commands, 5*time.Minute)
	if err != nil {
		global.LOG.Errorf("do handle untar failed, stdout: %s, err: %v", stdout, err)
		return errors.New(stdout)
	}
	return nil
}

func (u *CronjobService) handleDatabase(cronjob model.Cronjob, app *repo.RootInfo, backup model.BackupAccount, startTime time.Time) ([]string, error) {
	var paths []string
	localDir, err := loadLocalDir()
	if err != nil {
		return paths, err
	}

	var dblist []string
	if cronjob.DBName == "all" {
		mysqlService := NewIMysqlService()
		dblist, err = mysqlService.ListDBName()
		if err != nil {
			return paths, err
		}
	} else {
		dblist = append(dblist, cronjob.DBName)
	}

	var client cloud_storage.CloudStorageClient
	if backup.Type != "LOCAL" {
		client, err = NewIBackupService().NewClient(&backup)
		if err != nil {
			return paths, err
		}
	}

	for _, dbName := range dblist {
		var record model.BackupRecord

		record.Type = "mysql"
		record.Name = app.Name
		record.Source = "LOCAL"
		record.BackupType = backup.Type

		backupDir := fmt.Sprintf("%s/database/mysql/%s/%s", localDir, app.Name, dbName)
		record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbName, startTime.Format("20060102150405"))
		if err = handleMysqlBackup(app, backupDir, dbName, record.FileName); err != nil {
			return paths, err
		}
		record.DetailName = dbName
		record.FileDir = backupDir
		itemFileDir := strings.ReplaceAll(backupDir, localDir+"/", "")
		if !cronjob.KeepLocal && backup.Type != "LOCAL" {
			record.Source = backup.Type
			record.FileDir = itemFileDir
		}
		paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName))

		if err := backupRepo.CreateRecord(&record); err != nil {
			global.LOG.Errorf("save backup record failed, err: %v", err)
			return paths, err
		}
		if backup.Type != "LOCAL" {
			if !cronjob.KeepLocal {
				defer func() {
					_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName))
				}()
			}
			if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
				return paths, err
			}
		}
	}
	u.HandleRmExpired(backup.Type, localDir, &cronjob, client)
	return paths, nil
}

func (u *CronjobService) handleWebsite(cronjob model.Cronjob, backup model.BackupAccount, startTime time.Time) ([]string, error) {
	var paths []string
	localDir, err := loadLocalDir()
	if err != nil {
		return paths, err
	}

	var weblist []string
	if cronjob.Website == "all" {
		weblist, err = NewIWebsiteService().GetWebsiteOptions()
		if err != nil {
			return paths, err
		}
	} else {
		weblist = append(weblist, cronjob.Website)
	}

	var client cloud_storage.CloudStorageClient
	if backup.Type != "LOCAL" {
		client, err = NewIBackupService().NewClient(&backup)
		if err != nil {
			return paths, err
		}
	}

	for _, websiteItem := range weblist {
		var record model.BackupRecord
		record.Type = "website"
		record.Name = cronjob.Website
		record.Source = "LOCAL"
		record.BackupType = backup.Type
		website, err := websiteRepo.GetFirst(websiteRepo.WithDomain(websiteItem))
		if err != nil {
			return paths, err
		}
		backupDir := fmt.Sprintf("%s/website/%s", localDir, website.PrimaryDomain)
		record.FileDir = backupDir
		itemFileDir := strings.ReplaceAll(backupDir, localDir+"/", "")
		if !cronjob.KeepLocal && backup.Type != "LOCAL" {
			record.Source = backup.Type
			record.FileDir = strings.ReplaceAll(backupDir, localDir+"/", "")
		}
		record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", website.PrimaryDomain, startTime.Format("20060102150405"))
		paths = append(paths, fmt.Sprintf("%s/%s", record.FileDir, record.FileName))
		if err := handleWebsiteBackup(&website, backupDir, record.FileName); err != nil {
			return paths, err
		}
		record.Name = website.PrimaryDomain
		if err := backupRepo.CreateRecord(&record); err != nil {
			global.LOG.Errorf("save backup record failed, err: %v", err)
			return paths, err
		}
		if backup.Type != "LOCAL" {
			if !cronjob.KeepLocal {
				defer func() {
					_ = os.RemoveAll(fmt.Sprintf("%s/%s", backupDir, record.FileName))
				}()
			}
			if _, err = client.Upload(backupDir+"/"+record.FileName, itemFileDir+"/"+record.FileName); err != nil {
				return paths, err
			}
		}
	}
	u.HandleRmExpired(backup.Type, localDir, &cronjob, client)
	return paths, nil
}