1Panel/backend/utils/files/file_op.go

517 lines
12 KiB
Go
Raw Normal View History

2022-08-25 09:54:52 +00:00
package files
import (
2023-03-13 05:38:29 +00:00
"archive/zip"
"bufio"
2022-08-30 09:59:59 +00:00
"context"
2022-09-14 11:09:39 +00:00
"encoding/json"
"fmt"
"github.com/1Panel-dev/1Panel/backend/utils/cmd"
http2 "github.com/1Panel-dev/1Panel/backend/utils/http"
cZip "github.com/klauspost/compress/zip"
"golang.org/x/text/encoding/simplifiedchinese"
"golang.org/x/text/transform"
2022-08-25 09:54:52 +00:00
"io"
"io/fs"
2022-09-05 08:25:26 +00:00
"net/http"
2022-08-25 09:54:52 +00:00
"os"
2022-09-06 07:46:46 +00:00
"path"
2022-08-30 09:59:59 +00:00
"path/filepath"
2022-09-14 11:09:39 +00:00
"strconv"
2023-03-13 05:38:29 +00:00
"strings"
"sync"
2022-09-14 11:09:39 +00:00
"time"
2023-01-30 13:05:20 +00:00
"github.com/1Panel-dev/1Panel/backend/global"
"github.com/mholt/archiver/v4"
"github.com/pkg/errors"
"github.com/spf13/afero"
2022-08-25 09:54:52 +00:00
)
type FileOp struct {
Fs afero.Fs
}
func NewFileOp() FileOp {
return FileOp{
Fs: afero.NewOsFs(),
}
}
func (f FileOp) OpenFile(dst string) (fs.File, error) {
return f.Fs.Open(dst)
}
func (f FileOp) GetContent(dst string) ([]byte, error) {
afs := &afero.Afero{Fs: f.Fs}
cByte, err := afs.ReadFile(dst)
if err != nil {
return nil, err
}
return cByte, nil
}
2022-08-25 09:54:52 +00:00
func (f FileOp) CreateDir(dst string, mode fs.FileMode) error {
return f.Fs.MkdirAll(dst, mode)
}
func (f FileOp) CreateFile(dst string) error {
if _, err := f.Fs.Create(dst); err != nil {
return err
}
return nil
}
func (f FileOp) LinkFile(source string, dst string, isSymlink bool) error {
if isSymlink {
osFs := afero.OsFs{}
return osFs.SymlinkIfPossible(source, dst)
} else {
return os.Link(source, dst)
}
}
2022-08-25 10:48:03 +00:00
func (f FileOp) DeleteDir(dst string) error {
return f.Fs.RemoveAll(dst)
}
func (f FileOp) Stat(dst string) bool {
info, _ := f.Fs.Stat(dst)
2022-09-09 03:20:02 +00:00
return info != nil
}
2022-08-25 10:48:03 +00:00
func (f FileOp) DeleteFile(dst string) error {
return f.Fs.Remove(dst)
}
func (f FileOp) Delete(dst string) error {
return os.RemoveAll(dst)
}
func (f FileOp) RmRf(dst string) error {
return cmd.ExecCmd(fmt.Sprintf("rm -rf %s", dst))
}
2022-08-25 09:54:52 +00:00
func (f FileOp) WriteFile(dst string, in io.Reader, mode fs.FileMode) error {
file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
defer file.Close()
if _, err = io.Copy(file, in); err != nil {
return err
}
if _, err = file.Stat(); err != nil {
return err
}
return nil
}
func (f FileOp) SaveFile(dst string, content string, mode fs.FileMode) error {
if !f.Stat(path.Dir(dst)) {
_ = f.CreateDir(path.Dir(dst), mode.Perm())
}
file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
defer file.Close()
write := bufio.NewWriter(file)
_, _ = write.WriteString(content)
write.Flush()
return nil
}
func (f FileOp) Chmod(dst string, mode fs.FileMode) error {
return f.Fs.Chmod(dst, mode)
}
2022-08-30 09:59:59 +00:00
func (f FileOp) Chown(dst string, uid int, gid int) error {
return f.Fs.Chown(dst, uid, gid)
}
func (f FileOp) ChownR(dst string, uid string, gid string, sub bool) error {
cmdStr := fmt.Sprintf(`chown %s:%s "%s"`, uid, gid, dst)
if sub {
cmdStr = fmt.Sprintf(`chown -R %s:%s "%s"`, uid, gid, dst)
}
if cmd.HasNoPasswordSudo() {
cmdStr = fmt.Sprintf("sudo %s", cmdStr)
}
if msg, err := cmd.ExecWithTimeOut(cmdStr, 2*time.Second); err != nil {
if msg != "" {
return errors.New(msg)
}
return err
}
return nil
}
func (f FileOp) ChmodR(dst string, mode int64) error {
cmdStr := fmt.Sprintf(`chmod -R %v "%s"`, fmt.Sprintf("%04o", mode), dst)
if cmd.HasNoPasswordSudo() {
cmdStr = fmt.Sprintf("sudo %s", cmdStr)
}
if msg, err := cmd.ExecWithTimeOut(cmdStr, 2*time.Second); err != nil {
if msg != "" {
return errors.New(msg)
}
return err
}
return nil
}
2022-09-03 14:22:40 +00:00
func (f FileOp) Rename(oldName string, newName string) error {
return f.Fs.Rename(oldName, newName)
}
2022-09-14 11:09:39 +00:00
type WriteCounter struct {
Total uint64
Written uint64
Key string
Name string
}
type Process struct {
Total uint64 `json:"total"`
Written uint64 `json:"written"`
Percent float64 `json:"percent"`
Name string `json:"name"`
}
func (w *WriteCounter) Write(p []byte) (n int, err error) {
n = len(p)
w.Written += uint64(n)
w.SaveProcess()
return n, nil
}
func (w *WriteCounter) SaveProcess() {
percentValue := 0.0
if w.Total > 0 {
percent := float64(w.Written) / float64(w.Total) * 100
percentValue, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", percent), 64)
}
2022-09-14 11:09:39 +00:00
process := Process{
Total: w.Total,
Written: w.Written,
Percent: percentValue,
Name: w.Name,
}
by, _ := json.Marshal(process)
if percentValue < 100 {
if err := global.CACHE.Set(w.Key, string(by)); err != nil {
global.LOG.Errorf("save cache error, err %s", err.Error())
}
} else {
if err := global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10)); err != nil {
global.LOG.Errorf("save cache error, err %s", err.Error())
}
}
}
2022-10-14 06:48:55 +00:00
func (f FileOp) DownloadFileWithProcess(url, dst, key string) error {
client := &http.Client{}
request, err := http.NewRequest("GET", url, nil)
2022-09-05 08:25:26 +00:00
if err != nil {
return nil
2022-09-14 11:09:39 +00:00
}
request.Header.Set("Accept-Encoding", "identity")
resp, err := client.Do(request)
2022-09-14 11:09:39 +00:00
if err != nil {
global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error())
return err
2022-09-05 08:25:26 +00:00
}
out, err := os.Create(dst)
if err != nil {
2022-09-14 11:09:39 +00:00
global.LOG.Errorf("create download file [%s] error, err %s", dst, err.Error())
return err
2022-09-05 08:25:26 +00:00
}
go func() {
2022-09-14 11:09:39 +00:00
counter := &WriteCounter{}
counter.Key = key
if resp.ContentLength > 0 {
counter.Total = uint64(resp.ContentLength)
}
2022-09-14 11:09:39 +00:00
counter.Name = filepath.Base(dst)
if _, err = io.Copy(out, io.TeeReader(resp.Body, counter)); err != nil {
2022-09-05 08:25:26 +00:00
global.LOG.Errorf("save download file [%s] error, err %s", dst, err.Error())
}
2022-09-14 11:09:39 +00:00
out.Close()
resp.Body.Close()
value, err := global.CACHE.Get(counter.Key)
if err != nil {
global.LOG.Errorf("get cache error,err %s", err.Error())
return
}
process := &Process{}
_ = json.Unmarshal(value, process)
process.Percent = 100
process.Name = counter.Name
process.Total = process.Written
by, _ := json.Marshal(process)
if err := global.CACHE.SetWithTTL(counter.Key, string(by), time.Second*time.Duration(10)); err != nil {
global.LOG.Errorf("save cache error, err %s", err.Error())
}
2022-09-05 08:25:26 +00:00
}()
return nil
}
2022-10-14 06:48:55 +00:00
func (f FileOp) DownloadFile(url, dst string) error {
resp, err := http2.GetHttpRes(url)
2022-10-14 06:48:55 +00:00
if err != nil {
return err
2022-10-14 06:48:55 +00:00
}
out, err := os.Create(dst)
if err != nil {
2023-01-30 13:05:20 +00:00
return fmt.Errorf("create download file [%s] error, err %s", dst, err.Error())
2022-10-14 06:48:55 +00:00
}
2023-01-30 13:05:20 +00:00
defer out.Close()
2022-10-14 06:48:55 +00:00
if _, err = io.Copy(out, resp.Body); err != nil {
2023-01-30 13:05:20 +00:00
return fmt.Errorf("save download file [%s] error, err %s", dst, err.Error())
2022-10-14 06:48:55 +00:00
}
out.Close()
resp.Body.Close()
return nil
}
2022-09-06 02:35:35 +00:00
func (f FileOp) Cut(oldPaths []string, dst string) error {
for _, p := range oldPaths {
base := filepath.Base(p)
dstPath := filepath.Join(dst, base)
if err := cmd.ExecCmd(fmt.Sprintf("mv %s %s", p, dstPath)); err != nil {
2022-09-06 02:35:35 +00:00
return err
}
}
return nil
}
2022-09-06 07:46:46 +00:00
func (f FileOp) Copy(src, dst string) error {
if src = path.Clean("/" + src); src == "" {
return os.ErrNotExist
}
if dst = path.Clean("/" + dst); dst == "" {
return os.ErrNotExist
}
if src == "/" || dst == "/" {
return os.ErrInvalid
}
if dst == src {
return os.ErrInvalid
}
info, err := f.Fs.Stat(src)
if err != nil {
return err
}
if info.IsDir() {
return f.CopyDir(src, dst)
}
return f.CopyFile(src, dst)
}
func (f FileOp) CopyDir(src, dst string) error {
srcInfo, err := f.Fs.Stat(src)
if err != nil {
return err
}
dstDir := filepath.Join(dst, srcInfo.Name())
if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil {
2022-09-06 07:46:46 +00:00
return err
}
return cmd.ExecCmd(fmt.Sprintf("cp -rf %s %s", src, dst+"/"))
2022-09-06 07:46:46 +00:00
}
func (f FileOp) CopyFile(src, dst string) error {
return cmd.ExecCmd(fmt.Sprintf("cp -f %s %s", src, dst+"/"))
2022-09-06 07:46:46 +00:00
}
func (f FileOp) GetDirSize(path string) (float64, error) {
var m sync.Map
var wg sync.WaitGroup
wg.Add(1)
go ScanDir(f.Fs, path, &m, &wg)
wg.Wait()
var dirSize float64
m.Range(func(k, v interface{}) bool {
dirSize = dirSize + v.(float64)
return true
})
return dirSize, nil
}
2022-08-30 09:59:59 +00:00
func getFormat(cType CompressType) archiver.CompressedArchive {
format := archiver.CompressedArchive{}
switch cType {
case Tar:
format.Archival = archiver.Tar{}
case TarGz, Gz:
format.Compression = archiver.Gz{}
format.Archival = archiver.Tar{}
case Zip:
2023-03-13 05:38:29 +00:00
format.Archival = archiver.Zip{
Compression: zip.Deflate,
}
2022-08-30 09:59:59 +00:00
case Bz2:
format.Compression = archiver.Bz2{}
format.Archival = archiver.Tar{}
case Xz:
format.Compression = archiver.Xz{}
format.Archival = archiver.Tar{}
}
return format
}
func (f FileOp) Compress(srcRiles []string, dst string, name string, cType CompressType) error {
format := getFormat(cType)
fileMaps := make(map[string]string, len(srcRiles))
for _, s := range srcRiles {
base := filepath.Base(s)
fileMaps[s] = base
}
2022-10-12 10:57:22 +00:00
if !f.Stat(dst) {
_ = f.CreateDir(dst, 0755)
}
2022-08-30 09:59:59 +00:00
files, err := archiver.FilesFromDisk(nil, fileMaps)
if err != nil {
return err
}
dstFile := filepath.Join(dst, name)
out, err := f.Fs.Create(dstFile)
if err != nil {
return err
}
2023-03-13 05:38:29 +00:00
switch cType {
case Zip:
return NewZipArchiver().Compress(srcRiles, dstFile)
2023-03-13 05:38:29 +00:00
default:
err = format.Archive(context.Background(), out, files)
if err != nil {
_ = f.DeleteFile(dstFile)
return err
}
2022-08-30 09:59:59 +00:00
}
return nil
}
2023-05-18 08:48:19 +00:00
func isIgnoreFile(name string) bool {
return strings.HasPrefix(name, "__MACOSX") || strings.HasSuffix(name, ".DS_Store") || strings.HasPrefix(name, "._")
}
func decodeGBK(input string) (string, error) {
decoder := simplifiedchinese.GBK.NewDecoder()
decoded, _, err := transform.String(decoder, input)
if err != nil {
return "", err
}
return decoded, nil
}
2022-08-30 09:59:59 +00:00
func (f FileOp) Decompress(srcFile string, dst string, cType CompressType) error {
switch cType {
case Tar, Zip:
shellArchiver, err := NewShellArchiver(cType)
if err != nil {
return err
}
return shellArchiver.Extract(srcFile, dst)
default:
format := getFormat(cType)
handler := func(ctx context.Context, archFile archiver.File) error {
info := archFile.FileInfo
if isIgnoreFile(archFile.Name()) {
return nil
}
fileName := archFile.NameInArchive
var err error
if header, ok := archFile.Header.(cZip.FileHeader); ok {
if header.NonUTF8 && header.Flags == 0 {
fileName, err = decodeGBK(fileName)
if err != nil {
return err
}
}
}
filePath := filepath.Join(dst, fileName)
if archFile.FileInfo.IsDir() {
if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil {
return err
}
return nil
} else {
parentDir := path.Dir(filePath)
if !f.Stat(parentDir) {
if err := f.Fs.MkdirAll(parentDir, info.Mode()); err != nil {
return err
}
}
}
fr, err := archFile.Open()
if err != nil {
2022-08-30 09:59:59 +00:00
return err
}
defer fr.Close()
fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode())
if err != nil {
return err
}
defer fw.Close()
if _, err := io.Copy(fw, fr); err != nil {
return err
}
return nil
2022-08-30 09:59:59 +00:00
}
input, err := f.Fs.Open(srcFile)
2022-08-30 09:59:59 +00:00
if err != nil {
return err
}
return format.Extract(context.Background(), input, nil, handler)
2022-08-30 09:59:59 +00:00
}
}
2022-10-14 06:48:55 +00:00
func (f FileOp) Backup(srcFile string) (string, error) {
backupPath := srcFile + "_bak"
info, _ := f.Fs.Stat(backupPath)
if info != nil {
if info.IsDir() {
2022-11-18 10:02:14 +00:00
_ = f.DeleteDir(backupPath)
2022-10-14 06:48:55 +00:00
} else {
2022-11-18 10:02:14 +00:00
_ = f.DeleteFile(backupPath)
2022-10-14 06:48:55 +00:00
}
}
if err := f.Rename(srcFile, backupPath); err != nil {
return backupPath, err
}
2022-12-12 04:01:33 +00:00
return backupPath, nil
}
func (f FileOp) CopyAndBackup(src string) (string, error) {
backupPath := src + "_bak"
info, _ := f.Fs.Stat(backupPath)
if info != nil {
if info.IsDir() {
_ = f.DeleteDir(backupPath)
} else {
_ = f.DeleteFile(backupPath)
}
}
_ = f.CreateDir(backupPath, 0755)
if err := f.Copy(src, backupPath); err != nil {
return backupPath, err
}
2022-10-14 06:48:55 +00:00
return backupPath, nil
}