From bdd9774aa7684f7eb66f6758d537ee161ee14078 Mon Sep 17 00:00:00 2001 From: Sakana Date: Mon, 27 Jan 2025 20:28:44 +0800 Subject: [PATCH] feat(github_releases): add support for github_releases driver (#7844 close #7842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(github_releases): 添加对 GitHub Releases 的支持 * feat(github_releases): 增加目录大小和更新时间，增加请求缓存 * Feat(github_releases): 可选填入 GitHub token 来提高速率限制或访问私有仓库 * Fix(github_releases): 修复仓库无权限或不存在时的异常 * feat(github_releases): 支持显示所有版本，开启后不显示文件夹大小 * feat(github_releases): 兼容无子目录 --- drivers/all.go | 1 + drivers/github_releases/driver.go | 153 +++++++++++++++++++++ drivers/github_releases/meta.go | 34 +++++ drivers/github_releases/types.go | 68 ++++++++++ drivers/github_releases/util.go | 217 ++++++++++++++++++++++++++++++ 5 files changed, 473 insertions(+) create mode 100644 drivers/github_releases/driver.go create mode 100644 drivers/github_releases/meta.go create mode 100644 drivers/github_releases/types.go create mode 100644 drivers/github_releases/util.go diff --git a/drivers/all.go b/drivers/all.go index 8b253a08..bd051168 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/github" + _ "github.com/alist-org/alist/v3/drivers/github_releases" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go new file mode 100644 index 00000000..79f2b582 --- /dev/null +++ b/drivers/github_releases/driver.go @@ -0,0 +1,153 @@ +package github_releases + +import ( + "context" + "fmt" + "net/http" + "time" + + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type GithubReleases struct { + model.Storage + Addition + + releases []Release +} + +func (d *GithubReleases) Config() driver.Config { + return config +} + +func (d *GithubReleases) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GithubReleases) Init(ctx context.Context) error { + SetHeader(d.Addition.Token) + repos, err := ParseRepos(d.Addition.RepoStructure, d.Addition.ShowAllVersion) + if err != nil { + return err + } + d.releases = repos + return nil +} + +func (d *GithubReleases) Drop(ctx context.Context) error { + ClearCache() + return nil +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + for _, repo := range d.releases { + if repo.Path == path { // 与仓库路径相同 + resp, err := GetRepoReleaseInfo(repo.RepoName, repo.ID, path, d.Storage.CacheExpiration) + if err != nil { + return nil, err + } + files = append(files, resp.Files...) + + if d.Addition.ShowReadme { + resp, err := GetGithubOtherFile(repo.RepoName, path, d.Storage.CacheExpiration) + if err != nil { + return nil, err + } + files = append(files, *resp...) + } + + } else if strings.HasPrefix(repo.Path, path) { // 仓库路径是目录的子目录 + nextDir := GetNextDir(repo.Path, path) + if nextDir == "" { + continue + } + if d.Addition.ShowAllVersion { + files = append(files, File{ + FileName: nextDir, + Size: 0, + CreateAt: time.Time{}, + UpdateAt: time.Time{}, + Url: "", + Type: "dir", + Path: fmt.Sprintf("%s/%s", path, nextDir), + }) + continue + } + + repo, _ := GetRepoReleaseInfo(repo.RepoName, repo.Version, path, d.Storage.CacheExpiration) + + hasSameDir := false + for index, file := range files { + if file.FileName == nextDir { + hasSameDir = true + files[index].Size += repo.Size + files[index].UpdateAt = func(a time.Time, b time.Time) time.Time { + if a.After(b) { + return a + } + return b + }(files[index].UpdateAt, repo.UpdateAt) + break + } + } + + if !hasSameDir { + files = append(files, File{ + FileName: nextDir, + Size: repo.Size, + CreateAt: repo.CreateAt, + UpdateAt: repo.UpdateAt, + Url: repo.Url, + Type: "dir", + Path: fmt.Sprintf("%s/%s", path, nextDir), + }) + } + } + } + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link := model.Link{ + URL: file.GetID(), + Header: http.Header{}, + } + return &link, nil +} + +func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *GithubReleases) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*GithubReleases)(nil) diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go new file mode 100644 index 00000000..ca6ca5dc --- /dev/null +++ b/drivers/github_releases/meta.go @@ -0,0 +1,34 @@ +package github_releases + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"/path/to/alist-gh:alistGo/alist\n/path/to2/alist-web-gh:AlistGo/alist-web" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` +} + +var config = driver.Config{ + Name: "GitHub Releases", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GithubReleases{} + }) +} diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go new file mode 100644 index 00000000..733460dc --- /dev/null +++ b/drivers/github_releases/types.go @@ -0,0 +1,68 @@ +package github_releases + +import ( + "time" + + "github.com/alist-org/alist/v3/pkg/utils" +) + +type File struct { + FileName string `json:"name"` + Size int64 `json:"size"` + CreateAt time.Time `json:"time"` + UpdateAt time.Time `json:"chtime"` + Url string `json:"url"` + Type string `json:"type"` + Path string `json:"path"` +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) ModTime() time.Time { + return f.UpdateAt +} + +func (f File) CreateTime() time.Time { + return f.CreateAt +} + +func (f File) IsDir() bool { + return f.Type == "dir" +} + +func (f File) GetID() string { + return f.Url +} + +func (f File) Thumb() string { + return "" +} + +type ReleasesData struct { + Files []File `json:"files"` + Size int64 `json:"size"` + UpdateAt time.Time `json:"chtime"` + CreateAt time.Time `json:"time"` + Url string `json:"url"` +} + +type Release struct { + Path string // 挂载路径 + RepoName string // 仓库名称 + Version string // 版本号, tag + ID string // 版本ID +} diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go new file mode 100644 index 00000000..b2d79c0b --- /dev/null +++ b/drivers/github_releases/util.go @@ -0,0 +1,217 @@ +package github_releases + +import ( + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" + log "github.com/sirupsen/logrus" +) + +var ( + cache = make(map[string]*resty.Response) + created = make(map[string]time.Time) + mu sync.Mutex + req *resty.Request +) + +// 解析仓库列表 +func ParseRepos(text string, allVersion bool) ([]Release, error) { + lines := strings.Split(text, "\n") + var repos []Release + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ":") + path, repo := "", "" + if len(parts) == 1 { + path = "/" + repo = parts[0] + } else if len(parts) == 2 { + path = fmt.Sprintf("/%s", strings.Trim(parts[0], "/")) + repo = parts[1] + } else { + return nil, fmt.Errorf("invalid format: %s", line) + } + + if allVersion { + releases, _ := GetAllVersion(repo, path) + repos = append(repos, *releases...) + } else { + repos = append(repos, Release{ + Path: path, + RepoName: repo, + Version: "latest", + ID: "latest", + }) + } + + } + return repos, nil +} + +// 获取下一级目录 +func GetNextDir(wholePath string, basePath string) string { + if !strings.HasSuffix(basePath, "/") { + basePath += "/" + } + if !strings.HasPrefix(wholePath, basePath) { + return "" + } + remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/") + if remainingPath != "" { + parts := strings.Split(remainingPath, "/") + return parts[0] + } + return "" +} + +// 发送 GET 请求 +func GetRequest(url string, cacheExpiration int) (*resty.Response, error) { + mu.Lock() + if res, ok := cache[url]; ok && time.Now().Before(created[url].Add(time.Duration(cacheExpiration)*time.Minute)) { + mu.Unlock() + return res, nil + } + mu.Unlock() + + res, err := req.Get(url) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + log.Warn("failed to get request: ", res.StatusCode(), res.String()) + } + + mu.Lock() + cache[url] = res + created[url] = time.Now() + mu.Unlock() + + return res, nil +} + +// 获取 README、LICENSE 等文件 +func GetGithubOtherFile(repo string, basePath string, cacheExpiration int) (*[]File, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/contents/", strings.Trim(repo, "/")) + res, _ := GetRequest(url, cacheExpiration) + body := jsoniter.Get(res.Body()) + var files []File + for i := 0; i < body.Size(); i++ { + filename := body.Get(i, "name").ToString() + + re := regexp.MustCompile(`(?i)^(.*\.md|LICENSE)$`) + + if !re.MatchString(filename) { + continue + } + + files = append(files, File{ + FileName: filename, + Size: body.Get(i, "size").ToInt64(), + CreateAt: time.Time{}, + UpdateAt: time.Now(), + Url: body.Get(i, "download_url").ToString(), + Type: body.Get(i, "type").ToString(), + Path: fmt.Sprintf("%s/%s", basePath, filename), + }) + } + return &files, nil +} + +// 获取 GitHub Release 详细信息 +func GetRepoReleaseInfo(repo string, version string, basePath string, cacheExpiration int) (*ReleasesData, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases/%s", strings.Trim(repo, "/"), version) + res, _ := GetRequest(url, cacheExpiration) + body := res.Body() + + if jsoniter.Get(res.Body(), "status").ToInt64() != 0 { + return &ReleasesData{}, fmt.Errorf("%s", res.String()) + } + + assets := jsoniter.Get(res.Body(), "assets") + var files []File + + for i := 0; i < assets.Size(); i++ { + filename := assets.Get(i, "name").ToString() + + files = append(files, File{ + FileName: filename, + Size: assets.Get(i, "size").ToInt64(), + Url: assets.Get(i, "browser_download_url").ToString(), + Type: assets.Get(i, "content_type").ToString(), + Path: fmt.Sprintf("%s/%s", basePath, filename), + + CreateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, assets.Get(i, "created_at").ToString()) + return t + }(), + UpdateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, assets.Get(i, "updated_at").ToString()) + return t + }(), + }) + } + + return &ReleasesData{ + Files: files, + Url: jsoniter.Get(body, "html_url").ToString(), + + Size: func() int64 { + size := int64(0) + for _, file := range files { + size += file.Size + } + return size + }(), + UpdateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "published_at").ToString()) + return t + }(), + CreateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "created_at").ToString()) + return t + }(), + }, nil +} + +// 获取所有的版本号 +func GetAllVersion(repo string, path string) (*[]Release, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases", strings.Trim(repo, "/")) + res, _ := GetRequest(url, 0) + body := jsoniter.Get(res.Body()) + releases := make([]Release, 0) + for i := 0; i < body.Size(); i++ { + version := body.Get(i, "tag_name").ToString() + releases = append(releases, Release{ + Path: fmt.Sprintf("%s/%s", path, version), + Version: version, + RepoName: repo, + ID: body.Get(i, "id").ToString(), + }) + } + return &releases, nil +} + +func ClearCache() { + mu.Lock() + cache = make(map[string]*resty.Response) + created = make(map[string]time.Time) + mu.Unlock() +} + +func SetHeader(token string) { + req = base.RestyClient.R() + if token != "" { + req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)) + } + req.SetHeader("Accept", "application/vnd.github+json") + req.SetHeader("X-GitHub-Api-Version", "2022-11-28") +}