github
/
alist
mirror of https://github.com/Xhofe/alist
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(github_releases): support dir size for show all version (#7938) 

* refactor

* 修改默认 RepoStructure

* feat: 支持使用 gh-proxy
pull/7976/head
Sakana 2025-02-09 18:30:38 +08:00 committed by GitHub
parent 6164e4577b
commit f795807753
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 386 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
drivers/github_releases/driver.go
View File

@ -4,8 +4,6 @@ import (
"context"
"fmt"
"net/http"
"time"
"strings"
"github.com/alist-org/alist/v3/internal/driver"
@ -18,7 +16,7 @@ type GithubReleases struct {
model.Storage
Addition
releases []Release
points []MountPoint
}
func (d *GithubReleases) Config() driver.Config {
@ -30,17 +28,11 @@ func (d *GithubReleases) GetAddition() driver.Additional {
}
func (d *GithubReleases) Init(ctx context.Context) error {
SetHeader(d.Addition.Token)
repos, err := ParseRepos(d.Addition.RepoStructure, d.Addition.ShowAllVersion)
if err != nil {
return err
}
d.releases = repos
d.ParseRepos(d.Addition.RepoStructure)
return nil
}
func (d *GithubReleases) Drop(ctx context.Context) error {
ClearCache()
return nil
}
@ -48,68 +40,84 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis
files := make([]File, 0)
path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/"))
for _, repo := range d.releases {
if repo.Path == path { // 与仓库路径相同
resp, err := GetRepoReleaseInfo(repo.RepoName, repo.ID, path, d.Storage.CacheExpiration)
if err != nil {
return nil, err
}
files = append(files, resp.Files...)
for i := range d.points {
point := &d.points[i]
if !d.Addition.ShowAllVersion { // latest
point.RequestRelease(d.GetRequest, args.Refresh)
if point.Point == path { // 与仓库路径相同
files = append(files, point.GetLatestRelease()...)
if d.Addition.ShowReadme {
resp, err := GetGithubOtherFile(repo.RepoName, path, d.Storage.CacheExpiration)
if err != nil {
return nil, err
files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...)
}
files = append(files, *resp...)
}
} else if strings.HasPrefix(repo.Path, path) { // 仓库路径是目录的子目录
nextDir := GetNextDir(repo.Path, path)
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
nextDir := GetNextDir(point.Point, path)
if nextDir == "" {
continue
}
if d.Addition.ShowAllVersion {
files = append(files, File{
FileName: nextDir,
Size: 0,
CreateAt: time.Time{},
UpdateAt: time.Time{},
Url: "",
Type: "dir",
Path: fmt.Sprintf("%s/%s", path, nextDir),
})
continue
}
repo, _ := GetRepoReleaseInfo(repo.RepoName, repo.Version, path, d.Storage.CacheExpiration)
hasSameDir := false
for index, file := range files {
if file.FileName == nextDir {
for index := range files {
if files[index].GetName() == nextDir {
hasSameDir = true
files[index].Size += repo.Size
files[index].UpdateAt = func(a time.Time, b time.Time) time.Time {
if a.After(b) {
return a
}
return b
}(files[index].UpdateAt, repo.UpdateAt)
files[index].Size += point.GetLatestSize()
break
}
}
if !hasSameDir {
files = append(files, File{
Path: path + "/" + nextDir,
FileName: nextDir,
Size: point.GetLatestSize(),
UpdateAt: point.Release.PublishedAt,
CreateAt: point.Release.CreatedAt,
Type: "dir",
Url: "",
})
}
}
} else { // all version
point.RequestReleases(d.GetRequest, args.Refresh)
if point.Point == path { // 与仓库路径相同
files = append(files, point.GetAllVersion()...)
if d.Addition.ShowReadme {
files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...)
}
} else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录
nextDir := GetNextDir(point.Point, path)
if nextDir == "" {
continue
}
hasSameDir := false
for index := range files {
if files[index].GetName() == nextDir {
hasSameDir = true
files[index].Size += point.GetAllVersionSize()
break
}
}
if !hasSameDir {
files = append(files, File{
FileName: nextDir,
Size: repo.Size,
CreateAt: repo.CreateAt,
UpdateAt: repo.UpdateAt,
Url: repo.Url,
Path: path + "/" + nextDir,
Size: point.GetAllVersionSize(),
UpdateAt: (*point.Releases)[0].PublishedAt,
CreateAt: (*point.Releases)[0].CreatedAt,
Type: "dir",
Path: fmt.Sprintf("%s/%s", path, nextDir),
Url: "",
})
}
} else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录
tagName := GetNextDir(path, point.Point)
if tagName == "" {
continue
}
files = append(files, point.GetReleaseByTagName(tagName)...)
}
}
}
@ -119,35 +127,41 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis
}
func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
url := file.GetID()
gh_proxy := strings.TrimSpace(d.Addition.GitHubProxy)
if gh_proxy != "" {
url = strings.Replace(url, "https://github.com", gh_proxy, 1)
}
link := model.Link{
URL: file.GetID(),
URL: url,
Header: http.Header{},
}
return &link, nil
}
func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) {
// TODO create folder, optional
return nil, errs.NotImplement
}
func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
// TODO move obj, optional
return nil, errs.NotImplement
}
func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) {
// TODO rename obj, optional
return nil, errs.NotImplement
}
func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) {
// TODO copy obj, optional
return nil, errs.NotImplement
}
func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error {
// TODO remove obj, optional
return errs.NotImplement
}
func (d *GithubReleases) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
return nil, errs.NotImplement
}
var _ driver.Driver = (*GithubReleases)(nil)

3
drivers/github_releases/meta.go
View File

@ -7,10 +7,11 @@ import (
type Addition struct {
driver.RootID
RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"/path/to/alist-gh:alistGo/alist\n/path/to2/alist-web-gh:AlistGo/alist-web" help:"structure:[path:]org/repo"`
RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"`
ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"`
Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"`
ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"`
GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "`
}
var config = driver.Config{

86
drivers/github_releases/models.go Normal file
View File

@ -0,0 +1,86 @@
package github_releases
type Release struct {
Url string `json:"url"`
AssetsUrl string `json:"assets_url"`
UploadUrl string `json:"upload_url"`
HtmlUrl string `json:"html_url"`
Id int `json:"id"`
Author User `json:"author"`
NodeId string `json:"node_id"`
TagName string `json:"tag_name"`
TargetCommitish string `json:"target_commitish"`
Name string `json:"name"`
Draft bool `json:"draft"`
Prerelease bool `json:"prerelease"`
CreatedAt string `json:"created_at"`
PublishedAt string `json:"published_at"`
Assets []Asset `json:"assets"`
TarballUrl string `json:"tarball_url"`
ZipballUrl string `json:"zipball_url"`
Body string `json:"body"`
Reactions Reactions `json:"reactions"`
}
type User struct {
Login string `json:"login"`
Id int `json:"id"`
NodeId string `json:"node_id"`
AvatarUrl string `json:"avatar_url"`
GravatarId string `json:"gravatar_id"`
Url string `json:"url"`
HtmlUrl string `json:"html_url"`
FollowersUrl string `json:"followers_url"`
FollowingUrl string `json:"following_url"`
GistsUrl string `json:"gists_url"`
StarredUrl string `json:"starred_url"`
SubscriptionsUrl string `json:"subscriptions_url"`
OrganizationsUrl string `json:"organizations_url"`
ReposUrl string `json:"repos_url"`
EventsUrl string `json:"events_url"`
ReceivedEventsUrl string `json:"received_events_url"`
Type string `json:"type"`
UserViewType string `json:"user_view_type"`
SiteAdmin bool `json:"site_admin"`
}
type Asset struct {
Url string `json:"url"`
Id int `json:"id"`
NodeId string `json:"node_id"`
Name string `json:"name"`
Label string `json:"label"`
Uploader User `json:"uploader"`
ContentType string `json:"content_type"`
State string `json:"state"`
Size int64 `json:"size"`
DownloadCount int `json:"download_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
BrowserDownloadUrl string `json:"browser_download_url"`
}
type Reactions struct {
Url string `json:"url"`
TotalCount int `json:"total_count"`
PlusOne int `json:"+1"`
MinusOne int `json:"-1"`
Laugh int `json:"laugh"`
Hooray int `json:"hooray"`
Confused int `json:"confused"`
Heart int `json:"heart"`
Rocket int `json:"rocket"`
Eyes int `json:"eyes"`
}
type FileInfo struct {
Name string `json:"name"`
Path string `json:"path"`
Sha string `json:"sha"`
Size int64 `json:"size"`
Url string `json:"url"`
HtmlUrl string `json:"html_url"`
GitUrl string `json:"git_url"`
DownloadUrl string `json:"download_url"`
Type string `json:"type"`
}

201
drivers/github_releases/types.go
View File

@ -1,19 +1,181 @@
package github_releases
import (
"encoding/json"
"strings"
"time"
"github.com/alist-org/alist/v3/pkg/utils"
"github.com/go-resty/resty/v2"
)
type MountPoint struct {
Point string // 挂载点
Repo string // 仓库名 owner/repo
Release *Release // Release 指针 latest
Releases *[]Release // []Release 指针
OtherFile *[]FileInfo // 仓库根目录下的其他文件
}
// 请求最新版本
func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) {
if m.Repo == "" {
return
}
if m.Release == nil || refresh {
resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest")
m.Release = new(Release)
json.Unmarshal(resp.Body(), m.Release)
}
}
// 请求所有版本
func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) {
if m.Repo == "" {
return
}
if m.Releases == nil || refresh {
resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases")
m.Releases = new([]Release)
json.Unmarshal(resp.Body(), m.Releases)
}
}
// 获取最新版本
func (m *MountPoint) GetLatestRelease() []File {
files := make([]File, 0)
for _, asset := range m.Release.Assets {
files = append(files, File{
Path: m.Point + "/" + asset.Name,
FileName: asset.Name,
Size: asset.Size,
Type: "file",
UpdateAt: asset.UpdatedAt,
CreateAt: asset.CreatedAt,
Url: asset.BrowserDownloadUrl,
})
}
return files
}
// 获取最新版本大小
func (m *MountPoint) GetLatestSize() int64 {
size := int64(0)
for _, asset := range m.Release.Assets {
size += asset.Size
}
return size
}
// 获取所有版本
func (m *MountPoint) GetAllVersion() []File {
files := make([]File, 0)
for _, release := range *m.Releases {
file := File{
Path: m.Point + "/" + release.TagName,
FileName: release.TagName,
Size: m.GetSizeByTagName(release.TagName),
Type: "dir",
UpdateAt: release.PublishedAt,
CreateAt: release.CreatedAt,
Url: release.HtmlUrl,
}
for _, asset := range release.Assets {
file.Size += asset.Size
}
files = append(files, file)
}
return files
}
// 根据版本号获取版本
func (m *MountPoint) GetReleaseByTagName(tagName string) []File {
for _, item := range *m.Releases {
if item.TagName == tagName {
files := make([]File, 0)
for _, asset := range item.Assets {
files = append(files, File{
Path: m.Point + "/" + tagName + "/" + asset.Name,
FileName: asset.Name,
Size: asset.Size,
Type: "file",
UpdateAt: asset.UpdatedAt,
CreateAt: asset.CreatedAt,
Url: asset.BrowserDownloadUrl,
})
}
return files
}
}
return nil
}
// 根据版本号获取版本大小
func (m *MountPoint) GetSizeByTagName(tagName string) int64 {
if m.Releases == nil {
return 0
}
for _, item := range *m.Releases {
if item.TagName == tagName {
size := int64(0)
for _, asset := range item.Assets {
size += asset.Size
}
return size
}
}
return 0
}
// 获取所有版本大小
func (m *MountPoint) GetAllVersionSize() int64 {
if m.Releases == nil {
return 0
}
size := int64(0)
for _, release := range *m.Releases {
for _, asset := range release.Assets {
size += asset.Size
}
}
return size
}
func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File {
if m.OtherFile == nil || refresh {
resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents")
m.OtherFile = new([]FileInfo)
json.Unmarshal(resp.Body(), m.OtherFile)
}
files := make([]File, 0)
defaultTime := "1970-01-01T00:00:00Z"
for _, file := range *m.OtherFile {
if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") {
files = append(files, File{
Path: m.Point + "/" + file.Name,
FileName: file.Name,
Size: file.Size,
Type: "file",
UpdateAt: defaultTime,
CreateAt: defaultTime,
Url: file.DownloadUrl,
})
}
}
return files
}
type File struct {
FileName string `json:"name"`
Size int64 `json:"size"`
CreateAt time.Time `json:"time"`
UpdateAt time.Time `json:"chtime"`
Url string `json:"url"`
Type string `json:"type"`
Path string `json:"path"`
Path string // 文件路径
FileName string // 文件名
Size int64 // 文件大小
Type string // 文件类型
UpdateAt string // 更新时间 eg:"2025-01-27T16:10:16Z"
CreateAt string // 创建时间
Url string // 下载链接
}
func (f File) GetHash() utils.HashInfo {
@ -33,11 +195,13 @@ func (f File) GetName() string {
}
func (f File) ModTime() time.Time {
return f.UpdateAt
t, _ := time.Parse(time.RFC3339, f.CreateAt)
return t
}
func (f File) CreateTime() time.Time {
return f.CreateAt
t, _ := time.Parse(time.RFC3339, f.CreateAt)
return t
}
func (f File) IsDir() bool {
@ -47,22 +211,3 @@ func (f File) IsDir() bool {
func (f File) GetID() string {
return f.Url
}
func (f File) Thumb() string {
return ""
}
type ReleasesData struct {
Files []File `json:"files"`
Size int64 `json:"size"`
UpdateAt time.Time `json:"chtime"`
CreateAt time.Time `json:"time"`
Url string `json:"url"`
}
type Release struct {
Path string // 挂载路径
RepoName string // 仓库名称
Version string // 版本号, tag
ID string // 版本ID
}

208
drivers/github_releases/util.go
View File

@ -2,28 +2,36 @@ package github_releases
import (
"fmt"
"regexp"
"path/filepath"
"strings"
"sync"
"time"
"github.com/alist-org/alist/v3/drivers/base"
"github.com/go-resty/resty/v2"
jsoniter "github.com/json-iterator/go"
log "github.com/sirupsen/logrus"
)
var (
cache = make(map[string]*resty.Response)
created = make(map[string]time.Time)
mu sync.Mutex
req *resty.Request
)
// 发送 GET 请求
func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) {
req := base.RestyClient.R()
req.SetHeader("Accept", "application/vnd.github+json")
req.SetHeader("X-GitHub-Api-Version", "2022-11-28")
if d.Addition.Token != "" {
req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", d.Addition.Token))
}
res, err := req.Get(url)
if err != nil {
return nil, err
}
if res.StatusCode() != 200 {
log.Warn("failed to get request: ", res.StatusCode(), res.String())
}
return res, nil
}
// 解析仓库列表
func ParseRepos(text string, allVersion bool) ([]Release, error) {
// 解析挂载结构
func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) {
lines := strings.Split(text, "\n")
var repos []Release
points := make([]MountPoint, 0)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
@ -41,177 +49,37 @@ func ParseRepos(text string, allVersion bool) ([]Release, error) {
return nil, fmt.Errorf("invalid format: %s", line)
}
if allVersion {
releases, _ := GetAllVersion(repo, path)
repos = append(repos, *releases...)
} else {
repos = append(repos, Release{
Path: path,
RepoName: repo,
Version: "latest",
ID: "latest",
points = append(points, MountPoint{
Point: path,
Repo: repo,
Release: nil,
Releases: nil,
})
}
}
return repos, nil
d.points = points
return points, nil
}
// 获取下一级目录
func GetNextDir(wholePath string, basePath string) string {
if !strings.HasSuffix(basePath, "/") {
basePath += "/"
}
basePath = fmt.Sprintf("%s/", strings.TrimRight(basePath, "/"))
if !strings.HasPrefix(wholePath, basePath) {
return ""
}
remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/")
if remainingPath != "" {
parts := strings.Split(remainingPath, "/")
return parts[0]
nextDir := parts[0]
if strings.HasPrefix(wholePath, strings.TrimRight(basePath, "/")+"/"+nextDir) {
return nextDir
}
}
return ""
}
// 发送 GET 请求
func GetRequest(url string, cacheExpiration int) (*resty.Response, error) {
mu.Lock()
if res, ok := cache[url]; ok && time.Now().Before(created[url].Add(time.Duration(cacheExpiration)*time.Minute)) {
mu.Unlock()
return res, nil
}
mu.Unlock()
res, err := req.Get(url)
if err != nil {
return nil, err
}
if res.StatusCode() != 200 {
log.Warn("failed to get request: ", res.StatusCode(), res.String())
}
mu.Lock()
cache[url] = res
created[url] = time.Now()
mu.Unlock()
return res, nil
}
// 获取 README、LICENSE 等文件
func GetGithubOtherFile(repo string, basePath string, cacheExpiration int) (*[]File, error) {
url := fmt.Sprintf("https://api.github.com/repos/%s/contents/", strings.Trim(repo, "/"))
res, _ := GetRequest(url, cacheExpiration)
body := jsoniter.Get(res.Body())
var files []File
for i := 0; i < body.Size(); i++ {
filename := body.Get(i, "name").ToString()
re := regexp.MustCompile(`(?i)^(.*\.md|LICENSE)$`)
if !re.MatchString(filename) {
continue
}
files = append(files, File{
FileName: filename,
Size: body.Get(i, "size").ToInt64(),
CreateAt: time.Time{},
UpdateAt: time.Now(),
Url: body.Get(i, "download_url").ToString(),
Type: body.Get(i, "type").ToString(),
Path: fmt.Sprintf("%s/%s", basePath, filename),
})
}
return &files, nil
}
// 获取 GitHub Release 详细信息
func GetRepoReleaseInfo(repo string, version string, basePath string, cacheExpiration int) (*ReleasesData, error) {
url := fmt.Sprintf("https://api.github.com/repos/%s/releases/%s", strings.Trim(repo, "/"), version)
res, _ := GetRequest(url, cacheExpiration)
body := res.Body()
if jsoniter.Get(res.Body(), "status").ToInt64() != 0 {
return &ReleasesData{}, fmt.Errorf("%s", res.String())
}
assets := jsoniter.Get(res.Body(), "assets")
var files []File
for i := 0; i < assets.Size(); i++ {
filename := assets.Get(i, "name").ToString()
files = append(files, File{
FileName: filename,
Size: assets.Get(i, "size").ToInt64(),
Url: assets.Get(i, "browser_download_url").ToString(),
Type: assets.Get(i, "content_type").ToString(),
Path: fmt.Sprintf("%s/%s", basePath, filename),
CreateAt: func() time.Time {
t, _ := time.Parse(time.RFC3339, assets.Get(i, "created_at").ToString())
return t
}(),
UpdateAt: func() time.Time {
t, _ := time.Parse(time.RFC3339, assets.Get(i, "updated_at").ToString())
return t
}(),
})
}
return &ReleasesData{
Files: files,
Url: jsoniter.Get(body, "html_url").ToString(),
Size: func() int64 {
size := int64(0)
for _, file := range files {
size += file.Size
}
return size
}(),
UpdateAt: func() time.Time {
t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "published_at").ToString())
return t
}(),
CreateAt: func() time.Time {
t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "created_at").ToString())
return t
}(),
}, nil
}
// 获取所有的版本号
func GetAllVersion(repo string, path string) (*[]Release, error) {
url := fmt.Sprintf("https://api.github.com/repos/%s/releases", strings.Trim(repo, "/"))
res, _ := GetRequest(url, 0)
body := jsoniter.Get(res.Body())
releases := make([]Release, 0)
for i := 0; i < body.Size(); i++ {
version := body.Get(i, "tag_name").ToString()
releases = append(releases, Release{
Path: fmt.Sprintf("%s/%s", path, version),
Version: version,
RepoName: repo,
ID: body.Get(i, "id").ToString(),
})
}
return &releases, nil
}
func ClearCache() {
mu.Lock()
cache = make(map[string]*resty.Response)
created = make(map[string]time.Time)
mu.Unlock()
}
func SetHeader(token string) {
req = base.RestyClient.R()
if token != "" {
req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", token))
}
req.SetHeader("Accept", "application/vnd.github+json")
req.SetHeader("X-GitHub-Api-Version", "2022-11-28")
// 判断当前目录是否是目标目录的祖先目录
func IsAncestorDir(parentDir string, targetDir string) bool {
absTargetDir, _ := filepath.Abs(targetDir)
absParentDir, _ := filepath.Abs(parentDir)
return strings.HasPrefix(absTargetDir, absParentDir)
}