feat: 优化应用商店同步 (#2795)

backend/app/api/v1/app.go

@@ -44,8 +44,13 @@ func (b *BaseApi) SyncApp(c *gin.Context) {
 		helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
 		return
 	}
+
 	if !res.CanUpdate {
-		helper.SuccessWithMsg(c, i18n.GetMsgByKey("AppStoreIsUpToDate"))
+		if res.IsSyncing {
+			helper.SuccessWithMsg(c, i18n.GetMsgByKey("AppStoreIsSyncing"))
+		} else {
+			helper.SuccessWithMsg(c, i18n.GetMsgByKey("AppStoreIsUpToDate"))
+		}
 		return
 	}
 	go func() {

backend/app/dto/response/app.go

@@ -15,6 +15,7 @@ type AppRes struct {
 
 type AppUpdateRes struct {
 	CanUpdate            bool         `json:"canUpdate"`
+	IsSyncing            bool         `json:"isSyncing"`
 	AppStoreLastModified int          `json:"appStoreLastModified"`
 	AppList              *dto.AppList `json:"appList"`
 }

backend/app/dto/setting.go

@@ -48,6 +48,7 @@ type SettingInfo struct {
 
 	AppStoreVersion      string `json:"appStoreVersion"`
 	AppStoreLastModified string `json:"appStoreLastModified"`
+	AppStoreSyncStatus   string `json:"appStoreSyncStatus"`
 }
 
 type SettingUpdate struct {

backend/app/service/app.go

@@ -222,6 +222,25 @@ func (a AppService) GetAppDetail(appID uint, version, appType string) (response.
 		appDetailDTO.Params = paramMap
 	}
 
+	if appDetailDTO.DockerCompose == "" {
+		filename := path.Base(appDetailDTO.DownloadUrl)
+		dockerComposeUrl := fmt.Sprintf("%s%s", strings.TrimSuffix(appDetailDTO.DownloadUrl, filename), "docker-compose.yml")
+		composeRes, err := http.Get(dockerComposeUrl)
+		if err != nil {
+			return appDetailDTO, buserr.WithDetail("ErrGetCompose", err.Error(), err)
+		}
+		bodyContent, err := io.ReadAll(composeRes.Body)
+		if err != nil {
+			return appDetailDTO, buserr.WithDetail("ErrGetCompose", err.Error(), err)
+		}
+		if composeRes.StatusCode > 200 {
+			return appDetailDTO, buserr.WithDetail("ErrGetCompose", string(bodyContent), err)
+		}
+		detail.DockerCompose = string(bodyContent)
+		_ = appDetailRepo.Update(context.Background(), detail)
+		appDetailDTO.DockerCompose = string(bodyContent)
+	}
+
 	appDetailDTO.HostMode = isHostModel(appDetailDTO.DockerCompose)
 
 	app, err := appRepo.GetFirst(commonRepo.WithByID(detail.AppId))
@@ -696,6 +715,11 @@ func (a AppService) GetAppUpdate() (*response.AppUpdateRes, error) {
 	if err != nil {
 		return nil, err
 	}
+	if setting.AppStoreSyncStatus == constant.Syncing {
+		res.IsSyncing = true
+		return res, nil
+	}
+
 	appStoreLastModified, _ := strconv.Atoi(setting.AppStoreLastModified)
 	res.AppStoreLastModified = appStoreLastModified
 	if setting.AppStoreLastModified == "" || lastModified != appStoreLastModified {
@@ -722,7 +746,7 @@ func getAppFromRepo(downloadPath string) error {
 	if err := fileOp.DownloadFile(downloadUrl, packagePath); err != nil {
 		return err
 	}
-	if err := fileOp.Decompress(packagePath, constant.ResourceDir, files.Zip); err != nil {
+	if err := fileOp.Decompress(packagePath, constant.ResourceDir, files.SdkZip); err != nil {
 		return err
 	}
 	defer func() {
@@ -747,6 +771,12 @@ func getAppList() (*dto.AppList, error) {
 	return list, nil
 }
 
+var InitTypes = map[string]struct{}{
+	"runtime": {},
+	"php":     {},
+	"node":    {},
+}
+
 func (a AppService) SyncAppListFromRemote() (err error) {
 	global.LOG.Infof("Starting synchronization with App Store...")
 	updateRes, err := a.GetAppUpdate()
@@ -754,9 +784,14 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 		return err
 	}
 	if !updateRes.CanUpdate {
+		if updateRes.IsSyncing {
+			global.LOG.Infof("AppStore is Syncing!")
+			return
+		}
 		global.LOG.Infof("The App Store is at the latest version")
 		return
 	}
+
 	list := &dto.AppList{}
 	if updateRes.AppList == nil {
 		list, err = getAppList()
@@ -766,16 +801,8 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 	} else {
 		list = updateRes.AppList
 	}
-
+	settingService := NewISettingService()
-	if err = NewISettingService().Update("AppStoreLastModified", strconv.Itoa(list.LastModified)); err != nil {
+	_ = settingService.Update("AppStoreSyncStatus", constant.Syncing)
-		return err
-	}
-
-	defer func() {
-		if err != nil {
-			_ = NewISettingService().Update("AppStoreLastModified", strconv.Itoa(updateRes.AppStoreLastModified))
-		}
-	}()
 
 	var (
 		tags      []*model.Tag
@@ -799,6 +826,8 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 
 	baseRemoteUrl := fmt.Sprintf("%s/%s/1panel", global.CONF.System.AppRepo, global.CONF.System.Mode)
 	appsMap := getApps(oldApps, list.Apps)
+
+	global.LOG.Infof("Starting synchronization of application details...")
 	for _, l := range list.Apps {
 		app := appsMap[l.AppProperty.Key]
 		iconRes, err := http.Get(l.Icon)
@@ -825,16 +854,21 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 			version := v.Name
 			detail := detailsMap[version]
 			versionUrl := fmt.Sprintf("%s/%s/%s", baseRemoteUrl, app.Key, version)
-			dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml")
+
-			composeRes, err := http.Get(dockerComposeUrl)
+			if _, ok := InitTypes[app.Type]; ok {
-			if err != nil {
+				dockerComposeUrl := fmt.Sprintf("%s/%s", versionUrl, "docker-compose.yml")
-				return err
+				composeRes, err := http.Get(dockerComposeUrl)
+				if err != nil {
+					return err
+				}
+				bodyContent, err := io.ReadAll(composeRes.Body)
+				if err != nil {
+					return err
+				}
+				detail.DockerCompose = string(bodyContent)
+			} else {
+				detail.DockerCompose = ""
 			}
-			bodyContent, err := io.ReadAll(composeRes.Body)
-			if err != nil {
-				return err
-			}
-			detail.DockerCompose = string(bodyContent)
 
 			paramByte, _ := json.Marshal(v.AppForm)
 			detail.Params = string(paramByte)
@@ -852,6 +886,8 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 		appsMap[l.AppProperty.Key] = app
 	}
 
+	global.LOG.Infof("Synchronization of application details Success")
+
 	var (
 		addAppArray    []model.App
 		updateAppArray []model.App
@@ -976,6 +1012,10 @@ func (a AppService) SyncAppListFromRemote() (err error) {
 		}
 	}
 	tx.Commit()
+
+	_ = settingService.Update("AppStoreSyncStatus", constant.SyncSuccess)
+	_ = settingService.Update("AppStoreLastModified", strconv.Itoa(list.LastModified))
+
 	global.LOG.Infof("Synchronization with the App Store was successful!")
 	return
 }

backend/app/service/app_utils.go

@@ -649,7 +649,7 @@ func downloadApp(app model.App, appDetail model.AppDetail, appInstall *model.App
 		global.LOG.Errorf("download app[%s] error %v", app.Name, err)
 		return
 	}
-	if err = fileOp.Decompress(filePath, appResourceDir, files.TarGz); err != nil {
+	if err = fileOp.Decompress(filePath, appResourceDir, files.SdkTarGz); err != nil {
 		global.LOG.Errorf("decompress app[%s] error %v", app.Name, err)
 		return
 	}

backend/app/service/website_utils.go

@@ -679,7 +679,7 @@ func changeServiceName(composePath, newServiceName string) (composeByte []byte,
 
 func getWebsiteDomains(domains string, defaultPort int, websiteID uint) (domainModels []model.WebsiteDomain, addPorts []int, addDomains []string, err error) {
 	var (
-		ports map[int]struct{}
+		ports = make(map[int]struct{})
 	)
 	domainArray := strings.Split(domains, "\n")
 	for _, domain := range domainArray {

backend/constant/app.go

@@ -6,13 +6,13 @@ const (
 	Error       = "Error"
 	Stopped     = "Stopped"
 	Installing  = "Installing"
-	Syncing     = "Syncing"
 	DownloadErr = "DownloadErr"
-	DirNotFound = "DirNotFound"
 	Upgrading   = "Upgrading"
 	UpgradeErr  = "UpgradeErr"
-	PullErr     = "PullErr"
 	Rebuilding  = "Rebuilding"
+	Syncing     = "Syncing"
+	SyncSuccess = "SyncSuccess"
+	SyncErr     = "SyncErr"
 
 	ContainerPrefix = "1Panel-"
 

backend/i18n/lang/en.yaml

@@ -52,6 +52,8 @@ ErrAppBackup: '{{ .name }} application backup failed err {{.err}}'
 ErrImagePull: '{{ .name }} image pull failed err {{.err}}'
 ErrVersionTooLow: 'The current 1Panel version is too low to update the app store, please upgrade the version'
 ErrAppNameExist: 'App name is already exist'
+AppStoreIsSyncing: 'The App Store is syncing, please try again later'
+ErrGetCompose: "Failed to obtain docker-compose.yml file! {{ .detail }}"
 
 #file
 ErrFileCanNotRead: "File can not read"

backend/i18n/lang/zh-Hant.yaml

@@ -52,6 +52,8 @@ ErrAppBackup: '{{ .name }} 應用備份失敗 err {{.err}}'
 ErrImagePull: '{{ .name }} 鏡像拉取失敗 err {{.err}}'
 ErrVersionTooLow: '當前 1Panel 版本過低，無法更新應用商店，請升級版本之後操作'
 ErrAppNameExist: '應用名稱已存在'
+AppStoreIsSyncing: '應用程式商店正在同步中，請稍後再試'
+ErrGetCompose: "docker-compose.yml 檔案取得失敗！{{ .detail }}"
 
 #file
 ErrFileCanNotRead: "此文件不支持預覽"

backend/i18n/lang/zh.yaml

@@ -52,6 +52,8 @@ ErrAppBackup: '{{ .name }} 应用备份失败 err {{.err}}'
 ErrImagePull: '镜像拉取失败 {{.err}}'
 ErrVersionTooLow: '当前 1Panel 版本过低，无法更新应用商店，请升级版本之后操作'
 ErrAppNameExist: '应用名称已存在'
+AppStoreIsSyncing: '应用商店正在同步中，请稍后再试'
+ErrGetCompose: "docker-compose.yml 文件获取失败！{{ .detail }}"
 
 #file
 ErrFileCanNotRead: "此文件不支持预览"

backend/init/business/business.go

@@ -2,6 +2,7 @@ package business
 
 import (
 	"github.com/1Panel-dev/1Panel/backend/app/service"
+	"github.com/1Panel-dev/1Panel/backend/constant"
 	"github.com/1Panel-dev/1Panel/backend/global"
 )
 
@@ -12,6 +13,7 @@ func Init() {
 }
 
 func syncApp() {
+	_ = service.NewISettingService().Update("AppStoreSyncStatus", constant.SyncSuccess)
 	if err := service.NewIAppService().SyncAppListFromRemote(); err != nil {
 		global.LOG.Errorf("App Store synchronization failed")
 		return

backend/init/migration/migrate.go

@@ -51,6 +51,7 @@ func Init() {
 		migrations.AddFavorite,
 		migrations.AddBindAddress,
 		migrations.AddCommandGroup,
+		migrations.AddAppSyncStatus,
 	})
 	if err := m.Migrate(); err != nil {
 		global.LOG.Error(err)

backend/init/migration/migrations/v_1_8.go

@@ -45,3 +45,13 @@ var AddCommandGroup = &gormigrate.Migration{
 		return nil
 	},
 }
+
+var AddAppSyncStatus = &gormigrate.Migration{
+	ID: "20231103-update-table-setting",
+	Migrate: func(tx *gorm.DB) error {
+		if err := tx.Create(&model.Setting{Key: "AppStoreSyncStatus", Value: "SyncSuccess"}).Error; err != nil {
+			return err
+		}
+		return nil
+	},
+}

backend/utils/files/file_op.go

@@ -402,7 +402,10 @@ func getFormat(cType CompressType) archiver.CompressedArchive {
 	case TarGz, Gz:
 		format.Compression = archiver.Gz{}
 		format.Archival = archiver.Tar{}
-	case Zip:
+	case SdkTarGz:
+		format.Compression = archiver.Gz{}
+		format.Archival = archiver.Tar{}
+	case SdkZip:
 		format.Archival = archiver.Zip{
 			Compression: zip.Deflate,
 		}

backend/utils/files/fileinfo.go

@@ -351,10 +351,12 @@ func min(x, y int) int {
 type CompressType string
 
 const (
-	Zip   CompressType = "zip"
+	Zip      CompressType = "zip"
-	Gz    CompressType = "gz"
+	Gz       CompressType = "gz"
-	Bz2   CompressType = "bz2"
+	Bz2      CompressType = "bz2"
-	Tar   CompressType = "tar"
+	Tar      CompressType = "tar"
-	TarGz CompressType = "tar.gz"
+	TarGz    CompressType = "tar.gz"
-	Xz    CompressType = "xz"
+	Xz       CompressType = "xz"
+	SdkZip   CompressType = "sdkZip"
+	SdkTarGz CompressType = "sdkTarGz"
 )