change to s3

pull/882/head
chanhengseang 2025-05-25 11:49:56 -07:00
parent 81604701e7
commit a241ae038d
10 changed files with 135 additions and 134 deletions

View File

@ -64,7 +64,7 @@ task:
queue-capacity: 50
# Qiniu Cloud
qiniu:
s3:
# File size /M
max-size: 15

View File

@ -22,14 +22,14 @@ import javax.validation.constraints.NotBlank;
import java.io.Serializable;
/**
* Qiniu Cloud Object Storage Configuration Class
* S3 Cloud Object Storage Configuration Class
* @author Zheng Jie
* @date 2018-12-31
*/
@Data
@Entity
@Table(name = "tool_qiniu_config")
public class QiniuConfig implements Serializable {
@Table(name = "tool_s3_config")
public class S3Config implements Serializable {
@Id
@Column(name = "config_id")
@ -61,7 +61,7 @@ public class QiniuConfig implements Serializable {
private String zone;
@NotBlank
@ApiModelProperty(value = "External domain, customizable, needs to be bound in Qiniu Cloud")
@ApiModelProperty(value = "External domain, customizable, needs to be bound in S3 Cloud")
private String host;
@ApiModelProperty(value = "Space type: public/private")

View File

@ -29,8 +29,8 @@ import java.sql.Timestamp;
*/
@Data
@Entity
@Table(name = "tool_qiniu_content")
public class QiniuContent implements Serializable {
@Table(name = "tool_s3_content")
public class S3Content implements Serializable {
@Id
@Column(name = "content_id")

View File

@ -15,8 +15,9 @@
*/
package me.zhengjie.repository;
import me.zhengjie.domain.QiniuConfig;
import me.zhengjie.domain.S3Config;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
@ -24,13 +25,13 @@ import org.springframework.data.jpa.repository.Query;
* @author Zheng Jie
* @date 2018-12-31
*/
public interface QiNiuConfigRepository extends JpaRepository<QiniuConfig,Long> {
public interface S3ConfigRepository extends JpaRepository<S3Config,Long>, JpaSpecificationExecutor<S3Config> {
/**
* Edit type
* @param type /
*/
@Modifying
@Query(value = "update QiniuConfig set type = ?1")
@Query(value = "update S3Config set type = ?1")
void update(String type);
}

View File

@ -15,7 +15,7 @@
*/
package me.zhengjie.repository;
import me.zhengjie.domain.QiniuContent;
import me.zhengjie.domain.S3Content;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
@ -23,12 +23,12 @@ import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
* @author Zheng Jie
* @date 2018-12-31
*/
public interface QiniuContentRepository extends JpaRepository<QiniuContent,Long>, JpaSpecificationExecutor<QiniuContent> {
public interface S3ContentRepository extends JpaRepository<S3Content,Long>, JpaSpecificationExecutor<S3Content> {
/**
* Query by key
* @param key file name
* @return QiniuContent
* @return S3Content
*/
QiniuContent findByKey(String key);
S3Content findByKey(String key);
}

View File

@ -20,10 +20,10 @@ import io.swagger.annotations.ApiOperation;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import me.zhengjie.annotation.Log;
import me.zhengjie.domain.QiniuConfig;
import me.zhengjie.domain.QiniuContent;
import me.zhengjie.service.dto.QiniuQueryCriteria;
import me.zhengjie.service.QiNiuService;
import me.zhengjie.domain.S3Config;
import me.zhengjie.domain.S3Content;
import me.zhengjie.service.dto.S3QueryCriteria;
import me.zhengjie.service.S3Service;
import me.zhengjie.utils.PageResult;
import org.springframework.data.domain.Pageable;
import org.springframework.http.HttpStatus;
@ -44,79 +44,79 @@ import java.util.Map;
@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping("/api/qiNiuContent")
@Api(tags = "Tools: Qiniu Cloud Storage Management")
public class QiniuController {
@RequestMapping("/api/s3Content")
@Api(tags = "Tools: S3 Cloud Storage Management")
public class S3Controller {
private final QiNiuService qiNiuService;
private final S3Service s3Service;
@GetMapping(value = "/config")
public ResponseEntity<QiniuConfig> queryQiNiuConfig(){
return new ResponseEntity<>(qiNiuService.find(), HttpStatus.OK);
public ResponseEntity<S3Config> queryS3Config(){
return new ResponseEntity<>(s3Service.find(), HttpStatus.OK);
}
@Log("Configure Qiniu Cloud Storage")
@ApiOperation("Configure Qiniu Cloud Storage")
@Log("Configure S3 Cloud Storage")
@ApiOperation("Configure S3 Cloud Storage")
@PutMapping(value = "/config")
public ResponseEntity<Object> updateQiNiuConfig(@Validated @RequestBody QiniuConfig qiniuConfig){
qiNiuService.config(qiniuConfig);
qiNiuService.update(qiniuConfig.getType());
public ResponseEntity<Object> updateS3Config(@Validated @RequestBody S3Config s3Config){
s3Service.config(s3Config);
s3Service.update(s3Config.getType());
return new ResponseEntity<>(HttpStatus.OK);
}
@ApiOperation("Export Data")
@GetMapping(value = "/download")
public void exportQiNiu(HttpServletResponse response, QiniuQueryCriteria criteria) throws IOException {
qiNiuService.downloadList(qiNiuService.queryAll(criteria), response);
public void exportS3(HttpServletResponse response, S3QueryCriteria criteria) throws IOException {
s3Service.downloadList(s3Service.queryAll(criteria), response);
}
@ApiOperation("Query File")
@GetMapping
public ResponseEntity<PageResult<QiniuContent>> queryQiNiu(QiniuQueryCriteria criteria, Pageable pageable){
return new ResponseEntity<>(qiNiuService.queryAll(criteria,pageable),HttpStatus.OK);
public ResponseEntity<PageResult<S3Content>> queryS3(S3QueryCriteria criteria, Pageable pageable){
return new ResponseEntity<>(s3Service.queryAll(criteria,pageable),HttpStatus.OK);
}
@ApiOperation("Upload File")
@PostMapping
public ResponseEntity<Object> uploadQiNiu(@RequestParam MultipartFile file){
QiniuContent qiniuContent = qiNiuService.upload(file,qiNiuService.find());
public ResponseEntity<Object> uploadS3(@RequestParam MultipartFile file){
S3Content s3Content = s3Service.upload(file,s3Service.find());
Map<String,Object> map = new HashMap<>(3);
map.put("id",qiniuContent.getId());
map.put("id",s3Content.getId());
map.put("errno",0);
map.put("data",new String[]{qiniuContent.getUrl()});
map.put("data",new String[]{s3Content.getUrl()});
return new ResponseEntity<>(map,HttpStatus.OK);
}
@Log("Synchronize Qiniu Cloud Data")
@ApiOperation("Synchronize Qiniu Cloud Data")
@Log("Synchronize S3 Cloud Data")
@ApiOperation("Synchronize S3 Cloud Data")
@PostMapping(value = "/synchronize")
public ResponseEntity<Object> synchronizeQiNiu(){
qiNiuService.synchronize(qiNiuService.find());
public ResponseEntity<Object> synchronizeS3(){
s3Service.synchronize(s3Service.find());
return new ResponseEntity<>(HttpStatus.OK);
}
@Log("Download File")
@ApiOperation("Download File")
@GetMapping(value = "/download/{id}")
public ResponseEntity<Object> downloadQiNiu(@PathVariable Long id){
public ResponseEntity<Object> downloadS3(@PathVariable Long id){
Map<String,Object> map = new HashMap<>(1);
map.put("url", qiNiuService.download(qiNiuService.findByContentId(id),qiNiuService.find()));
map.put("url", s3Service.download(s3Service.findByContentId(id),s3Service.find()));
return new ResponseEntity<>(map,HttpStatus.OK);
}
@Log("Delete File")
@ApiOperation("Delete File")
@DeleteMapping(value = "/{id}")
public ResponseEntity<Object> deleteQiNiu(@PathVariable Long id){
qiNiuService.delete(qiNiuService.findByContentId(id),qiNiuService.find());
public ResponseEntity<Object> deleteS3(@PathVariable Long id){
s3Service.delete(s3Service.findByContentId(id),s3Service.find());
return new ResponseEntity<>(HttpStatus.OK);
}
@Log("Delete Multiple Images")
@ApiOperation("Delete Multiple Images")
@DeleteMapping
public ResponseEntity<Object> deleteAllQiNiu(@RequestBody Long[] ids) {
qiNiuService.deleteAll(ids, qiNiuService.find());
public ResponseEntity<Object> deleteAllS3(@RequestBody Long[] ids) {
s3Service.deleteAll(ids, s3Service.find());
return new ResponseEntity<>(HttpStatus.OK);
}
}

View File

@ -15,9 +15,9 @@
*/
package me.zhengjie.service;
import me.zhengjie.domain.QiniuConfig;
import me.zhengjie.domain.QiniuContent;
import me.zhengjie.service.dto.QiniuQueryCriteria;
import me.zhengjie.domain.S3Content;
import me.zhengjie.domain.S3Config;
import me.zhengjie.service.dto.S3QueryCriteria;
import me.zhengjie.utils.PageResult;
import org.springframework.data.domain.Pageable;
import org.springframework.web.multipart.MultipartFile;
@ -30,20 +30,20 @@ import java.util.List;
* @author Zheng Jie
* @date 2018-12-31
*/
public interface QiNiuService {
public interface S3Service {
/**
* Query configuration
* @return QiniuConfig
* @return S3Config
*/
QiniuConfig find();
S3Config find();
/**
* Update configuration
* @param qiniuConfig configuration
* @return QiniuConfig
* @param s3Config configuration
* @return S3Config
*/
QiniuConfig config(QiniuConfig qiniuConfig);
S3Config config(S3Config s3Config);
/**
* Paginated query
@ -51,29 +51,29 @@ public interface QiNiuService {
* @param pageable pagination parameters
* @return /
*/
PageResult<QiniuContent> queryAll(QiniuQueryCriteria criteria, Pageable pageable);
PageResult<S3Content> queryAll(S3QueryCriteria criteria, Pageable pageable);
/**
* Query all
* @param criteria criteria
* @return /
*/
List<QiniuContent> queryAll(QiniuQueryCriteria criteria);
List<S3Content> queryAll(S3QueryCriteria criteria);
/**
* Upload file
* @param file file
* @param qiniuConfig configuration
* @return QiniuContent
* @param s3Config configuration
* @return S3Content
*/
QiniuContent upload(MultipartFile file, QiniuConfig qiniuConfig);
S3Content upload(MultipartFile file, S3Config s3Config);
/**
* Query file
* @param id file ID
* @return QiniuContent
* @return S3Content
*/
QiniuContent findByContentId(Long id);
S3Content findByContentId(Long id);
/**
* Download file
@ -81,27 +81,27 @@ public interface QiNiuService {
* @param config configuration
* @return String
*/
String download(QiniuContent content, QiniuConfig config);
String download(S3Content content, S3Config config);
/**
* Delete file
* @param content file
* @param config configuration
*/
void delete(QiniuContent content, QiniuConfig config);
void delete(S3Content content, S3Config config);
/**
* Sync data
* @param config configuration
*/
void synchronize(QiniuConfig config);
void synchronize(S3Config config);
/**
* Delete file
* @param ids file ID array
* @param config configuration
*/
void deleteAll(Long[] ids, QiniuConfig config);
void deleteAll(Long[] ids, S3Config config);
/**
* Update data
@ -115,5 +115,5 @@ public interface QiNiuService {
* @param response /
* @throws IOException /
*/
void downloadList(List<QiniuContent> queryAll, HttpServletResponse response) throws IOException;
void downloadList(List<S3Content> queryAll, HttpServletResponse response) throws IOException;
}

View File

@ -27,7 +27,7 @@ import java.util.List;
* @date 2019-6-4 09:54:37
*/
@Data
public class QiniuQueryCriteria{
public class S3QueryCriteria{
@ApiModelProperty(value = "Name search")
@Query(type = Query.Type.INNER_LIKE)

View File

@ -25,14 +25,14 @@ import com.qiniu.storage.model.DefaultPutRet;
import com.qiniu.storage.model.FileInfo;
import com.qiniu.util.Auth;
import lombok.RequiredArgsConstructor;
import me.zhengjie.domain.QiniuConfig;
import me.zhengjie.domain.QiniuContent;
import me.zhengjie.repository.QiniuContentRepository;
import me.zhengjie.service.dto.QiniuQueryCriteria;
import me.zhengjie.domain.S3Content;
import me.zhengjie.domain.S3Config;
import me.zhengjie.repository.S3ConfigRepository;
import me.zhengjie.repository.S3ContentRepository;
import me.zhengjie.service.dto.S3QueryCriteria;
import me.zhengjie.utils.*;
import me.zhengjie.exception.BadRequestException;
import me.zhengjie.repository.QiNiuConfigRepository;
import me.zhengjie.service.QiNiuService;
import me.zhengjie.service.S3Service;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CachePut;
@ -51,75 +51,75 @@ import java.util.*;
*/
@Service
@RequiredArgsConstructor
@CacheConfig(cacheNames = "qiNiu")
public class QiNiuServiceImpl implements QiNiuService {
@CacheConfig(cacheNames = "s3")
public class S3ServiceImpl implements S3Service {
private final QiNiuConfigRepository qiNiuConfigRepository;
private final QiniuContentRepository qiniuContentRepository;
private final S3ConfigRepository s3ConfigRepository;
private final S3ContentRepository s3ContentRepository;
@Value("${qiniu.max-size}")
@Value("${s3.max-size}")
private Long maxSize;
@Override
@Cacheable(key = "'config'")
public QiniuConfig find() {
Optional<QiniuConfig> qiniuConfig = qiNiuConfigRepository.findById(1L);
return qiniuConfig.orElseGet(QiniuConfig::new);
public S3Config find() {
Optional<S3Config> s3Config = s3ConfigRepository.findById(1L);
return s3Config.orElseGet(S3Config::new);
}
@Override
@CachePut(key = "'config'")
@Transactional(rollbackFor = Exception.class)
public QiniuConfig config(QiniuConfig qiniuConfig) {
qiniuConfig.setId(1L);
public S3Config config(S3Config s3Config) {
s3Config.setId(1L);
String http = "http://", https = "https://";
if (!(qiniuConfig.getHost().toLowerCase().startsWith(http)||qiniuConfig.getHost().toLowerCase().startsWith(https))) {
if (!(s3Config.getHost().toLowerCase().startsWith(http)||s3Config.getHost().toLowerCase().startsWith(https))) {
throw new BadRequestException("External link domain must start with http:// or https://");
}
return qiNiuConfigRepository.save(qiniuConfig);
return s3ConfigRepository.save(s3Config);
}
@Override
public PageResult<QiniuContent> queryAll(QiniuQueryCriteria criteria, Pageable pageable){
return PageUtil.toPage(qiniuContentRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root,criteria,criteriaBuilder),pageable));
public PageResult<S3Content> queryAll(S3QueryCriteria criteria, Pageable pageable){
return PageUtil.toPage(s3ContentRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root,criteria,criteriaBuilder),pageable));
}
@Override
public List<QiniuContent> queryAll(QiniuQueryCriteria criteria) {
return qiniuContentRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root,criteria,criteriaBuilder));
public List<S3Content> queryAll(S3QueryCriteria criteria) {
return s3ContentRepository.findAll((root, criteriaQuery, criteriaBuilder) -> QueryHelp.getPredicate(root,criteria,criteriaBuilder));
}
@Override
@Transactional(rollbackFor = Exception.class)
public QiniuContent upload(MultipartFile file, QiniuConfig qiniuConfig) {
public S3Content upload(MultipartFile file, S3Config s3Config) {
FileUtil.checkSize(maxSize, file.getSize());
if(qiniuConfig.getId() == null){
if(s3Config.getId() == null){
throw new BadRequestException("Please add the corresponding configuration first, then operate");
}
// Construct a configuration class with the specified Zone object
Configuration cfg = new Configuration(QiNiuUtil.getRegion(qiniuConfig.getZone()));
Configuration cfg = new Configuration(S3Util.getRegion(s3Config.getZone()));
UploadManager uploadManager = new UploadManager(cfg);
Auth auth = Auth.create(qiniuConfig.getAccessKey(), qiniuConfig.getSecretKey());
String upToken = auth.uploadToken(qiniuConfig.getBucket());
Auth auth = Auth.create(s3Config.getAccessKey(), s3Config.getSecretKey());
String upToken = auth.uploadToken(s3Config.getBucket());
try {
String key = file.getOriginalFilename();
if(qiniuContentRepository.findByKey(key) != null) {
key = QiNiuUtil.getKey(key);
if(s3ContentRepository.findByKey(key) != null) {
key = S3Util.getKey(key);
}
Response response = uploadManager.put(file.getBytes(), key, upToken);
// Parse the result of successful upload
DefaultPutRet putRet = JSON.parseObject(response.bodyString(), DefaultPutRet.class);
QiniuContent content = qiniuContentRepository.findByKey(FileUtil.getFileNameNoEx(putRet.key));
S3Content content = s3ContentRepository.findByKey(FileUtil.getFileNameNoEx(putRet.key));
if(content == null){
// Store in database
QiniuContent qiniuContent = new QiniuContent();
qiniuContent.setSuffix(FileUtil.getExtensionName(putRet.key));
qiniuContent.setBucket(qiniuConfig.getBucket());
qiniuContent.setType(qiniuConfig.getType());
qiniuContent.setKey(FileUtil.getFileNameNoEx(putRet.key));
qiniuContent.setUrl(qiniuConfig.getHost()+"/"+putRet.key);
qiniuContent.setSize(FileUtil.getSize(Integer.parseInt(String.valueOf(file.getSize()))));
return qiniuContentRepository.save(qiniuContent);
S3Content s3Content = new S3Content();
s3Content.setSuffix(FileUtil.getExtensionName(putRet.key));
s3Content.setBucket(s3Config.getBucket());
s3Content.setType(s3Config.getType());
s3Content.setKey(FileUtil.getFileNameNoEx(putRet.key));
s3Content.setUrl(s3Config.getHost()+"/"+putRet.key);
s3Content.setSize(FileUtil.getSize(Integer.parseInt(String.valueOf(file.getSize()))));
return s3ContentRepository.save(s3Content);
}
return content;
} catch (Exception e) {
@ -128,14 +128,14 @@ public class QiNiuServiceImpl implements QiNiuService {
}
@Override
public QiniuContent findByContentId(Long id) {
QiniuContent qiniuContent = qiniuContentRepository.findById(id).orElseGet(QiniuContent::new);
ValidationUtil.isNull(qiniuContent.getId(),"QiniuContent", "id",id);
return qiniuContent;
public S3Content findByContentId(Long id) {
S3Content s3Content = s3ContentRepository.findById(id).orElseGet(S3Content::new);
ValidationUtil.isNull(s3Content.getId(),"S3Content", "id",id);
return s3Content;
}
@Override
public String download(QiniuContent content,QiniuConfig config){
public String download(S3Content content,S3Config config){
String finalUrl;
String type = "Public";
if(type.equals(content.getType())){
@ -151,27 +151,27 @@ public class QiNiuServiceImpl implements QiNiuService {
@Override
@Transactional(rollbackFor = Exception.class)
public void delete(QiniuContent content, QiniuConfig config) {
public void delete(S3Content content, S3Config config) {
// Construct a configuration class with the specified Zone object
Configuration cfg = new Configuration(QiNiuUtil.getRegion(config.getZone()));
Configuration cfg = new Configuration(S3Util.getRegion(config.getZone()));
Auth auth = Auth.create(config.getAccessKey(), config.getSecretKey());
BucketManager bucketManager = new BucketManager(auth, cfg);
try {
bucketManager.delete(content.getBucket(), content.getKey() + "." + content.getSuffix());
qiniuContentRepository.delete(content);
s3ContentRepository.delete(content);
} catch (QiniuException ex) {
qiniuContentRepository.delete(content);
s3ContentRepository.delete(content);
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public void synchronize(QiniuConfig config) {
public void synchronize(S3Config config) {
if(config.getId() == null){
throw new BadRequestException("Please add the corresponding configuration first, then operate");
}
// Construct a configuration class with the specified Zone object
Configuration cfg = new Configuration(QiNiuUtil.getRegion(config.getZone()));
Configuration cfg = new Configuration(S3Util.getRegion(config.getZone()));
Auth auth = Auth.create(config.getAccessKey(), config.getSecretKey());
BucketManager bucketManager = new BucketManager(auth, cfg);
// File name prefix
@ -184,18 +184,18 @@ public class QiNiuServiceImpl implements QiNiuService {
BucketManager.FileListIterator fileListIterator = bucketManager.createFileListIterator(config.getBucket(), prefix, limit, delimiter);
while (fileListIterator.hasNext()) {
// Process the obtained file list result
QiniuContent qiniuContent;
S3Content s3Content;
FileInfo[] items = fileListIterator.next();
for (FileInfo item : items) {
if(qiniuContentRepository.findByKey(FileUtil.getFileNameNoEx(item.key)) == null){
qiniuContent = new QiniuContent();
qiniuContent.setSize(FileUtil.getSize(Integer.parseInt(String.valueOf(item.fsize))));
qiniuContent.setSuffix(FileUtil.getExtensionName(item.key));
qiniuContent.setKey(FileUtil.getFileNameNoEx(item.key));
qiniuContent.setType(config.getType());
qiniuContent.setBucket(config.getBucket());
qiniuContent.setUrl(config.getHost()+"/"+item.key);
qiniuContentRepository.save(qiniuContent);
if(s3ContentRepository.findByKey(FileUtil.getFileNameNoEx(item.key)) == null){
s3Content = new S3Content();
s3Content.setSize(FileUtil.getSize(Integer.parseInt(String.valueOf(item.fsize))));
s3Content.setSuffix(FileUtil.getExtensionName(item.key));
s3Content.setKey(FileUtil.getFileNameNoEx(item.key));
s3Content.setType(config.getType());
s3Content.setBucket(config.getBucket());
s3Content.setUrl(config.getHost()+"/"+item.key);
s3ContentRepository.save(s3Content);
}
}
}
@ -203,7 +203,7 @@ public class QiNiuServiceImpl implements QiNiuService {
@Override
@Transactional(rollbackFor = Exception.class)
public void deleteAll(Long[] ids, QiniuConfig config) {
public void deleteAll(Long[] ids, S3Config config) {
for (Long id : ids) {
delete(findByContentId(id), config);
}
@ -212,13 +212,13 @@ public class QiNiuServiceImpl implements QiNiuService {
@Override
@Transactional(rollbackFor = Exception.class)
public void update(String type) {
qiNiuConfigRepository.update(type);
s3ConfigRepository.update(type);
}
@Override
public void downloadList(List<QiniuContent> queryAll, HttpServletResponse response) throws IOException {
public void downloadList(List<S3Content> queryAll, HttpServletResponse response) throws IOException {
List<Map<String, Object>> list = new ArrayList<>();
for (QiniuContent content : queryAll) {
for (S3Content content : queryAll) {
Map<String,Object> map = new LinkedHashMap<>();
map.put("File name", content.getKey());
map.put("File type", content.getSuffix());

View File

@ -20,11 +20,11 @@ import java.text.SimpleDateFormat;
import java.util.Date;
/**
* Qiniu cloud storage utility class
* S3 cloud storage utility class
* @author Zheng Jie
* @date 2018-12-31
*/
public class QiNiuUtil {
public class S3Util {
private static final String HUAD = "East China";