From a77ebed29916eb71307ac9defedcf595c05e4412 Mon Sep 17 00:00:00 2001 From: Ryan Wang Date: Tue, 17 Mar 2020 20:22:51 +0800 Subject: [PATCH] refactor: backup service. (#687) * refactor: backup service. * fix: test case error. --- .../app/config/properties/HaloProperties.java | 7 ++ .../admin/api/BackupController.java | 79 ++++++++++----- .../admin/api/DataProcessController.java | 4 +- .../halo/app/listener/StartedListener.java | 6 ++ .../run/halo/app/model/dto/BackupDTO.java | 3 - .../halo/app/model/entity/PostCategory.java | 2 +- .../run/halo/app/model/support/HaloConst.java | 5 + .../run/halo/app/service/BackupService.java | 27 +++-- .../app/service/impl/BackupServiceImpl.java | 98 ++++++++++++++++--- 9 files changed, 181 insertions(+), 50 deletions(-) diff --git a/src/main/java/run/halo/app/config/properties/HaloProperties.java b/src/main/java/run/halo/app/config/properties/HaloProperties.java index 5769ff205..41c726775 100644 --- a/src/main/java/run/halo/app/config/properties/HaloProperties.java +++ b/src/main/java/run/halo/app/config/properties/HaloProperties.java @@ -14,6 +14,8 @@ import static run.halo.app.utils.HaloUtils.ensureSuffix; * Halo configuration properties. * * @author johnniang + * @author ryanwang + * @date 2019-03-15 */ @Data @ConfigurationProperties("halo") @@ -54,6 +56,11 @@ public class HaloProperties { */ private String backupDir = ensureSuffix(TEMP_DIR, FILE_SEPARATOR) + "halo-backup" + FILE_SEPARATOR; + /** + * Halo data export directory. + */ + private String dataExportDir = ensureSuffix(TEMP_DIR, FILE_SEPARATOR) + "halo-data-export" + FILE_SEPARATOR; + /** * Upload prefix. */ diff --git a/src/main/java/run/halo/app/controller/admin/api/BackupController.java b/src/main/java/run/halo/app/controller/admin/api/BackupController.java index 8f70a88c6..2c4d64ae2 100644 --- a/src/main/java/run/halo/app/controller/admin/api/BackupController.java +++ b/src/main/java/run/halo/app/controller/admin/api/BackupController.java @@ -1,6 +1,5 @@ package run.halo.app.controller.admin.api; -import cn.hutool.core.date.DateUtil; import io.swagger.annotations.ApiOperation; import lombok.extern.slf4j.Slf4j; import org.springframework.core.io.Resource; @@ -9,6 +8,7 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; +import run.halo.app.config.properties.HaloProperties; import run.halo.app.model.annotation.DisableOnCondition; import run.halo.app.model.dto.BackupDTO; import run.halo.app.model.dto.post.BasePostDetailDTO; @@ -16,13 +16,13 @@ import run.halo.app.service.BackupService; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.util.Date; import java.util.List; /** * Backup controller * * @author johnniang + * @author ryanwang * @date 2019-04-26 */ @RestController @@ -32,31 +32,35 @@ public class BackupController { private final BackupService backupService; - public BackupController(BackupService backupService) { + private final HaloProperties haloProperties; + + public BackupController(BackupService backupService, + HaloProperties haloProperties) { this.backupService = backupService; + this.haloProperties = haloProperties; } - @PostMapping("halo") - @ApiOperation("Backups halo") + @PostMapping("work-dir") + @ApiOperation("Backups work directory") @DisableOnCondition public BackupDTO backupHalo() { - return backupService.zipWorkDirectory(); + return backupService.backupWorkDirectory(); } - @GetMapping("halo") - @ApiOperation("Gets all backups") + @GetMapping("work-dir") + @ApiOperation("Gets all work directory backups") public List listBackups() { - return backupService.listHaloBackups(); + return backupService.listWorkDirBackups(); } - @GetMapping("halo/{fileName:.+}") - @ApiOperation("Downloads backup file") + @GetMapping("work-dir/{fileName:.+}") + @ApiOperation("Downloads a work directory backup file") @DisableOnCondition public ResponseEntity downloadBackup(@PathVariable("fileName") String fileName, HttpServletRequest request) { log.info("Try to download backup file: [{}]", fileName); // Load file as resource - Resource backupResource = backupService.loadFileAsResource(fileName); + Resource backupResource = backupService.loadFileAsResource(haloProperties.getBackupDir(), fileName); String contentType = "application/octet-stream"; // Try to determine file's content type @@ -73,30 +77,59 @@ public class BackupController { .body(backupResource); } - @DeleteMapping("halo") - @ApiOperation("Deletes a backup") + @DeleteMapping("work-dir") + @ApiOperation("Deletes a work directory backup") @DisableOnCondition public void deleteBackup(@RequestParam("filename") String filename) { - backupService.deleteHaloBackup(filename); + backupService.deleteWorkDirBackup(filename); } - @PostMapping("import/markdown") - @ApiOperation("Import markdown") + @PostMapping("markdown") + @ApiOperation("Imports markdown") public BasePostDetailDTO backupMarkdowns(@RequestPart("file") MultipartFile file) throws IOException { return backupService.importMarkdown(file); } - @GetMapping("export/data") + @PostMapping("data") + @ApiOperation("Exports all data") @DisableOnCondition - public ResponseEntity exportData() { + public BackupDTO exportData() { + return backupService.exportData(); + } - String contentType = "application/octet-stream;charset=UTF-8"; + @GetMapping("data") + @ApiOperation("Lists all exported data") + public List listExportedData() { + return backupService.listExportedData(); + } - String filename = "halo-data-" + DateUtil.format(new Date(), "yyyy-MM-dd-HH-mm-ss.json"); + @DeleteMapping("data") + @ApiOperation("Deletes a exported data") + @DisableOnCondition + public void deleteExportedData(@RequestParam("filename") String filename) { + backupService.deleteExportedData(filename); + } + + @GetMapping("data/{fileName:.+}") + @ApiOperation("Downloads a exported data") + @DisableOnCondition + public ResponseEntity downloadExportedData(@PathVariable("fileName") String fileName, HttpServletRequest request) { + log.info("Try to download exported data file: [{}]", fileName); + + // Load exported data as resource + Resource exportDataResource = backupService.loadFileAsResource(haloProperties.getDataExportDir(), fileName); + + String contentType = "application/octet-stream"; + // Try to determine file's content type + try { + contentType = request.getServletContext().getMimeType(exportDataResource.getFile().getAbsolutePath()); + } catch (IOException e) { + log.warn("Could not determine file type", e); + } return ResponseEntity.ok() .contentType(MediaType.parseMediaType(contentType)) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"") - .body(backupService.exportData().toJSONString()); + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + exportDataResource.getFilename() + "\"") + .body(exportDataResource); } } diff --git a/src/main/java/run/halo/app/controller/admin/api/DataProcessController.java b/src/main/java/run/halo/app/controller/admin/api/DataProcessController.java index b98f6fee2..adeac68c7 100644 --- a/src/main/java/run/halo/app/controller/admin/api/DataProcessController.java +++ b/src/main/java/run/halo/app/controller/admin/api/DataProcessController.java @@ -1,7 +1,9 @@ package run.halo.app.controller.admin.api; import io.swagger.annotations.ApiOperation; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; import run.halo.app.service.DataProcessService; import run.halo.app.service.ThemeSettingService; diff --git a/src/main/java/run/halo/app/listener/StartedListener.java b/src/main/java/run/halo/app/listener/StartedListener.java index dd1ee60c3..3a3ed2c0c 100644 --- a/src/main/java/run/halo/app/listener/StartedListener.java +++ b/src/main/java/run/halo/app/listener/StartedListener.java @@ -148,6 +148,7 @@ public class StartedListener implements ApplicationListener listHaloBackups(); + List listWorkDirBackups(); /** * Deletes backup. * * @param fileName filename must not be blank */ - void deleteHaloBackup(@NonNull String fileName); + void deleteWorkDirBackup(@NonNull String fileName); /** * Loads file as resource. * * @param fileName backup file name must not be blank. + * @param basePath base path * @return resource of the given file */ @NonNull - Resource loadFileAsResource(@NonNull String fileName); + Resource loadFileAsResource(@NonNull String basePath, @NonNull String fileName); /** @@ -67,7 +68,21 @@ public interface BackupService { * @return data */ @NonNull - JSONObject exportData(); + BackupDTO exportData(); + + /** + * List all exported data. + * + * @return list of backup dto + */ + List listExportedData(); + + /** + * Deletes exported data. + * + * @param fileName fileName + */ + void deleteExportedData(@NonNull String fileName); /** * Import data diff --git a/src/main/java/run/halo/app/service/impl/BackupServiceImpl.java b/src/main/java/run/halo/app/service/impl/BackupServiceImpl.java index e9765ae63..1029a2932 100644 --- a/src/main/java/run/halo/app/service/impl/BackupServiceImpl.java +++ b/src/main/java/run/halo/app/service/impl/BackupServiceImpl.java @@ -2,6 +2,8 @@ package run.halo.app.service.impl; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.IoUtil; +import cn.hutool.core.io.file.FileWriter; +import cn.hutool.core.util.CharsetUtil; import cn.hutool.core.util.IdUtil; import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; @@ -51,7 +53,9 @@ import java.util.stream.Stream; @Slf4j public class BackupServiceImpl implements BackupService { - private static final String BACKUP_RESOURCE_BASE_URI = "/api/admin/backups/halo"; + private static final String BACKUP_RESOURCE_BASE_URI = "/api/admin/backups/work-dir"; + + private static final String DATA_EXPORT_BASE_URI = "/api/admin/backups/data"; private static final String LINE_SEPARATOR = System.getProperty("line.separator"); @@ -155,7 +159,7 @@ public class BackupServiceImpl implements BackupService { } @Override - public BackupDTO zipWorkDirectory() { + public BackupDTO backupWorkDirectory() { // Zip work directory to temporary file try { // Create zip path for halo zip @@ -169,14 +173,14 @@ public class BackupServiceImpl implements BackupService { run.halo.app.utils.FileUtils.zip(Paths.get(this.haloProperties.getWorkDir()), haloZipPath); // Build backup dto - return buildBackupDto(haloZipPath); + return buildBackupDto(BACKUP_RESOURCE_BASE_URI, haloZipPath); } catch (IOException e) { throw new ServiceException("Failed to backup halo", e); } } @Override - public List listHaloBackups() { + public List listWorkDirBackups() { // Ensure the parent folder exist Path backupParentPath = Paths.get(haloProperties.getBackupDir()); if (Files.notExists(backupParentPath)) { @@ -187,7 +191,7 @@ public class BackupServiceImpl implements BackupService { try (Stream subPathStream = Files.list(backupParentPath)) { return subPathStream .filter(backupPath -> StringUtils.startsWithIgnoreCase(backupPath.getFileName().toString(), HaloConst.HALO_BACKUP_PREFIX)) - .map(this::buildBackupDto) + .map(backupPath -> buildBackupDto(BACKUP_RESOURCE_BASE_URI, backupPath)) .sorted((leftBackup, rightBackup) -> { // Sort the result if (leftBackup.getUpdateTime() < rightBackup.getUpdateTime()) { @@ -203,7 +207,7 @@ public class BackupServiceImpl implements BackupService { } @Override - public void deleteHaloBackup(String fileName) { + public void deleteWorkDirBackup(String fileName) { Assert.hasText(fileName, "File name must not be blank"); Path backupRootPath = Paths.get(haloProperties.getBackupDir()); @@ -225,10 +229,11 @@ public class BackupServiceImpl implements BackupService { } @Override - public Resource loadFileAsResource(String fileName) { + public Resource loadFileAsResource(String basePath, String fileName) { + Assert.hasText(basePath, "Base path must not be blank"); Assert.hasText(fileName, "Backup file name must not be blank"); - Path backupParentPath = Paths.get(haloProperties.getBackupDir()); + Path backupParentPath = Paths.get(basePath); try { if (Files.notExists(backupParentPath)) { @@ -237,7 +242,7 @@ public class BackupServiceImpl implements BackupService { } // Get backup file path - Path backupFilePath = Paths.get(haloProperties.getBackupDir(), fileName).normalize(); + Path backupFilePath = Paths.get(basePath, fileName).normalize(); // Check directory traversal run.halo.app.utils.FileUtils.checkDirectoryTraversal(backupParentPath, backupFilePath); @@ -258,7 +263,7 @@ public class BackupServiceImpl implements BackupService { } @Override - public JSONObject exportData() { + public BackupDTO exportData() { JSONObject data = new JSONObject(); data.put("version", HaloConst.HALO_VERSION); data.put("export_date", DateUtil.now()); @@ -283,7 +288,67 @@ public class BackupServiceImpl implements BackupService { data.put("tags", tagService.listAll()); data.put("theme_settings", themeSettingService.listAll()); data.put("user", userService.listAll()); - return data; + + try { + String haloDataFileName = HaloConst.HALO_DATA_EXPORT_PREFIX + + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd-HH-mm-ss-")) + + IdUtil.simpleUUID().hashCode() + ".json"; + + Path haloDataPath = Files.createFile(Paths.get(haloProperties.getDataExportDir(), haloDataFileName)); + + FileWriter fileWriter = new FileWriter(haloDataPath.toFile(), CharsetUtil.UTF_8); + fileWriter.write(data.toJSONString()); + + return buildBackupDto(DATA_EXPORT_BASE_URI, haloDataPath); + } catch (IOException e) { + throw new ServiceException("导出数据失败", e); + } + } + + @Override + public List listExportedData() { + + Path exportedDataParentPath = Paths.get(haloProperties.getDataExportDir()); + if (Files.notExists(exportedDataParentPath)) { + return Collections.emptyList(); + } + + try (Stream subPathStream = Files.list(exportedDataParentPath)) { + return subPathStream + .filter(backupPath -> StringUtils.startsWithIgnoreCase(backupPath.getFileName().toString(), HaloConst.HALO_DATA_EXPORT_PREFIX)) + .map(backupPath -> buildBackupDto(DATA_EXPORT_BASE_URI, backupPath)) + .sorted((leftBackup, rightBackup) -> { + // Sort the result + if (leftBackup.getUpdateTime() < rightBackup.getUpdateTime()) { + return 1; + } else if (leftBackup.getUpdateTime() > rightBackup.getUpdateTime()) { + return -1; + } + return 0; + }).collect(Collectors.toList()); + } catch (IOException e) { + throw new ServiceException("Failed to fetch exported data", e); + } + } + + @Override + public void deleteExportedData(String fileName) { + Assert.hasText(fileName, "File name must not be blank"); + + Path dataExportRootPath = Paths.get(haloProperties.getDataExportDir()); + + Path backupPath = dataExportRootPath.resolve(fileName); + + run.halo.app.utils.FileUtils.checkDirectoryTraversal(dataExportRootPath, backupPath); + + try { + // Delete backup file + Files.delete(backupPath); + } catch (NoSuchFileException e) { + throw new NotFoundException("The file " + fileName + " was not found", e); + } catch (IOException e) { + throw new ServiceException("Failed to delete backup", e); + } } @Override @@ -363,14 +428,14 @@ public class BackupServiceImpl implements BackupService { * @param backupPath backup path must not be null * @return backup dto */ - private BackupDTO buildBackupDto(@NonNull Path backupPath) { + private BackupDTO buildBackupDto(@NonNull String basePath, @NonNull Path backupPath) { + Assert.notNull(basePath, "Base path must not be null"); Assert.notNull(backupPath, "Backup path must not be null"); String backupFileName = backupPath.getFileName().toString(); BackupDTO backup = new BackupDTO(); try { - backup.setDownloadUrl(buildDownloadUrl(backupFileName)); - backup.setDownloadLink(backup.getDownloadUrl()); + backup.setDownloadLink(buildDownloadUrl(basePath, backupFileName)); backup.setFilename(backupFileName); backup.setUpdateTime(Files.getLastModifiedTime(backupPath).toMillis()); backup.setFileSize(Files.size(backupPath)); @@ -388,11 +453,12 @@ public class BackupServiceImpl implements BackupService { * @return download url */ @NonNull - private String buildDownloadUrl(@NonNull String filename) { + private String buildDownloadUrl(@NonNull String basePath, @NonNull String filename) { + Assert.notNull(basePath, "Base path must not be null"); Assert.hasText(filename, "File name must not be blank"); // Composite http url - String backupUri = BACKUP_RESOURCE_BASE_URI + HaloUtils.URL_SEPARATOR + filename; + String backupUri = basePath + HaloUtils.URL_SEPARATOR + filename; // Get a one-time token String oneTimeToken = oneTimeTokenService.create(backupUri);