Add support for restoring from backup root

Signed-off-by: JohnNiang <johnniang@foxmail.com>

api-docs/openapi/v3_0/aggregated.json

@@ -5239,55 +5239,6 @@
         ]
       }
     },
-    "/apis/api.console.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
-      "get": {
-        "operationId": "DownloadBackups",
-        "parameters": [
-          {
-            "description": "Backup name.",
-            "in": "path",
-            "name": "name",
-            "required": true,
-            "schema": {
-              "type": "string"
-            }
-          },
-          {
-            "description": "Backup filename.",
-            "in": "path",
-            "name": "filename",
-            "required": true,
-            "schema": {
-              "type": "string"
-            }
-          }
-        ],
-        "responses": {},
-        "tags": [
-          "MigrationV1alpha1Console"
-        ]
-      }
-    },
-    "/apis/api.console.migration.halo.run/v1alpha1/restorations": {
-      "post": {
-        "description": "Restore backup by uploading file or providing download link or backup name.",
-        "operationId": "RestoreBackup",
-        "requestBody": {
-          "content": {
-            "multipart/form-data": {
-              "schema": {
-                "$ref": "#/components/schemas/RestoreRequest"
-              }
-            }
-          },
-          "required": true
-        },
-        "responses": {},
-        "tags": [
-          "MigrationV1alpha1Console"
-        ]
-      }
-    },
     "/apis/api.content.halo.run/v1alpha1/categories": {
       "get": {
         "description": "Lists categories.",
@@ -7411,6 +7362,79 @@
         ]
       }
     },
+    "/apis/console.api.migration.halo.run/v1alpha1/backup-files": {
+      "get": {
+        "description": "Get backup files from backup root.",
+        "operationId": "getBackupFiles",
+        "responses": {
+          "default": {
+            "content": {
+              "*/*": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/BackupFile"
+                  }
+                }
+              }
+            },
+            "description": "default response"
+          }
+        },
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
+    "/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
+      "get": {
+        "operationId": "DownloadBackups",
+        "parameters": [
+          {
+            "description": "Backup name.",
+            "in": "path",
+            "name": "name",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "description": "Backup filename.",
+            "in": "path",
+            "name": "filename",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {},
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
+    "/apis/console.api.migration.halo.run/v1alpha1/restorations": {
+      "post": {
+        "description": "Restore backup by uploading file or providing download link or backup name.",
+        "operationId": "RestoreBackup",
+        "requestBody": {
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "$ref": "#/components/schemas/RestoreRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {},
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
     "/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": {
       "post": {
         "description": "Verify email sender config.",
@@ -15300,6 +15324,22 @@
           }
         }
       },
+      "BackupFile": {
+        "type": "object",
+        "properties": {
+          "filename": {
+            "type": "string"
+          },
+          "lastModifiedTime": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "size": {
+            "type": "integer",
+            "format": "int64"
+          }
+        }
+      },
       "BackupList": {
         "required": [
           "first",
@@ -20684,6 +20724,10 @@
           "file": {
             "type": "string",
             "format": "binary"
+          },
+          "filename": {
+            "type": "string",
+            "description": "Filename of backup file in backups root."
           }
         }
       },

api-docs/openapi/v3_0/apis_console.api_v1alpha1.json

@@ -3106,6 +3106,79 @@
         ]
       }
     },
+    "/apis/console.api.migration.halo.run/v1alpha1/backup-files": {
+      "get": {
+        "description": "Get backup files from backup root.",
+        "operationId": "getBackupFiles",
+        "responses": {
+          "default": {
+            "content": {
+              "*/*": {
+                "schema": {
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/components/schemas/BackupFile"
+                  }
+                }
+              }
+            },
+            "description": "default response"
+          }
+        },
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
+    "/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
+      "get": {
+        "operationId": "DownloadBackups",
+        "parameters": [
+          {
+            "description": "Backup name.",
+            "in": "path",
+            "name": "name",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "description": "Backup filename.",
+            "in": "path",
+            "name": "filename",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          }
+        ],
+        "responses": {},
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
+    "/apis/console.api.migration.halo.run/v1alpha1/restorations": {
+      "post": {
+        "description": "Restore backup by uploading file or providing download link or backup name.",
+        "operationId": "RestoreBackup",
+        "requestBody": {
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "$ref": "#/components/schemas/RestoreRequest"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {},
+        "tags": [
+          "MigrationV1alpha1Console"
+        ]
+      }
+    },
     "/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": {
       "post": {
         "description": "Verify email sender config.",
@@ -3376,6 +3449,22 @@
           }
         }
       },
+      "BackupFile": {
+        "type": "object",
+        "properties": {
+          "filename": {
+            "type": "string"
+          },
+          "lastModifiedTime": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "size": {
+            "type": "integer",
+            "format": "int64"
+          }
+        }
+      },
       "Category": {
         "required": [
           "apiVersion",
@@ -5405,6 +5494,27 @@
           }
         }
       },
+      "RestoreRequest": {
+        "type": "object",
+        "properties": {
+          "backupName": {
+            "type": "string",
+            "description": "Backup metadata name."
+          },
+          "downloadUrl": {
+            "type": "string",
+            "description": "Remote backup HTTP URL."
+          },
+          "file": {
+            "type": "string",
+            "format": "binary"
+          },
+          "filename": {
+            "type": "string",
+            "description": "Filename of backup file in backups root."
+          }
+        }
+      },
       "RevertSnapshotForPostParam": {
         "required": [
           "snapshotName"

application/src/main/java/run/halo/app/migration/BackupFile.java

@@ -0,0 +1,34 @@
+package run.halo.app.migration;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import java.nio.file.Path;
+import java.time.Instant;
+import lombok.Data;
+
+/**
+ * Backup file.
+ *
+ * @author johnniang
+ */
+@Data
+public class BackupFile {
+
+    @JsonIgnore
+    private Path path;
+
+    /**
+     * Filename of backup file.
+     */
+    private String filename;
+
+    /**
+     * Size of backup file.
+     */
+    private long size;
+
+    /**
+     * Last modified time of backup file.
+     */
+    private Instant lastModifiedTime;
+
+}

application/src/main/java/run/halo/app/migration/MigrationEndpoint.java

@@ -1,6 +1,7 @@
 package run.halo.app.migration;
 
 import static io.swagger.v3.oas.annotations.media.Schema.RequiredMode.NOT_REQUIRED;
+import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder;
 import static org.springdoc.core.fn.builders.content.Builder.contentBuilder;
 import static org.springdoc.core.fn.builders.parameter.Builder.parameterBuilder;
 import static org.springdoc.core.fn.builders.requestbody.Builder.requestBodyBuilder;
@@ -12,10 +13,12 @@ import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.Optional;
+import java.util.function.Supplier;
 import org.springdoc.webflux.core.fn.SpringdocRouteBuilder;
 import org.springframework.core.io.buffer.DataBuffer;
 import org.springframework.core.io.buffer.DataBufferUtils;
 import org.springframework.core.io.buffer.DefaultDataBufferFactory;
+import org.springframework.data.util.Optionals;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.codec.multipart.FilePart;
@@ -24,7 +27,9 @@ import org.springframework.http.codec.multipart.Part;
 import org.springframework.stereotype.Component;
 import org.springframework.util.MultiValueMap;
 import org.springframework.util.StreamUtils;
+import org.springframework.util.StringUtils;
 import org.springframework.web.reactive.function.server.RouterFunction;
+import org.springframework.web.reactive.function.server.ServerRequest;
 import org.springframework.web.reactive.function.server.ServerResponse;
 import org.springframework.web.server.ServerWebInputException;
 import reactor.core.publisher.Flux;
@@ -55,6 +60,15 @@ public class MigrationEndpoint implements CustomEndpoint {
     public RouterFunction<ServerResponse> endpoint() {
         var tag = "MigrationV1alpha1Console";
         return SpringdocRouteBuilder.route()
+            .GET("/backup-files",
+                this::getBackups,
+                builder -> builder.operationId("getBackupFiles")
+                    .tag(tag)
+                    .description("Get backup files from backup root.")
+                    .response(responseBuilder()
+                        .implementationArray(BackupFile.class)
+                    )
+            )
             .GET("/backups/{name}/files/{filename}",
                 request -> {
                     var name = request.pathVariable("name");
@@ -86,7 +100,7 @@ public class MigrationEndpoint implements CustomEndpoint {
                         var content = getContent(restoreRequest)
                             .switchIfEmpty(Mono.error(() -> new ServerWebInputException(
                                 "Please upload a file "
-                                + "or provide a download link or backup name.")));
+                                    + "or provide a download link or backup name.")));
                         return migrationService.restore(content);
                     })
                     .then(Mono.defer(
@@ -95,7 +109,7 @@ public class MigrationEndpoint implements CustomEndpoint {
                 builder -> builder
                     .tag(tag)
                     .description("Restore backup by uploading file "
-                                 + "or providing download link or backup name.")
+                        + "or providing download link or backup name.")
                     .operationId("RestoreBackup")
                     .requestBody(requestBodyBuilder()
                         .required(true)
@@ -108,8 +122,22 @@ public class MigrationEndpoint implements CustomEndpoint {
             .build();
     }
 
+    private Mono<ServerResponse> getBackups(ServerRequest request) {
+        var backupFiles = migrationService.getBackupFiles();
+        return ServerResponse.ok().body(backupFiles, BackupFile.class);
+    }
+
     private Flux<DataBuffer> getContent(RestoreRequest request) {
-        var downloadContent = request.getDownloadUrl()
+        Supplier<Optional<Flux<DataBuffer>>> contentFromFilename = () ->
+            request.getFilename().map(filename -> migrationService.getBackupFile(filename)
+                .map(BackupFile::getPath)
+                .flatMapMany(
+                    path -> DataBufferUtils.read(
+                        path,
+                        DefaultDataBufferFactory.sharedInstance,
+                        StreamUtils.BUFFER_SIZE)));
+
+        Supplier<Optional<Flux<DataBuffer>>> contentFromDownloadUrl = () -> request.getDownloadUrl()
             .map(downloadURL -> {
                 try {
                     var url = new URL(downloadURL);
@@ -121,23 +149,27 @@ public class MigrationEndpoint implements CustomEndpoint {
                     // Should never happen
                     return Flux.<DataBuffer>error(e);
                 }
-            })
+            });
-            .orElseGet(Flux::empty);
 
-        var uploadContent = request.getFile()
+        Supplier<Optional<Flux<DataBuffer>>> contentFromUpload = () -> request.getFile()
-            .map(Part::content)
+            .map(Part::content);
-            .orElseGet(Flux::empty);
 
-        var backupFileContent = request.getBackupName()
+        Supplier<Optional<Flux<DataBuffer>>> contentFromBackupName = () -> request.getBackupName()
             .map(backupName -> client.get(Backup.class, backupName)
                 .flatMap(migrationService::download)
                 .flatMapMany(resource -> DataBufferUtils.read(resource,
                     DefaultDataBufferFactory.sharedInstance,
-                    StreamUtils.BUFFER_SIZE)))
+                    StreamUtils.BUFFER_SIZE)));
-            .orElseGet(Flux::empty);
+
-        return uploadContent
+        return Optionals.firstNonEmpty(
-            .switchIfEmpty(downloadContent)
+                contentFromUpload,
-            .switchIfEmpty(backupFileContent);
+                contentFromDownloadUrl,
+                contentFromBackupName,
+                contentFromFilename
+            )
+            .orElseGet(() -> Flux.error(new ServerWebInputException("""
+                Please upload a file or provide a download link or backup name or backup filename.\
+                """)));
     }
 
     @Schema(types = "object")
@@ -157,13 +189,26 @@ public class MigrationEndpoint implements CustomEndpoint {
             return Optional.empty();
         }
 
+        @Schema(requiredMode = NOT_REQUIRED, name = "filename", description = """
+            Filename of backup file in backups root.\
+            """)
+        public Optional<String> getFilename() {
+            var part = multipart.getFirst("filename");
+            if (part instanceof FormFieldPart filenamePart) {
+                return Optional.of(filenamePart.value())
+                    .filter(StringUtils::hasText);
+            }
+            return Optional.empty();
+        }
+
         @Schema(requiredMode = NOT_REQUIRED,
             name = "downloadUrl",
             description = "Remote backup HTTP URL.")
         public Optional<String> getDownloadUrl() {
             var part = multipart.getFirst("downloadUrl");
             if (part instanceof FormFieldPart downloadUrlPart) {
-                return Optional.of(downloadUrlPart.value());
+                return Optional.of(downloadUrlPart.value())
+                    .filter(StringUtils::hasText);
             }
             return Optional.empty();
         }
@@ -174,16 +219,16 @@ public class MigrationEndpoint implements CustomEndpoint {
         public Optional<String> getBackupName() {
             var part = multipart.getFirst("backupName");
             if (part instanceof FormFieldPart backupNamePart) {
-                return Optional.of(backupNamePart.value());
+                return Optional.of(backupNamePart.value())
+                    .filter(StringUtils::hasText);
             }
             return Optional.empty();
         }
     }
 
-
     @Override
     public GroupVersion groupVersion() {
         return GroupVersion.parseAPIVersion(
-            "api.console." + Constant.GROUP + "/" + Constant.VERSION);
+            "console.api." + Constant.GROUP + "/" + Constant.VERSION);
     }
 }

application/src/main/java/run/halo/app/migration/MigrationService.java

@@ -3,6 +3,7 @@ package run.halo.app.migration;
 import org.reactivestreams.Publisher;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.buffer.DataBuffer;
+import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 public interface MigrationService {
@@ -21,4 +22,19 @@ public interface MigrationService {
      */
     Mono<Void> cleanup(Backup backup);
 
+    /**
+     * Gets backup files.
+     *
+     * @return backup files, sorted by last modified time.
+     */
+    Flux<BackupFile> getBackupFiles();
+
+    /**
+     * Get backup file by filename.
+     *
+     * @param filename filename of backup file
+     * @return backup file or empty if file is not found
+     */
+    Mono<BackupFile> getBackupFile(String filename);
+
 }

application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java

@@ -1,6 +1,8 @@
 package run.halo.app.migration.impl;
 
 import static java.nio.file.Files.deleteIfExists;
+import static java.util.Comparator.comparing;
+import static org.apache.commons.io.FilenameUtils.isExtension;
 import static org.springframework.util.FileSystemUtils.copyRecursively;
 import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal;
 import static run.halo.app.infra.utils.FileUtils.copyRecursively;
@@ -19,8 +21,10 @@ import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;
 import java.util.Set;
+import java.util.stream.BaseStream;
 import lombok.extern.slf4j.Slf4j;
 import org.reactivestreams.Publisher;
+import org.springframework.beans.factory.InitializingBean;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
 import org.springframework.core.io.buffer.DataBuffer;
@@ -40,11 +44,12 @@ import run.halo.app.infra.exception.NotFoundException;
 import run.halo.app.infra.properties.HaloProperties;
 import run.halo.app.infra.utils.FileUtils;
 import run.halo.app.migration.Backup;
+import run.halo.app.migration.BackupFile;
 import run.halo.app.migration.MigrationService;
 
 @Slf4j
 @Service
-public class MigrationServiceImpl implements MigrationService {
+public class MigrationServiceImpl implements MigrationService, InitializingBean {
 
     private final ExtensionStoreRepository repository;
 
@@ -163,6 +168,49 @@ public class MigrationServiceImpl implements MigrationService {
         }).subscribeOn(scheduler);
     }
 
+    @Override
+    public Flux<BackupFile> getBackupFiles() {
+        return Flux.using(
+                () -> Files.list(getBackupsRoot()),
+                Flux::fromStream,
+                BaseStream::close
+            )
+            .filter(Files::isRegularFile)
+            .filter(Files::isReadable)
+            .filter(path -> isExtension(path.getFileName().toString(), "zip"))
+            .map(this::toBackupFile)
+            .sort(comparing(BackupFile::getLastModifiedTime).reversed()
+                .thenComparing(BackupFile::getFilename)
+            )
+            .subscribeOn(this.scheduler);
+    }
+
+    @Override
+    public Mono<BackupFile> getBackupFile(String filename) {
+        return Mono.fromCallable(() -> {
+            var backupsRoot = getBackupsRoot();
+            var backupFilePath = backupsRoot.resolve(filename);
+            checkDirectoryTraversal(backupsRoot, backupFilePath);
+            if (Files.notExists(backupFilePath)) {
+                return null;
+            }
+            return toBackupFile(backupFilePath);
+        }).subscribeOn(this.scheduler);
+    }
+
+    private BackupFile toBackupFile(Path path) {
+        var backupFile = new BackupFile();
+        backupFile.setPath(path);
+        backupFile.setFilename(path.getFileName().toString());
+        try {
+            backupFile.setSize(Files.size(path));
+            backupFile.setLastModifiedTime(Files.getLastModifiedTime(path).toInstant());
+            return backupFile;
+        } catch (IOException e) {
+            throw Exceptions.propagate(e);
+        }
+    }
+
     private Mono<Void> restoreWorkdir(Path backupRoot) {
         return Mono.<Void>create(sink -> {
             try {
@@ -264,4 +312,9 @@ public class MigrationServiceImpl implements MigrationService {
                 FileUtils::closeQuietly))
             .subscribeOn(scheduler);
     }
+
+    @Override
+    public void afterPropertiesSet() throws Exception {
+        Files.createDirectories(getBackupsRoot());
+    }
 }

application/src/main/resources/extensions/role-template-migration.yaml

@@ -10,9 +10,15 @@ metadata:
     rbac.authorization.halo.run/ui-permissions: |
       ["system:migrations:manage"]
 rules:
-  - apiGroups: ["api.console.migration.halo.run"]
+  - apiGroups: [ "console.api.migration.halo.run" ]
-    resources: ["restorations"]
+    resources: [ "restorations" ]
-    verbs: ["create"]
+    verbs: [ "create" ]
-  - apiGroups: ["migration.halo.run"]
+  - apiGroups: [ "console.api.migration.halo.run" ]
-    resources: ["backups"]
+    resources: [ "backup-files" ]
-    verbs: ["list", "get", "create", "update", "delete"]
+    verbs: [ "list" ]
+  - apiGroups: [ "console.api.migration.halo.run" ]
+    resources: [ "backups/files" ]
+    verbs: [ "get" ]
+  - apiGroups: [ "migration.halo.run" ]
+    resources: [ "backups" ]
+    verbs: [ "list", "get", "create", "update", "delete", "patch" ]

application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java

@@ -14,6 +14,8 @@ import java.net.URISyntaxException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileTime;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
 import java.util.zip.ZipInputStream;
@@ -198,6 +200,66 @@ class MigrationServiceImplTest {
         verify(backupRoot).get();
     }
 
+    @Test
+    void getBackupFilesTest() throws Exception {
+        var now = Instant.now();
+        var backup1 = tempDir.resolve("backup1.zip");
+        Files.writeString(backup1, "fake-content");
+        Files.setLastModifiedTime(backup1, FileTime.from(now));
+
+        var backup2 = tempDir.resolve("backup2.zip");
+        Files.writeString(backup2, "fake--content");
+        Files.setLastModifiedTime(
+            backup2,
+            FileTime.from(now.plus(Duration.ofSeconds(1)))
+        );
+
+        var backup3 = tempDir.resolve("backup3.not-a-zip");
+        Files.writeString(backup3, "fake-content");
+        Files.setLastModifiedTime(
+            backup3,
+            FileTime.from(now.plus(Duration.ofSeconds(2)))
+        );
+        when(backupRoot.get()).thenReturn(tempDir);
+
+        migrationService.afterPropertiesSet();
+        migrationService.getBackupFiles()
+            .as(StepVerifier::create)
+            .assertNext(backupFile -> {
+                assertEquals("backup2.zip", backupFile.getFilename());
+                assertEquals(13, backupFile.getSize());
+                assertEquals(now.plus(Duration.ofSeconds(1)), backupFile.getLastModifiedTime());
+            })
+            .assertNext(backupFile -> {
+                assertEquals("backup1.zip", backupFile.getFilename());
+                assertEquals(12, backupFile.getSize());
+                assertEquals(now, backupFile.getLastModifiedTime());
+            })
+            .verifyComplete();
+    }
+
+    @Test
+    void getBackupFileTest() throws Exception {
+        var now = Instant.now();
+        Files.writeString(tempDir.resolve("backup.zip"), "fake-content");
+        Files.setLastModifiedTime(tempDir.resolve("backup.zip"), FileTime.from(now));
+        when(backupRoot.get()).thenReturn(tempDir);
+
+        migrationService.afterPropertiesSet();
+        migrationService.getBackupFile("backup.zip")
+            .as(StepVerifier::create)
+            .assertNext(backupFile -> {
+                assertEquals("backup.zip", backupFile.getFilename());
+                assertEquals(12, backupFile.getSize());
+                assertEquals(now, backupFile.getLastModifiedTime());
+            })
+            .verifyComplete();
+
+        migrationService.getBackupFile("backup-not-exist.zip")
+            .as(StepVerifier::create)
+            .verifyComplete();
+    }
+
     Backup createSucceededBackup(String name, String filename) {
         var metadata = new Metadata();
         metadata.setName(name);