github
/
halo
mirror of https://github.com/halo-dev/halo
1
0
Fork 0
Code Issues Releases Wiki Activity

Add support for restoring from backup root 

Signed-off-by: JohnNiang <johnniang@foxmail.com>
pull/6486/head
JohnNiang 2024-08-21 11:45:49 +08:00
parent 3a782be607
commit 3460d4c94b
8 changed files with 444 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
api-docs/openapi/v3_0/aggregated.json
View File

@ -5239,55 +5239,6 @@
]
}
},
"/apis/api.console.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
"get": {
"operationId": "DownloadBackups",
"parameters": [
{
"description": "Backup name.",
"in": "path",
"name": "name",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "Backup filename.",
"in": "path",
"name": "filename",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/api.console.migration.halo.run/v1alpha1/restorations": {
"post": {
"description": "Restore backup by uploading file or providing download link or backup name.",
"operationId": "RestoreBackup",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"$ref": "#/components/schemas/RestoreRequest"
}
}
},
"required": true
},
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/api.content.halo.run/v1alpha1/categories": {
"get": {
"description": "Lists categories.",
@ -7411,6 +7362,79 @@
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/backup-files": {
"get": {
"description": "Get backup files from backup root.",
"operationId": "getBackupFiles",
"responses": {
"default": {
"content": {
"*/*": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BackupFile"
}
}
}
},
"description": "default response"
}
},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
"get": {
"operationId": "DownloadBackups",
"parameters": [
{
"description": "Backup name.",
"in": "path",
"name": "name",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "Backup filename.",
"in": "path",
"name": "filename",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/restorations": {
"post": {
"description": "Restore backup by uploading file or providing download link or backup name.",
"operationId": "RestoreBackup",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"$ref": "#/components/schemas/RestoreRequest"
}
}
},
"required": true
},
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": {
"post": {
"description": "Verify email sender config.",
@ -15300,6 +15324,22 @@
}
}
},
"BackupFile": {
"type": "object",
"properties": {
"filename": {
"type": "string"
},
"lastModifiedTime": {
"type": "string",
"format": "date-time"
},
"size": {
"type": "integer",
"format": "int64"
}
}
},
"BackupList": {
"required": [
"first",
@ -20684,6 +20724,10 @@
"file": {
"type": "string",
"format": "binary"
},
"filename": {
"type": "string",
"description": "Filename of backup file in backups root."
}
}
},

110
api-docs/openapi/v3_0/apis_console.api_v1alpha1.json
View File

@ -3106,6 +3106,79 @@
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/backup-files": {
"get": {
"description": "Get backup files from backup root.",
"operationId": "getBackupFiles",
"responses": {
"default": {
"content": {
"*/*": {
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/BackupFile"
}
}
}
},
"description": "default response"
}
},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/backups/{name}/files/{filename}": {
"get": {
"operationId": "DownloadBackups",
"parameters": [
{
"description": "Backup name.",
"in": "path",
"name": "name",
"required": true,
"schema": {
"type": "string"
}
},
{
"description": "Backup filename.",
"in": "path",
"name": "filename",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.migration.halo.run/v1alpha1/restorations": {
"post": {
"description": "Restore backup by uploading file or providing download link or backup name.",
"operationId": "RestoreBackup",
"requestBody": {
"content": {
"multipart/form-data": {
"schema": {
"$ref": "#/components/schemas/RestoreRequest"
}
}
},
"required": true
},
"responses": {},
"tags": [
"MigrationV1alpha1Console"
]
}
},
"/apis/console.api.notification.halo.run/v1alpha1/notifiers/default-email-notifier/verify-connection": {
"post": {
"description": "Verify email sender config.",
@ -3376,6 +3449,22 @@
}
}
},
"BackupFile": {
"type": "object",
"properties": {
"filename": {
"type": "string"
},
"lastModifiedTime": {
"type": "string",
"format": "date-time"
},
"size": {
"type": "integer",
"format": "int64"
}
}
},
"Category": {
"required": [
"apiVersion",
@ -5405,6 +5494,27 @@
}
}
},
"RestoreRequest": {
"type": "object",
"properties": {
"backupName": {
"type": "string",
"description": "Backup metadata name."
},
"downloadUrl": {
"type": "string",
"description": "Remote backup HTTP URL."
},
"file": {
"type": "string",
"format": "binary"
},
"filename": {
"type": "string",
"description": "Filename of backup file in backups root."
}
}
},
"RevertSnapshotForPostParam": {
"required": [
"snapshotName"

34
application/src/main/java/run/halo/app/migration/BackupFile.java Normal file
View File

@ -0,0 +1,34 @@
package run.halo.app.migration;
import com.fasterxml.jackson.annotation.JsonIgnore;
import java.nio.file.Path;
import java.time.Instant;
import lombok.Data;
/**
* Backup file.
*
* @author johnniang
*/
@Data
public class BackupFile {
@JsonIgnore
private Path path;
/**
* Filename of backup file.
*/
private String filename;
/**
* Size of backup file.
*/
private long size;
/**
* Last modified time of backup file.
*/
private Instant lastModifiedTime;
}

81
application/src/main/java/run/halo/app/migration/MigrationEndpoint.java
View File

@ -1,6 +1,7 @@
package run.halo.app.migration;
import static io.swagger.v3.oas.annotations.media.Schema.RequiredMode.NOT_REQUIRED;
import static org.springdoc.core.fn.builders.apiresponse.Builder.responseBuilder;
import static org.springdoc.core.fn.builders.content.Builder.contentBuilder;
import static org.springdoc.core.fn.builders.parameter.Builder.parameterBuilder;
import static org.springdoc.core.fn.builders.requestbody.Builder.requestBodyBuilder;
@ -12,10 +13,12 @@ import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Optional;
import java.util.function.Supplier;
import org.springdoc.webflux.core.fn.SpringdocRouteBuilder;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.data.util.Optionals;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.codec.multipart.FilePart;
@ -24,7 +27,9 @@ import org.springframework.http.codec.multipart.Part;
import org.springframework.stereotype.Component;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StreamUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.reactive.function.server.RouterFunction;
import org.springframework.web.reactive.function.server.ServerRequest;
import org.springframework.web.reactive.function.server.ServerResponse;
import org.springframework.web.server.ServerWebInputException;
import reactor.core.publisher.Flux;
@ -55,6 +60,15 @@ public class MigrationEndpoint implements CustomEndpoint {
public RouterFunction<ServerResponse> endpoint() {
var tag = "MigrationV1alpha1Console";
return SpringdocRouteBuilder.route()
.GET("/backup-files",
this::getBackups,
builder -> builder.operationId("getBackupFiles")
.tag(tag)
.description("Get backup files from backup root.")
.response(responseBuilder()
.implementationArray(BackupFile.class)
)
)
.GET("/backups/{name}/files/{filename}",
request -> {
var name = request.pathVariable("name");
@ -86,7 +100,7 @@ public class MigrationEndpoint implements CustomEndpoint {
var content = getContent(restoreRequest)
.switchIfEmpty(Mono.error(() -> new ServerWebInputException(
"Please upload a file "
+ "or provide a download link or backup name.")));
+ "or provide a download link or backup name.")));
return migrationService.restore(content);
})
.then(Mono.defer(
@ -95,7 +109,7 @@ public class MigrationEndpoint implements CustomEndpoint {
builder -> builder
.tag(tag)
.description("Restore backup by uploading file "
+ "or providing download link or backup name.")
+ "or providing download link or backup name.")
.operationId("RestoreBackup")
.requestBody(requestBodyBuilder()
.required(true)
@ -108,8 +122,22 @@ public class MigrationEndpoint implements CustomEndpoint {
.build();
}
private Mono<ServerResponse> getBackups(ServerRequest request) {
var backupFiles = migrationService.getBackupFiles();
return ServerResponse.ok().body(backupFiles, BackupFile.class);
}
private Flux<DataBuffer> getContent(RestoreRequest request) {
var downloadContent = request.getDownloadUrl()
Supplier<Optional<Flux<DataBuffer>>> contentFromFilename = () ->
request.getFilename().map(filename -> migrationService.getBackupFile(filename)
.map(BackupFile::getPath)
.flatMapMany(
path -> DataBufferUtils.read(
path,
DefaultDataBufferFactory.sharedInstance,
StreamUtils.BUFFER_SIZE)));
Supplier<Optional<Flux<DataBuffer>>> contentFromDownloadUrl = () -> request.getDownloadUrl()
.map(downloadURL -> {
try {
var url = new URL(downloadURL);
@ -121,23 +149,27 @@ public class MigrationEndpoint implements CustomEndpoint {
// Should never happen
return Flux.<DataBuffer>error(e);
}
})
.orElseGet(Flux::empty);
});
var uploadContent = request.getFile()
.map(Part::content)
.orElseGet(Flux::empty);
Supplier<Optional<Flux<DataBuffer>>> contentFromUpload = () -> request.getFile()
.map(Part::content);
var backupFileContent = request.getBackupName()
Supplier<Optional<Flux<DataBuffer>>> contentFromBackupName = () -> request.getBackupName()
.map(backupName -> client.get(Backup.class, backupName)
.flatMap(migrationService::download)
.flatMapMany(resource -> DataBufferUtils.read(resource,
DefaultDataBufferFactory.sharedInstance,
StreamUtils.BUFFER_SIZE)))
.orElseGet(Flux::empty);
return uploadContent
.switchIfEmpty(downloadContent)
.switchIfEmpty(backupFileContent);
StreamUtils.BUFFER_SIZE)));
return Optionals.firstNonEmpty(
contentFromUpload,
contentFromDownloadUrl,
contentFromBackupName,
contentFromFilename
)
.orElseGet(() -> Flux.error(new ServerWebInputException("""
Please upload a file or provide a download link or backup name or backup filename.\
""")));
}
@Schema(types = "object")
@ -157,13 +189,26 @@ public class MigrationEndpoint implements CustomEndpoint {
return Optional.empty();
}
@Schema(requiredMode = NOT_REQUIRED, name = "filename", description = """
Filename of backup file in backups root.\
""")
public Optional<String> getFilename() {
var part = multipart.getFirst("filename");
if (part instanceof FormFieldPart filenamePart) {
return Optional.of(filenamePart.value())
.filter(StringUtils::hasText);
}
return Optional.empty();
}
@Schema(requiredMode = NOT_REQUIRED,
name = "downloadUrl",
description = "Remote backup HTTP URL.")
public Optional<String> getDownloadUrl() {
var part = multipart.getFirst("downloadUrl");
if (part instanceof FormFieldPart downloadUrlPart) {
return Optional.of(downloadUrlPart.value());
return Optional.of(downloadUrlPart.value())
.filter(StringUtils::hasText);
}
return Optional.empty();
}
@ -174,16 +219,16 @@ public class MigrationEndpoint implements CustomEndpoint {
public Optional<String> getBackupName() {
var part = multipart.getFirst("backupName");
if (part instanceof FormFieldPart backupNamePart) {
return Optional.of(backupNamePart.value());
return Optional.of(backupNamePart.value())
.filter(StringUtils::hasText);
}
return Optional.empty();
}
}
@Override
public GroupVersion groupVersion() {
return GroupVersion.parseAPIVersion(
"api.console." + Constant.GROUP + "/" + Constant.VERSION);
"console.api." + Constant.GROUP + "/" + Constant.VERSION);
}
}

16
application/src/main/java/run/halo/app/migration/MigrationService.java
View File

@ -3,6 +3,7 @@ package run.halo.app.migration;
import org.reactivestreams.Publisher;
import org.springframework.core.io.Resource;
import org.springframework.core.io.buffer.DataBuffer;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
public interface MigrationService {
@ -21,4 +22,19 @@ public interface MigrationService {
*/
Mono<Void> cleanup(Backup backup);
/**
* Gets backup files.
*
* @return backup files, sorted by last modified time.
*/
Flux<BackupFile> getBackupFiles();
/**
* Get backup file by filename.
*
* @param filename filename of backup file
* @return backup file or empty if file is not found
*/
Mono<BackupFile> getBackupFile(String filename);
}

55
application/src/main/java/run/halo/app/migration/impl/MigrationServiceImpl.java
View File

@ -1,6 +1,8 @@
package run.halo.app.migration.impl;
import static java.nio.file.Files.deleteIfExists;
import static java.util.Comparator.comparing;
import static org.apache.commons.io.FilenameUtils.isExtension;
import static org.springframework.util.FileSystemUtils.copyRecursively;
import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal;
import static run.halo.app.infra.utils.FileUtils.copyRecursively;
@ -19,8 +21,10 @@ import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Locale;
import java.util.Set;
import java.util.stream.BaseStream;
import lombok.extern.slf4j.Slf4j;
import org.reactivestreams.Publisher;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.core.io.buffer.DataBuffer;
@ -40,11 +44,12 @@ import run.halo.app.infra.exception.NotFoundException;
import run.halo.app.infra.properties.HaloProperties;
import run.halo.app.infra.utils.FileUtils;
import run.halo.app.migration.Backup;
import run.halo.app.migration.BackupFile;
import run.halo.app.migration.MigrationService;
@Slf4j
@Service
public class MigrationServiceImpl implements MigrationService {
public class MigrationServiceImpl implements MigrationService, InitializingBean {
private final ExtensionStoreRepository repository;
@ -163,6 +168,49 @@ public class MigrationServiceImpl implements MigrationService {
}).subscribeOn(scheduler);
}
@Override
public Flux<BackupFile> getBackupFiles() {
return Flux.using(
() -> Files.list(getBackupsRoot()),
Flux::fromStream,
BaseStream::close
)
.filter(Files::isRegularFile)
.filter(Files::isReadable)
.filter(path -> isExtension(path.getFileName().toString(), "zip"))
.map(this::toBackupFile)
.sort(comparing(BackupFile::getLastModifiedTime).reversed()
.thenComparing(BackupFile::getFilename)
)
.subscribeOn(this.scheduler);
}
@Override
public Mono<BackupFile> getBackupFile(String filename) {
return Mono.fromCallable(() -> {
var backupsRoot = getBackupsRoot();
var backupFilePath = backupsRoot.resolve(filename);
checkDirectoryTraversal(backupsRoot, backupFilePath);
if (Files.notExists(backupFilePath)) {
return null;
}
return toBackupFile(backupFilePath);
}).subscribeOn(this.scheduler);
}
private BackupFile toBackupFile(Path path) {
var backupFile = new BackupFile();
backupFile.setPath(path);
backupFile.setFilename(path.getFileName().toString());
try {
backupFile.setSize(Files.size(path));
backupFile.setLastModifiedTime(Files.getLastModifiedTime(path).toInstant());
return backupFile;
} catch (IOException e) {
throw Exceptions.propagate(e);
}
}
private Mono<Void> restoreWorkdir(Path backupRoot) {
return Mono.<Void>create(sink -> {
try {
@ -264,4 +312,9 @@ public class MigrationServiceImpl implements MigrationService {
FileUtils::closeQuietly))
.subscribeOn(scheduler);
}
@Override
public void afterPropertiesSet() throws Exception {
Files.createDirectories(getBackupsRoot());
}
}

18
application/src/main/resources/extensions/role-template-migration.yaml
View File

@ -10,9 +10,15 @@ metadata:
rbac.authorization.halo.run/ui-permissions: |
["system:migrations:manage"]
rules:
- apiGroups: ["api.console.migration.halo.run"]
resources: ["restorations"]
verbs: ["create"]
- apiGroups: ["migration.halo.run"]
resources: ["backups"]
verbs: ["list", "get", "create", "update", "delete"]
- apiGroups: [ "console.api.migration.halo.run" ]
resources: [ "restorations" ]
verbs: [ "create" ]
- apiGroups: [ "console.api.migration.halo.run" ]
resources: [ "backup-files" ]
verbs: [ "list" ]
- apiGroups: [ "console.api.migration.halo.run" ]
resources: [ "backups/files" ]
verbs: [ "get" ]
- apiGroups: [ "migration.halo.run" ]
resources: [ "backups" ]
verbs: [ "list", "get", "create", "update", "delete", "patch" ]

62
application/src/test/java/run/halo/app/migration/impl/MigrationServiceImplTest.java
View File

@ -14,6 +14,8 @@ import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.FileTime;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import java.util.zip.ZipInputStream;
@ -198,6 +200,66 @@ class MigrationServiceImplTest {
verify(backupRoot).get();
}
@Test
void getBackupFilesTest() throws Exception {
var now = Instant.now();
var backup1 = tempDir.resolve("backup1.zip");
Files.writeString(backup1, "fake-content");
Files.setLastModifiedTime(backup1, FileTime.from(now));
var backup2 = tempDir.resolve("backup2.zip");
Files.writeString(backup2, "fake--content");
Files.setLastModifiedTime(
backup2,
FileTime.from(now.plus(Duration.ofSeconds(1)))
);
var backup3 = tempDir.resolve("backup3.not-a-zip");
Files.writeString(backup3, "fake-content");
Files.setLastModifiedTime(
backup3,
FileTime.from(now.plus(Duration.ofSeconds(2)))
);
when(backupRoot.get()).thenReturn(tempDir);
migrationService.afterPropertiesSet();
migrationService.getBackupFiles()
.as(StepVerifier::create)
.assertNext(backupFile -> {
assertEquals("backup2.zip", backupFile.getFilename());
assertEquals(13, backupFile.getSize());
assertEquals(now.plus(Duration.ofSeconds(1)), backupFile.getLastModifiedTime());
})
.assertNext(backupFile -> {
assertEquals("backup1.zip", backupFile.getFilename());
assertEquals(12, backupFile.getSize());
assertEquals(now, backupFile.getLastModifiedTime());
})
.verifyComplete();
}
@Test
void getBackupFileTest() throws Exception {
var now = Instant.now();
Files.writeString(tempDir.resolve("backup.zip"), "fake-content");
Files.setLastModifiedTime(tempDir.resolve("backup.zip"), FileTime.from(now));
when(backupRoot.get()).thenReturn(tempDir);
migrationService.afterPropertiesSet();
migrationService.getBackupFile("backup.zip")
.as(StepVerifier::create)
.assertNext(backupFile -> {
assertEquals("backup.zip", backupFile.getFilename());
assertEquals(12, backupFile.getSize());
assertEquals(now, backupFile.getLastModifiedTime());
})
.verifyComplete();
migrationService.getBackupFile("backup-not-exist.zip")
.as(StepVerifier::create)
.verifyComplete();
}
Backup createSucceededBackup(String name, String filename) {
var metadata = new Metadata();
metadata.setName(name);