github
/
halo
mirror of https://github.com/halo-dev/halo
1
0
Fork 0
Code Issues Releases Wiki Activity

Implement thumbnail cleanup and add width parameter support in reconciler

John Niang 2025-09-30 19:09:15 +08:00
parent c5736326a3
commit b941047c99
No known key found for this signature in database
GPG Key ID: D7363C015BBCAA59
11 changed files with 107 additions and 1045 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
api/src/main/java/run/halo/app/core/attachment/ThumbnailProvider.java
View File

@ -6,7 +6,16 @@ import lombok.Builder;
import lombok.Data; import lombok.Data;
import org.pf4j.ExtensionPoint; import org.pf4j.ExtensionPoint;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import run.halo.app.core.extension.attachment.endpoint.AttachmentHandler;
/**
* Thumbnail provider extension.
*
* @since 2.22.0
* @deprecated Use {@link AttachmentHandler} instead. We are planing to remove this extension
* point in future release.
*/
@Deprecated(forRemoval = true, since = "2.22.0")
public interface ThumbnailProvider extends ExtensionPoint { public interface ThumbnailProvider extends ExtensionPoint {
/** /**

8
api/src/main/java/run/halo/app/core/attachment/ThumbnailSize.java
View File

@ -1,5 +1,6 @@
package run.halo.app.core.attachment; package run.halo.app.core.attachment;
import java.util.Arrays;
import java.util.Optional; import java.util.Optional;
import lombok.Getter; import lombok.Getter;
@ -50,4 +51,11 @@ public enum ThumbnailSize {
} }
return Optional.empty(); return Optional.empty();
} }
public static Integer[] allowedWidths() {
return Arrays.stream(ThumbnailSize.values())
.map(ThumbnailSize::getWidth)
.toArray(Integer[]::new);
}
} }

43
application/src/main/java/run/halo/app/core/attachment/LocalThumbnailProvider.java
View File

@ -1,43 +0,0 @@
package run.halo.app.core.attachment;
import java.net.URI;
import java.net.URL;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import reactor.core.publisher.Mono;
import run.halo.app.infra.ExternalUrlSupplier;
@Component
@RequiredArgsConstructor
public class LocalThumbnailProvider implements ThumbnailProvider {
private final ExternalUrlSupplier externalUrlSupplier;
private final LocalThumbnailService localThumbnailService;
@Override
public Mono<URI> generate(ThumbnailContext context) {
return localThumbnailService.create(context.getImageUrl(), context.getSize())
.map(localThumbnail -> localThumbnail.getSpec().getThumbnailUri())
.map(URI::create);
}
@Override
public Mono<Void> delete(URL imageUrl) {
Assert.notNull(imageUrl, "Image URL must not be null");
return localThumbnailService.delete(URI.create(imageUrl.toString()));
}
@Override
public Mono<Boolean> supports(ThumbnailContext context) {
var imageUrl = context.getImageUrl();
var externalUrl = externalUrlSupplier.getRaw();
return Mono.fromSupplier(() -> externalUrl != null
&& isSameOrigin(imageUrl, externalUrl));
}
private boolean isSameOrigin(URL imageUrl, URL externalUrl) {
return StringUtils.equals(imageUrl.getHost(), externalUrl.getHost())
&& imageUrl.getPort() == externalUrl.getPort();
}
}

81
application/src/main/java/run/halo/app/core/attachment/LocalThumbnailService.java
View File

@ -1,81 +0,0 @@
package run.halo.app.core.attachment;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import org.springframework.core.io.Resource;
import org.springframework.lang.NonNull;
import reactor.core.publisher.Mono;
import run.halo.app.core.attachment.extension.LocalThumbnail;
import run.halo.app.infra.ExternalLinkProcessor;
import run.halo.app.infra.exception.NotFoundException;
public interface LocalThumbnailService {
/**
* Gets original image URI for the given thumbnail URI.
*
* @param thumbnailUri The thumbnail URI string
* @return The original image URI, {@link NotFoundException} will be thrown if the thumbnail
* record does not exist by the given thumbnail URI
*/
Mono<URI> getOriginalImageUri(URI thumbnailUri);
/**
* <p>Gets thumbnail file resource for the given year, size and filename.</p>
* {@link Mono#empty()} will be returned if the thumbnail file does not generate yet or the
* thumbnail record does not exist.
*
* @param thumbnailUri The thumbnail URI string
* @return The thumbnail file resource
*/
Mono<Resource> getThumbnail(URI thumbnailUri);
/**
* <p>Gets thumbnail file resource for the given URI and size.</p>
* {@link Mono#empty()} will be returned if the thumbnail file does not generate yet.
*
* @param originalImageUri original image URI to get thumbnail
* @param size thumbnail size
*/
Mono<Resource> getThumbnail(URI originalImageUri, ThumbnailSize size);
/**
* Generate thumbnail file for the given thumbnail.
* Do nothing if the thumbnail file already exists.
*
* @param thumbnail The thumbnail to generate.
* @return The generated thumbnail file resource.
*/
Mono<Resource> generate(LocalThumbnail thumbnail);
/**
* Creates a {@link LocalThumbnail} record for the given image URL and size.
* The thumbnail file will be generated asynchronously according to the thumbnail record.
*
* @param imageUrl original image URL
* @param size thumbnail size to generate
* @return The created thumbnail record.
*/
Mono<LocalThumbnail> create(URL imageUrl, ThumbnailSize size);
/**
* Deletes the all size thumbnail files for the given image URI.
* If the image URI is not absolute, it will be processed by {@link ExternalLinkProcessor}.
*
* @param imageUri original image URI to delete thumbnails
* @return A {@link Mono} indicates the completion of the deletion.
*/
Mono<Void> delete(URI imageUri);
/**
* Ensures the image URI is an url path if it's an in-site image.
* If it's not an in-site image, it will return directly.
*/
@NonNull
URI ensureInSiteUriIsRelative(URI imageUri);
Path toFilePath(String thumbRelativeUnixPath);
URI buildThumbnailUri(String year, ThumbnailSize size, String filename);
}

237
application/src/main/java/run/halo/app/core/attachment/ThumbnailGenerator.java
View File

@ -1,237 +0,0 @@
package run.halo.app.core.attachment;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import com.drew.imaging.ImageMetadataReader;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.exif.ExifIFD0Directory;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Iterator;
import java.util.Optional;
import java.util.Set;
import javax.imageio.ImageIO;
import javax.imageio.ImageReader;
import javax.imageio.stream.ImageInputStream;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.imgscalr.Scalr;
import org.springframework.lang.NonNull;
@Slf4j
@AllArgsConstructor
public class ThumbnailGenerator {
/**
* Max file size in bytes for downloading.
* 30MB
*/
static final int MAX_FILE_SIZE = 30 * 1024 * 1024;
private static final Set<String> UNSUPPORTED_FORMATS = Set.of("gif", "svg", "webp");
private final ImageDownloader imageDownloader = new ImageDownloader();
private final ThumbnailSize size;
private final Path storePath;
/**
* Generate thumbnail and save it to store path.
*/
public void generate(URL imageUrl) {
Path tempImagePath = null;
try {
tempImagePath = imageDownloader.downloadFile(imageUrl);
generateThumbnail(tempImagePath);
} catch (IOException e) {
throw new IllegalStateException(e);
} finally {
if (tempImagePath != null) {
try {
Files.deleteIfExists(tempImagePath);
} catch (IOException e) {
// Ignore
}
}
}
}
/**
* Generate thumbnail by image file path.
*/
public void generate(Path imageFilePath) {
try {
generateThumbnail(imageFilePath);
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
private void generateThumbnail(Path tempImagePath) throws IOException {
var file = tempImagePath.toFile();
if (file.length() > MAX_FILE_SIZE) {
throw new IOException("File size exceeds the limit: " + MAX_FILE_SIZE);
}
String formatName = getFormatName(file)
.orElseThrow(() -> new UnsupportedOperationException("Unknown format"));
if (isUnsupportedFormat(formatName)) {
throw new UnsupportedOperationException("Unsupported image format for: " + formatName);
}
var img = ImageIO.read(file);
if (img == null) {
throw new UnsupportedOperationException("Cannot read image file: " + file);
}
var thumbnailFile = getThumbnailFile(formatName);
if (img.getWidth() <= size.getWidth()) {
Files.copy(tempImagePath, thumbnailFile.toPath(), REPLACE_EXISTING);
return;
}
var thumbnail = Scalr.resize(img, Scalr.Method.AUTOMATIC, Scalr.Mode.FIT_TO_WIDTH,
size.getWidth());
// Rotate image if needed
var orientation = readExifOrientation(file);
if (orientation != null) {
thumbnail = Scalr.rotate(thumbnail, orientation);
}
ImageIO.write(thumbnail, formatName, thumbnailFile);
}
private static Scalr.Rotation readExifOrientation(File inputFile) {
try {
Metadata metadata = ImageMetadataReader.readMetadata(inputFile);
Directory directory = metadata.getFirstDirectoryOfType(ExifIFD0Directory.class);
if (directory != null && directory.containsTag(ExifIFD0Directory.TAG_ORIENTATION)) {
return getScalrRotationFromExifOrientation(
directory.getInt(ExifIFD0Directory.TAG_ORIENTATION));
}
} catch (Exception e) {
log.debug("Failed to read EXIF orientation from file: {}", inputFile, e);
}
return null;
}
private static Scalr.Rotation getScalrRotationFromExifOrientation(int orientation) {
// https://www.media.mit.edu/pia/Research/deepview/exif.html#:~:text=0x0112-,Orientation,-unsigned%20short
return switch (orientation) {
case 3 -> Scalr.Rotation.CW_180;
case 6 -> Scalr.Rotation.CW_90;
case 8 -> Scalr.Rotation.CW_270;
default -> null;
};
}
private static boolean isUnsupportedFormat(@NonNull String formatName) {
return UNSUPPORTED_FORMATS.contains(formatName.toLowerCase());
}
private File getThumbnailFile(String formatName) {
return Optional.of(storePath)
.map(path -> {
if (storePath.endsWith(formatName)) {
return storePath.resolve("." + formatName);
}
return storePath;
})
.map(path -> {
if (!Files.exists(path.getParent())) {
try {
Files.createDirectories(path.getParent());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return path;
})
.map(Path::toFile)
.orElseThrow();
}
/**
* Sanitize file name.
*
* @param fileName file name to sanitize
*/
public static String sanitizeFileName(String fileName) {
String sanitizedFileName = fileName.replaceAll("[^a-zA-Z0-9\\.\\-]", "");
if (sanitizedFileName.length() > 255) {
sanitizedFileName = sanitizedFileName.substring(0, 255);
}
return sanitizedFileName;
}
private static Optional<String> getFormatName(File file) {
try {
return Optional.of(doGetFormatName(file));
} catch (IOException e) {
// Ignore
}
return Optional.empty();
}
private static String doGetFormatName(File file) throws IOException {
try (ImageInputStream imageStream = ImageIO.createImageInputStream(file)) {
Iterator<ImageReader> readers = ImageIO.getImageReaders(imageStream);
if (!readers.hasNext()) {
throw new IOException("No ImageReader found for the image.");
}
ImageReader reader = readers.next();
return reader.getFormatName().toLowerCase();
}
}
static class ImageDownloader {
public Path downloadFile(URL url) throws IOException {
return downloadFileInternal(encodedUrl(url));
}
private static URL encodedUrl(URL url) {
try {
return new URL(url.toURI().toASCIIString());
} catch (MalformedURLException | URISyntaxException e) {
throw new IllegalArgumentException(e);
}
}
Path downloadFileInternal(URL url) throws IOException {
File tempFile = File.createTempFile("halo-image-thumb-", ".tmp");
long totalBytesDownloaded = 0;
var tempFilePath = tempFile.toPath();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
+ " Chrome/92.0.4515.131 Safari/537.36");
try (InputStream inputStream = connection.getInputStream();
FileOutputStream outputStream = new FileOutputStream(tempFile)) {
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
totalBytesDownloaded += bytesRead;
if (totalBytesDownloaded > MAX_FILE_SIZE) {
outputStream.close();
Files.deleteIfExists(tempFilePath);
throw new IOException("File size exceeds the limit: " + MAX_FILE_SIZE);
}
outputStream.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
Files.deleteIfExists(tempFilePath);
throw e;
}
return tempFile.toPath();
}
}
}

294
application/src/main/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImpl.java
View File

@ -1,294 +0,0 @@
package run.halo.app.core.attachment.impl;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
import static org.apache.commons.lang3.StringUtils.defaultIfBlank;
import static org.apache.commons.lang3.StringUtils.removeStart;
import static org.apache.commons.lang3.StringUtils.substringAfterLast;
import static run.halo.app.extension.MetadataUtil.nullSafeAnnotations;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.isNull;
import java.net.URI;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.concurrent.Callable;
import java.util.function.Consumer;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.dao.OptimisticLockingFailureException;
import org.springframework.data.domain.Sort;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import reactor.core.Exceptions;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import reactor.util.retry.Retry;
import run.halo.app.core.attachment.AttachmentRootGetter;
import run.halo.app.core.attachment.LocalThumbnailService;
import run.halo.app.core.attachment.ThumbnailGenerator;
import run.halo.app.core.attachment.ThumbnailSigner;
import run.halo.app.core.attachment.ThumbnailSize;
import run.halo.app.core.attachment.extension.LocalThumbnail;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.Metadata;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.infra.ExternalUrlSupplier;
import run.halo.app.infra.exception.NotFoundException;
@Slf4j
@Component
@RequiredArgsConstructor
public class LocalThumbnailServiceImpl implements LocalThumbnailService {
private final AttachmentRootGetter attachmentDirGetter;
private final ReactiveExtensionClient client;
private final ExternalUrlSupplier externalUrlSupplier;
private static Path buildThumbnailStorePath(Path rootPath, String fileName, String year,
ThumbnailSize size) {
return rootPath
.resolve("thumbnails")
.resolve(year)
.resolve("w" + size.getWidth())
.resolve(fileName);
}
static String geImageFileName(URI imageUri) {
var fileName = substringAfterLast(imageUri.getPath(), "/");
fileName = defaultIfBlank(fileName, randomAlphanumeric(10));
return ThumbnailGenerator.sanitizeFileName(fileName);
}
static String getYear() {
return String.valueOf(LocalDateTime.now().getYear());
}
@Override
public Mono<URI> getOriginalImageUri(URI thumbnailUri) {
return fetchThumbnail(thumbnailUri)
.map(local -> URI.create(local.getSpec().getImageUri()))
.switchIfEmpty(Mono.error(() -> new NotFoundException("Resource not found.")));
}
@Override
public Mono<Resource> getThumbnail(URI thumbnailUri) {
Assert.notNull(thumbnailUri, "Thumbnail URI must not be null.");
return fetchThumbnail(thumbnailUri)
.flatMap(thumbnail -> {
var filePath = toFilePath(thumbnail.getSpec().getFilePath());
if (Files.exists(filePath)) {
return getResourceMono(() -> new FileSystemResource(filePath));
}
return generate(thumbnail)
.then(Mono.empty());
});
}
@Override
public Mono<Resource> getThumbnail(URI originalImageUri, ThumbnailSize size) {
var imageHash = signatureForImageUri(originalImageUri);
return fetchByImageHashAndSize(imageHash, size)
.flatMap(this::generate);
}
private Mono<LocalThumbnail> fetchThumbnail(URI thumbnailUri) {
Assert.notNull(thumbnailUri, "Thumbnail URI must not be null.");
var thumbSignature = ThumbnailSigner.generateSignature(thumbnailUri);
return client.listBy(LocalThumbnail.class, ListOptions.builder()
.fieldQuery(equal("spec.thumbSignature", thumbSignature))
.build(), PageRequestImpl.ofSize(1))
.flatMap(result -> Mono.justOrEmpty(ListResult.first(result)));
}
private Mono<LocalThumbnail> fetchByImageHashAndSize(String imageSignature,
ThumbnailSize size) {
var indexValue = LocalThumbnail.uniqueImageAndSize(imageSignature, size);
return client.listBy(LocalThumbnail.class, ListOptions.builder()
.fieldQuery(equal(LocalThumbnail.UNIQUE_IMAGE_AND_SIZE_INDEX, indexValue))
.build(), PageRequestImpl.ofSize(ThumbnailSize.values().length)
)
.flatMapMany(result -> Flux.fromIterable(result.getItems()))
.next();
}
@Override
public Mono<Resource> generate(LocalThumbnail thumbnail) {
Assert.notNull(thumbnail, "Thumbnail must not be null.");
var filePath = toFilePath(thumbnail.getSpec().getFilePath());
if (Files.exists(filePath)) {
return getResourceMono(() -> new FileSystemResource(filePath));
}
return updateWithRetry(thumbnail,
record -> nullSafeAnnotations(record)
.put(LocalThumbnail.REQUEST_TO_GENERATE_ANNO, "true"))
.then(Mono.empty());
}
private static Mono<Resource> getResourceMono(Callable<Resource> callable) {
return Mono.fromCallable(callable)
.subscribeOn(Schedulers.boundedElastic());
}
private Mono<Void> updateWithRetry(LocalThumbnail localThumbnail, Consumer<LocalThumbnail> op) {
op.accept(localThumbnail);
return client.update(localThumbnail)
.onErrorResume(OptimisticLockingFailureException.class,
e -> client.fetch(LocalThumbnail.class, localThumbnail.getMetadata().getName())
.flatMap(latest -> {
op.accept(latest);
return client.update(latest);
})
.retryWhen(Retry.backoff(5, Duration.ofMillis(100))
.filter(OptimisticLockingFailureException.class::isInstance))
)
.then();
}
@Override
public Mono<LocalThumbnail> create(URL imageUrl, ThumbnailSize size) {
Assert.notNull(imageUrl, "Image URL must not be null.");
Assert.notNull(size, "Thumbnail size must not be null.");
var imageUri = URI.create(imageUrl.toString());
var imageHash = signatureForImageUri(imageUri);
return fetchByImageHashAndSize(imageHash, size)
.switchIfEmpty(Mono.defer(() -> doCreate(imageUri, size)));
}
private Mono<LocalThumbnail> doCreate(URI imageUri, ThumbnailSize size) {
var year = getYear();
var originalFileName = geImageFileName(imageUri);
return generateUniqueThumbFileName(originalFileName, year, size)
.flatMap(thumbFileName -> {
var filePath =
buildThumbnailStorePath(attachmentDirGetter.get(), thumbFileName, year, size);
var thumbnail = new LocalThumbnail();
thumbnail.setMetadata(new Metadata());
thumbnail.getMetadata().setGenerateName("thumbnail-");
var thumbnailUri = buildThumbnailUri(year, size, thumbFileName);
var thumbSignature = ThumbnailSigner.generateSignature(thumbnailUri);
thumbnail.setSpec(new LocalThumbnail.Spec()
.setImageSignature(signatureForImageUri(imageUri))
.setFilePath(toRelativeUnixPath(filePath))
.setImageUri(ensureInSiteUriIsRelative(imageUri).toASCIIString())
.setSize(size)
.setThumbSignature(thumbSignature)
.setThumbnailUri(thumbnailUri.toASCIIString()));
return client.create(thumbnail);
});
}
@Override
public Mono<Void> delete(URI imageUri) {
var signature = signatureForImageUri(imageUri);
return client.listAll(LocalThumbnail.class, ListOptions.builder()
.fieldQuery(and(
equal("spec.imageSignature", signature),
isNull("metadata.deletionTimestamp"))
)
.build(), Sort.unsorted())
.flatMap(thumbnail -> {
var filePath = toFilePath(thumbnail.getSpec().getFilePath());
return deleteFile(filePath)
.then(client.delete(thumbnail));
})
.then();
}
@Override
@NonNull
public URI ensureInSiteUriIsRelative(URI imageUri) {
Assert.notNull(imageUri, "Image URI must not be null.");
var externalUrl = externalUrlSupplier.getRaw();
if (externalUrl == null || !isSameOrigin(imageUri, externalUrl)) {
return imageUri;
}
var uriStr = imageUri.toString().replaceFirst("^\\w+://", "");
uriStr = StringUtils.removeStart(uriStr, imageUri.getAuthority());
return URI.create(uriStr);
}
Mono<String> generateUniqueThumbFileName(String originalFileName, String year,
ThumbnailSize size) {
Assert.notNull(originalFileName, "Original file name must not be null.");
return generateUniqueThumbFileName(originalFileName, originalFileName, year, size);
}
private Mono<String> generateUniqueThumbFileName(String originalFileName, String tryFileName,
String year, ThumbnailSize size) {
var thumbnailUri = buildThumbnailUri(year, size, tryFileName);
return fetchThumbnail(thumbnailUri)
.flatMap(thumbnail -> {
// use the original file name to generate a new file name
var newTryFileName = appendRandomSuffix(originalFileName);
return generateUniqueThumbFileName(originalFileName, newTryFileName, year, size);
})
.switchIfEmpty(Mono.just(tryFileName));
}
@Override
public Path toFilePath(String relativeUnixPath) {
Assert.notNull(relativeUnixPath, "Relative path must not be null.");
var systemPath = removeStart(relativeUnixPath, "/")
.replace("/", FileSystems.getDefault().getSeparator());
return attachmentDirGetter.get().resolve(systemPath);
}
@Override
public URI buildThumbnailUri(String year, ThumbnailSize size, String filename) {
return URI.create("/upload/thumbnails/%s/w%s/%s".formatted(year, size.getWidth(),
filename));
}
private String toRelativeUnixPath(Path filePath) {
var dir = attachmentDirGetter.get().toString();
var relativePath = removeStart(filePath.toString(), dir);
return relativePath.replace("\\", "/");
}
private Mono<Void> deleteFile(Path path) {
return Mono.fromRunnable(
() -> {
try {
Files.deleteIfExists(path);
} catch (Exception e) {
throw Exceptions.propagate(e);
}
})
.subscribeOn(Schedulers.boundedElastic())
.then();
}
/**
* Generate signature for the given image URI.
* <p>if externalUrl is not configured, it will return the signature generated by the image URI
* directly, otherwise, it will return the signature generated by the relative path of the
* image URL to the external URL.</p>
*/
String signatureForImageUri(URI imageUri) {
var uriToSign = ensureInSiteUriIsRelative(imageUri);
return ThumbnailSigner.generateSignature(uriToSign);
}
private boolean isSameOrigin(URI imageUri, URL externalUrl) {
return StringUtils.equals(imageUri.getHost(), externalUrl.getHost())
&& imageUri.getPort() == externalUrl.getPort();
}
static String appendRandomSuffix(String fileName) {
var baseName = StringUtils.substringBeforeLast(fileName, ".");
var extension = substringAfterLast(fileName, ".");
var randomSuffix = randomAlphanumeric(6);
return String.format("%s_%s.%s", baseName, randomSuffix, extension);
}
}

188
application/src/main/java/run/halo/app/core/attachment/reconciler/LocalThumbnailsReconciler.java
View File

@ -1,187 +1,73 @@
package run.halo.app.core.attachment.reconciler; package run.halo.app.core.attachment.reconciler;
import static org.springframework.data.domain.Sort.Order.desc; import static run.halo.app.extension.ExtensionUtil.addFinalizers;
import static run.halo.app.core.attachment.extension.LocalThumbnail.REQUEST_TO_GENERATE_ANNO; import static run.halo.app.extension.ExtensionUtil.removeFinalizers;
import static run.halo.app.extension.MetadataUtil.nullSafeAnnotations;
import static run.halo.app.extension.index.query.QueryFactory.and;
import static run.halo.app.extension.index.query.QueryFactory.equal;
import static run.halo.app.extension.index.query.QueryFactory.isNull;
import static run.halo.app.infra.utils.FileUtils.checkDirectoryTraversal;
import java.net.MalformedURLException; import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.util.Set;
import java.util.Optional;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component;
import org.springframework.data.domain.Sort; import org.springframework.util.StringUtils;
import run.halo.app.core.attachment.AttachmentRootGetter; import run.halo.app.core.attachment.AttachmentRootGetter;
import run.halo.app.core.attachment.AttachmentUtils;
import run.halo.app.core.attachment.LocalThumbnailService;
import run.halo.app.core.attachment.ThumbnailGenerator;
import run.halo.app.core.attachment.extension.LocalThumbnail; import run.halo.app.core.attachment.extension.LocalThumbnail;
import run.halo.app.core.extension.attachment.Attachment;
import run.halo.app.core.extension.attachment.Constant;
import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionUtil; import run.halo.app.extension.ExtensionUtil;
import run.halo.app.extension.ListOptions;
import run.halo.app.extension.PageRequestImpl;
import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.Controller;
import run.halo.app.extension.controller.ControllerBuilder; import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.Reconciler; import run.halo.app.extension.controller.Reconciler;
import run.halo.app.infra.ExternalLinkProcessor;
@Slf4j @Slf4j
// @Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class LocalThumbnailsReconciler implements Reconciler<Reconciler.Request> { class LocalThumbnailsReconciler implements Reconciler<Reconciler.Request> {
private final LocalThumbnailService localThumbnailService;
private static final String CLEAN_UP_FINALIZER = "thumbnail-cleaner";
private final ExtensionClient client; private final ExtensionClient client;
private final ExternalLinkProcessor externalLinkProcessor;
private final AttachmentRootGetter attachmentRootGetter; private final AttachmentRootGetter attachmentRoot;
@Override @Override
public Result reconcile(Request request) { public Result reconcile(Request request) {
client.fetch(LocalThumbnail.class, request.name()) client.fetch(LocalThumbnail.class, request.name())
.ifPresent(thumbnail -> { .ifPresent(thumbnail -> {
if (ExtensionUtil.isDeleted(thumbnail)) { if (ExtensionUtil.isDeleted(thumbnail)) {
return; if (removeFinalizers(thumbnail.getMetadata(), Set.of(CLEAN_UP_FINALIZER))) {
} // clean up thumbnail file
if (shouldGenerate(thumbnail)) { cleanUpThumbnailFile(thumbnail);
requestGenerateThumbnail(thumbnail);
nullSafeAnnotations(thumbnail).remove(REQUEST_TO_GENERATE_ANNO);
client.update(thumbnail); client.update(thumbnail);
} }
return;
}
// Cleanup all existing local thumbnails
addFinalizers(thumbnail.getMetadata(), Set.of(CLEAN_UP_FINALIZER));
log.info("Cleaning up local thumbnail: {}", thumbnail.getMetadata().getName());
client.delete(thumbnail);
}); });
return Result.doNotRetry(); return Result.doNotRetry();
} }
private boolean shouldGenerate(LocalThumbnail thumbnail) {
var annotations = nullSafeAnnotations(thumbnail);
return annotations.containsKey(REQUEST_TO_GENERATE_ANNO)
|| thumbnailFileNotExists(thumbnail);
}
private boolean thumbnailFileNotExists(LocalThumbnail thumbnail) {
var filePath = localThumbnailService.toFilePath(thumbnail.getSpec().getFilePath());
return !Files.exists(filePath);
}
void requestGenerateThumbnail(LocalThumbnail thumbnail) {
// If the thumbnail generation has failed, we should not retry it
if (isGenerationFailed(thumbnail)) {
return;
}
var imageUri = thumbnail.getSpec().getImageUri();
var filePath = localThumbnailService.toFilePath(thumbnail.getSpec().getFilePath());
if (Files.exists(filePath)) {
return;
}
var generator = new ThumbnailGenerator(thumbnail.getSpec().getSize(), filePath);
var imageUrlOpt = toImageUrl(imageUri);
if (imageUrlOpt.isEmpty()) {
if (tryGenerateByAttachment(imageUri, generator)) {
thumbnail.getStatus().setPhase(LocalThumbnail.Phase.SUCCEEDED);
} else {
log.debug("Failed to parse image URL，please check external-url configuration for "
+ "record: {}", thumbnail.getMetadata().getName());
thumbnail.getStatus().setPhase(LocalThumbnail.Phase.FAILED);
}
return;
}
var imageUrl = imageUrlOpt.get();
if (generateThumbnail(thumbnail, imageUrl, generator)) {
thumbnail.getStatus().setPhase(LocalThumbnail.Phase.SUCCEEDED);
} else {
thumbnail.getStatus().setPhase(LocalThumbnail.Phase.FAILED);
}
}
private boolean isGenerationFailed(LocalThumbnail thumbnail) {
return LocalThumbnail.Phase.FAILED.equals(thumbnail.getStatus().getPhase());
}
private boolean generateThumbnail(LocalThumbnail thumbnail, URL imageUrl,
ThumbnailGenerator generator) {
return tryGenerateByAttachment(thumbnail.getSpec().getImageUri(), generator)
|| tryGenerateByUrl(imageUrl, generator);
}
private boolean tryGenerate(String resourceIdentifier, Runnable generateAction) {
try {
generateAction.run();
return true;
} catch (Throwable e) {
log.debug("Failed to generate thumbnail for: {}", resourceIdentifier, e);
return false;
}
}
private boolean tryGenerateByUrl(URL imageUrl, ThumbnailGenerator generator) {
return tryGenerate(imageUrl.toString(), () -> {
log.debug("Generating thumbnail for image URL: {}", imageUrl);
generator.generate(imageUrl);
});
}
private boolean tryGenerateByAttachment(String imageUri, ThumbnailGenerator generator) {
return fetchAttachmentFilePath(imageUri)
.map(path -> tryGenerate(imageUri, () -> {
log.debug("Generating thumbnail for attachment file path: {}", path);
generator.generate(path);
}))
.orElse(false);
}
Optional<URL> toImageUrl(String imageUriStr) {
var imageUri = URI.create(imageUriStr);
try {
var url = new URL(externalLinkProcessor.processLink(imageUri.toString()));
return Optional.of(url);
} catch (MalformedURLException e) {
// Ignore
}
return Optional.empty();
}
Optional<Path> fetchAttachmentFilePath(String imageUri) {
return fetchAttachmentByPermalink(imageUri)
.filter(AttachmentUtils::isImage)
.map(attachment -> {
var annotations = nullSafeAnnotations(attachment);
var localRelativePath = annotations.get(Constant.LOCAL_REL_PATH_ANNO_KEY);
if (StringUtils.isBlank(localRelativePath)) {
return null;
}
var attachmentsRoot = attachmentRootGetter.get();
var filePath = attachmentsRoot.resolve(localRelativePath);
checkDirectoryTraversal(attachmentsRoot, filePath);
return filePath;
});
}
Optional<Attachment> fetchAttachmentByPermalink(String permalink) {
var listOptions = ListOptions.builder()
.fieldQuery(and(
equal("status.permalink", permalink),
isNull("metadata.deletionTimestamp")
))
.build();
var pageRequest = PageRequestImpl.ofSize(1)
.withSort(Sort.by(desc("metadata.creationTimestamp")));
return client.listBy(Attachment.class, listOptions, pageRequest)
.get()
.findFirst();
}
@Override @Override
public Controller setupWith(ControllerBuilder builder) { public Controller setupWith(ControllerBuilder builder) {
return builder return builder
.extension(new LocalThumbnail()) .extension(new LocalThumbnail())
.syncAllOnStart(true)
.build(); .build();
} }
private void cleanUpThumbnailFile(LocalThumbnail thumbnail) {
var filePath = thumbnail.getSpec().getFilePath();
if (StringUtils.hasText(filePath)) {
var thumbnailFile = attachmentRoot.get().resolve(filePath);
try {
if (Files.deleteIfExists(thumbnailFile)) {
log.info("Deleted thumbnail file: {} for {}",
thumbnailFile, thumbnail.getMetadata().getName());
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
} }

42
application/src/main/java/run/halo/app/core/attachment/reconciler/ThumbnailReconciler.java Normal file
View File

@ -0,0 +1,42 @@
package run.halo.app.core.attachment.reconciler;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import run.halo.app.core.attachment.extension.Thumbnail;
import run.halo.app.extension.ExtensionClient;
import run.halo.app.extension.ExtensionUtil;
import run.halo.app.extension.controller.Controller;
import run.halo.app.extension.controller.ControllerBuilder;
import run.halo.app.extension.controller.Reconciler;
@Slf4j
@Component
class ThumbnailReconciler implements Reconciler<Reconciler.Request> {
private final ExtensionClient client;
ThumbnailReconciler(ExtensionClient client) {
this.client = client;
}
@Override
public Result reconcile(Request request) {
client.fetch(Thumbnail.class, request.name())
.ifPresent(thumbnail -> {
if (ExtensionUtil.isDeleted(thumbnail)) {
return;
}
log.info("Clean up thumbnail: {}", thumbnail.getMetadata().getName());
client.delete(thumbnail);
});
return null;
}
@Override
public Controller setupWith(ControllerBuilder builder) {
return builder
.extension(new Thumbnail())
.build();
}
}

13
application/src/main/java/run/halo/app/core/endpoint/theme/ThumbnailEndpoint.java
View File

@ -5,8 +5,10 @@ import static org.springdoc.core.fn.builders.parameter.Builder.parameterBuilder;
import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.enums.ParameterIn;
import java.net.URI; import java.net.URI;
import java.util.Arrays;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springdoc.core.fn.builders.schema.Builder;
import org.springdoc.webflux.core.fn.SpringdocRouteBuilder; import org.springdoc.webflux.core.fn.SpringdocRouteBuilder;
import org.springframework.core.io.Resource; import org.springframework.core.io.Resource;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
@ -52,6 +54,7 @@ public class ThumbnailEndpoint implements CustomEndpoint {
.parameter(parameterBuilder() .parameter(parameterBuilder()
.in(ParameterIn.QUERY) .in(ParameterIn.QUERY)
.name("size") .name("size")
.implementation(ThumbnailSize.class) .implementation(ThumbnailSize.class)
.description("The size of the thumbnail") .description("The size of the thumbnail")
.required(true) .required(true)
@ -59,13 +62,19 @@ public class ThumbnailEndpoint implements CustomEndpoint {
.parameter(parameterBuilder() .parameter(parameterBuilder()
.in(ParameterIn.QUERY) .in(ParameterIn.QUERY)
.name("width") .name("width")
.schema(Builder.schemaBuilder()
.type("integer")
.allowableValues(Arrays.stream(ThumbnailSize.allowedWidths())
.map(String::valueOf)
.toArray(String[]::new)
)
)
.description(""" .description("""
The width of the thumbnail, if 'size' is not provided, this \ The width of the thumbnail, if 'size' is not provided, this \
parameter will be used to determine the size\ parameter will be used to determine the size\
""") """)
.required(false) .required(false)
) );
;
}) })
.build(); .build();
} }

96
application/src/test/java/run/halo/app/core/attachment/ThumbnailGeneratorTest.java
View File

@ -1,96 +0,0 @@
package run.halo.app.core.attachment;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Tests for {@link ThumbnailGenerator}.
*
* @author guqing
* @since 2.19.0
*/
@ExtendWith(MockitoExtension.class)
class ThumbnailGeneratorTest {
@Test
void sanitizeFileName() {
String sanitizedFileName = ThumbnailGenerator.sanitizeFileName("example.jpg");
assertThat(sanitizedFileName).isEqualTo("example.jpg");
sanitizedFileName = ThumbnailGenerator.sanitizeFileName("exampl./e$%^7*—=.jpg");
assertThat(sanitizedFileName).isEqualTo("exampl.e7.jpg");
}
@Nested
class ImageDownloaderTest {
private final ThumbnailGenerator.ImageDownloader imageDownloader =
new ThumbnailGenerator.ImageDownloader();
private Path tempFile;
@AfterEach
void tearDown() throws IOException {
if (tempFile != null && Files.exists(tempFile)) {
Files.delete(tempFile);
}
}
@Test
void testDownloadImage_Success() throws Exception {
var imageUrl = new URL("https://example.com/sample-image.jpg");
URL spyImageUrl = spy(imageUrl);
String mockImageData = "fakeImageData";
InputStream mockInputStream = new ByteArrayInputStream(mockImageData.getBytes());
var urlConnection = mock(HttpURLConnection.class);
doAnswer(invocation -> urlConnection).when(spyImageUrl).openConnection();
doReturn(mockInputStream).when(urlConnection).getInputStream();
var path = imageDownloader.downloadFileInternal(spyImageUrl);
assertThat(path).isNotNull();
tempFile = path;
assertThat(Files.exists(path)).isTrue();
try {
assertThat(Files.size(path)).isEqualTo(mockImageData.length());
} catch (IOException e) {
throw new RuntimeException(e);
}
String fileName = path.getFileName().toString();
assertThat(fileName).endsWith(".tmp");
}
@Test
void downloadImage_FileSizeLimitExceeded() throws Exception {
String largeImageUrl = "https://example.com/large-image.jpg";
URL spyImageUrl = spy(new URL(largeImageUrl));
// larger than MAX_FILE_SIZE
var fileSizeByte = ThumbnailGenerator.MAX_FILE_SIZE + 10;
byte[] largeImageData = new byte[fileSizeByte];
InputStream mockInputStream = new ByteArrayInputStream(largeImageData);
var urlConnection = mock(HttpURLConnection.class);
doAnswer(invocation -> urlConnection).when(spyImageUrl).openConnection();
doReturn(mockInputStream).when(urlConnection).getInputStream();
assertThatThrownBy(() -> imageDownloader.downloadFileInternal(spyImageUrl))
.isInstanceOf(IOException.class)
.hasMessageContaining("File size exceeds the limit");
}
}
}

141
application/src/test/java/run/halo/app/core/attachment/impl/LocalThumbnailServiceImplTest.java
View File

@ -1,141 +0,0 @@
package run.halo.app.core.attachment.impl;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static run.halo.app.core.attachment.ThumbnailSigner.generateSignature;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.web.util.UriUtils;
import reactor.core.publisher.Mono;
import reactor.test.StepVerifier;
import run.halo.app.core.attachment.AttachmentRootGetter;
import run.halo.app.core.attachment.ThumbnailSize;
import run.halo.app.core.attachment.extension.LocalThumbnail;
import run.halo.app.extension.ListResult;
import run.halo.app.extension.PageRequest;
import run.halo.app.extension.ReactiveExtensionClient;
import run.halo.app.infra.ExternalUrlSupplier;
/**
* Tests for {@link LocalThumbnailServiceImpl}.
*
* @author guqing
* @since 2.19.0
*/
@ExtendWith(MockitoExtension.class)
class LocalThumbnailServiceImplTest {
@Mock
private AttachmentRootGetter attachmentWorkDirGetter;
@Mock
private ExternalUrlSupplier externalUrlSupplier;
@Mock
private ReactiveExtensionClient client;
@InjectMocks
private LocalThumbnailServiceImpl localThumbnailService;
@Test
void endpointForTest() {
var endpoint =
localThumbnailService.buildThumbnailUri("2024", ThumbnailSize.L, "example.jpg");
assertThat(endpoint.toString()).isEqualTo("/upload/thumbnails/2024/w1200/example.jpg");
}
@Test
void geImageFileNameTest() throws MalformedURLException {
var fileName =
LocalThumbnailServiceImpl.geImageFileName(URI.create("https://halo.run/example.jpg"));
assertThat(fileName).isEqualTo("example.jpg");
fileName = LocalThumbnailServiceImpl.geImageFileName(URI.create("https://halo.run/"));
assertThat(fileName).isNotBlank();
var encoded = UriUtils.encode("https://halo.run/.1fasfg(*&^%$.jpg", StandardCharsets.UTF_8);
fileName = LocalThumbnailServiceImpl.geImageFileName(URI.create(encoded));
assertThat(fileName).isNotBlank();
}
@Test
void appendRandomSuffixTest() {
var result = LocalThumbnailServiceImpl.appendRandomSuffix("example.jpg");
assertThat(result).isNotEqualTo("example.jpg");
assertThat(result).endsWith(".jpg");
assertThat(result).startsWith("example_");
}
@Test
void toFilePathTest() {
when(attachmentWorkDirGetter.get()).thenReturn(Path.of("/tmp"));
var path = localThumbnailService.toFilePath("/thumbnails/2024/w1200/example.jpg");
assertThat(path).isEqualTo(Path.of("/tmp/thumbnails/2024/w1200/example.jpg"));
}
@Test
void signatureForImageUriTest() throws MalformedURLException {
when(externalUrlSupplier.getRaw()).thenReturn(new URL("http://localhost:8090"));
var signature = signatureForImageUriStr("http://localhost:8090/example.jpg");
assertThat(signature).isEqualTo(generateSignature("/example.jpg"));
signature = signatureForImageUriStr("http://localhost:8090/example.jpg");
assertThat(signature).isEqualTo(generateSignature("/example.jpg"));
signature = signatureForImageUriStr("http://localhost:8091/example.jpg");
assertThat(signature).isEqualTo(generateSignature("http://localhost:8091/example.jpg"));
signature = signatureForImageUriStr("localhost:8090/example.jpg");
assertThat(signature).isEqualTo(generateSignature("localhost:8090/example.jpg"));
when(externalUrlSupplier.getRaw()).thenReturn(null);
signature = signatureForImageUriStr("http://localhost:8090/example.jpg");
assertThat(signature).isEqualTo(generateSignature("http://localhost:8090/example.jpg"));
}
String signatureForImageUriStr(String uriStr) {
return localThumbnailService.signatureForImageUri(URI.create(uriStr));
}
@Test
void generateUniqueThumbFileNameTest() {
var count = new AtomicInteger(0);
when(client.listBy(eq(LocalThumbnail.class), any(), isA(PageRequest.class)))
.thenAnswer(invocation -> {
if (count.get() > 2) {
return Mono.just(ListResult.emptyResult());
}
count.incrementAndGet();
var result = new ListResult<>(List.of(new LocalThumbnail()));
return Mono.just(result);
});
localThumbnailService.generateUniqueThumbFileName("example.jpg", "2024", ThumbnailSize.L)
.as(StepVerifier::create)
.consumeNextWith(fileName -> {
assertThat(fileName).startsWith("example_");
assertThat(fileName).endsWith(".jpg");
// 6 is the length of the random suffix
assertThat(fileName.length()).isEqualTo("example_.jpg".length() + 6);
})
.verifyComplete();
verify(client, times(4)).listBy(eq(LocalThumbnail.class), any(), isA(PageRequest.class));
}
}